From ddde488d7bbb06faadc543177ecc952b871b4c94 Mon Sep 17 00:00:00 2001 From: Rebecca Corcillo <4566203+corcillo@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:11:20 -0700 Subject: [PATCH] Count retries, recoveries and exhausted retries per client A store or gRPC client that retries does it silently. When the retries run out the operation fails with the last error and nothing is logged, so an incident caused by a throttled backend (an Azure storage account at its egress limit, say) shows up only as failed actions, and how long the retries held out has to be reconstructed from sampled logs. Retrier now records retry.outcomes{retry.client, retry.outcome}: - retried: an attempt failed with a retryable error and another follows - recovered: the operation succeeded after at least one retry - exhausted: the retries ran out on a retryable error - permanent: an error that is never retried; NotFound is left out, since on a store it is the ordinary miss Each retrier is labeled with what it retries for: azure, s3, s3_http, gcs, ontap_s3, grpc_store, grpc_scheduler and connection. Running out of retries now also logs a warning naming the client and the attempt. --- nativelink-scheduler/src/grpc_scheduler.rs | 3 +- nativelink-store/src/azure_blob_store.rs | 3 +- nativelink-store/src/common_s3_utils.rs | 3 +- nativelink-store/src/gcs_store.rs | 3 +- nativelink-store/src/grpc_store.rs | 3 +- nativelink-store/src/ontap_s3_store.rs | 3 +- nativelink-store/src/s3_store.rs | 3 +- nativelink-util/src/connection_manager.rs | 3 +- nativelink-util/src/metrics.rs | 40 +++ nativelink-util/src/retry.rs | 80 +++++- nativelink-util/tests/retry_test.rs | 98 +++++++- .../docs/content/docs/reference/metrics.mdx | 228 ++++++++++-------- 12 files changed, 357 insertions(+), 113 deletions(-) diff --git a/nativelink-scheduler/src/grpc_scheduler.rs b/nativelink-scheduler/src/grpc_scheduler.rs index 431b5b3a888..62cba20e132 100644 --- a/nativelink-scheduler/src/grpc_scheduler.rs +++ b/nativelink-scheduler/src/grpc_scheduler.rs @@ -109,7 +109,8 @@ impl GrpcScheduler { Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn.clone(), spec.retry.clone(), - ), + ) + .with_client("grpc_scheduler"), connection_manager: ConnectionManager::new( core::iter::once(endpoint), spec.connections_per_endpoint, diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs index 2a834630062..fbedab577ad 100644 --- a/nativelink-store/src/azure_blob_store.rs +++ b/nativelink-store/src/azure_blob_store.rs @@ -197,7 +197,8 @@ where Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn, spec.common.retry.clone(), - ), + ) + .with_client("azure"), consider_expired_after_s: i64::from(spec.common.consider_expired_after_s), max_retry_buffer_per_request: spec .common diff --git a/nativelink-store/src/common_s3_utils.rs b/nativelink-store/src/common_s3_utils.rs index 6003e0ffbc7..fe86b53ee65 100644 --- a/nativelink-store/src/common_s3_utils.rs +++ b/nativelink-store/src/common_s3_utils.rs @@ -113,7 +113,8 @@ impl TlsClient { Arc::new(|duration| Box::pin(sleep(duration))), common.retry.make_jitter_fn(), common.retry.clone(), - ), + ) + .with_client("s3_http"), } } } diff --git a/nativelink-store/src/gcs_store.rs b/nativelink-store/src/gcs_store.rs index dd9731a5cc5..a82945167b0 100644 --- a/nativelink-store/src/gcs_store.rs +++ b/nativelink-store/src/gcs_store.rs @@ -136,7 +136,8 @@ where Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn, spec.common.retry.clone(), - ), + ) + .with_client("gcs"), consider_expired_after_s: u64::from(spec.common.consider_expired_after_s), max_retry_buffer_size, max_chunk_size, diff --git a/nativelink-store/src/grpc_store.rs b/nativelink-store/src/grpc_store.rs index bfc96920062..cf49ea35b35 100644 --- a/nativelink-store/src/grpc_store.rs +++ b/nativelink-store/src/grpc_store.rs @@ -341,7 +341,8 @@ impl GrpcStore { Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn.clone(), spec.retry.clone(), - ), + ) + .with_client("grpc_store"), connection_manager: ConnectionManager::new( endpoints, spec.connections_per_endpoint, diff --git a/nativelink-store/src/ontap_s3_store.rs b/nativelink-store/src/ontap_s3_store.rs index be7868de681..401c01d17f6 100644 --- a/nativelink-store/src/ontap_s3_store.rs +++ b/nativelink-store/src/ontap_s3_store.rs @@ -204,7 +204,8 @@ where Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn, spec.common.retry.clone(), - ), + ) + .with_client("ontap_s3"), consider_expired_after_s: u64::from(spec.common.consider_expired_after_s), max_retry_buffer_per_request: spec .common diff --git a/nativelink-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs index 18d9a57c042..32ede2e16ef 100644 --- a/nativelink-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -201,7 +201,8 @@ where Arc::new(|duration| Box::pin(sleep(duration))), jitter_fn, spec.common.retry.clone(), - ), + ) + .with_client("s3"), consider_expired_after_s: i64::from(spec.common.consider_expired_after_s), max_retry_buffer_per_request: spec .common diff --git a/nativelink-util/src/connection_manager.rs b/nativelink-util/src/connection_manager.rs index 9dfad7bdcfb..753e7cb956a 100644 --- a/nativelink-util/src/connection_manager.rs +++ b/nativelink-util/src/connection_manager.rs @@ -169,7 +169,8 @@ impl ConnectionManager { Arc::new(|duration| Box::pin(tokio::time::sleep(duration))), jitter_fn, retry, - ), + ) + .with_client("connection"), }; background_spawn!("connection_manager_worker_spawn", async move { worker diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index 25d2389425c..6ea3e4218b2 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -1453,3 +1453,43 @@ pub fn record_connection_reconnect(pool: &'static str) { .reconnects .add(1, &[KeyValue::new(CONNECTION_POOL, pool)]); } + +// Metric attribute keys for retried operations. +pub const RETRY_CLIENT: &str = "retry.client"; +pub const RETRY_OUTCOME: &str = "retry.outcome"; + +/// Global metrics for operations run under a [`crate::retry::Retrier`]. +pub static RETRY_METRICS: LazyLock = LazyLock::new(|| { + let meter = global::meter_with_scope(InstrumentationScope::builder("nativelink").build()); + + RetryMetrics { + outcomes: meter + .u64_counter("retry.outcomes") + .with_description( + "Retried operations by client and outcome: each retry (retried), a success after at least one retry (recovered), retries run out (exhausted), or an error that is never retried, NotFound aside (permanent)", + ) + .with_unit("{event}") + .build(), + } +}); + +/// Metrics for operations run under a [`crate::retry::Retrier`]. +#[derive(Debug)] +pub struct RetryMetrics { + /// Retry events, by client and outcome. + pub outcomes: metrics::Counter, +} + +/// Records one retry event. `client` names what is retrying (`azure`, `s3`, +/// `grpc_store`, ...), `outcome` is one of `retried`, `recovered`, +/// `exhausted` or `permanent`. An `exhausted` count rising is a backend that +/// stayed down, or throttled, for longer than the retry policy waits. +pub fn record_retry_outcome(client: &'static str, outcome: &'static str) { + RETRY_METRICS.outcomes.add( + 1, + &[ + KeyValue::new(RETRY_CLIENT, client), + KeyValue::new(RETRY_OUTCOME, outcome), + ], + ); +} diff --git a/nativelink-util/src/retry.rs b/nativelink-util/src/retry.rs index 1cd0ba9fb14..f3a3952ebc1 100644 --- a/nativelink-util/src/retry.rs +++ b/nativelink-util/src/retry.rs @@ -20,7 +20,9 @@ use futures::future::Future; use futures::stream::StreamExt; use nativelink_config::stores::{ErrorCode, Retry}; use nativelink_error::{Code, Error, make_err}; -use tracing::{error, info}; +use tracing::{error, info, warn}; + +use crate::metrics::record_retry_outcome; struct ExponentialBackoff { current: Duration, @@ -51,18 +53,49 @@ pub enum RetryResult { Err(Error), } +/// What became of an operation, or one attempt of it, under a [`Retrier`]. +/// Recorded as `retry.outcomes`, labeled with the retrier's client. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RetryOutcome { + /// An attempt failed with a retryable error and another follows. + Retried, + /// The operation succeeded after at least one retry. + Recovered, + /// The operation failed with a retryable error and the retries ran out. + Exhausted, + /// The operation failed with an error that is never retried. `NotFound` + /// is left out: on a store it is the ordinary miss. + Permanent, +} + +impl RetryOutcome { + const fn as_str(self) -> &'static str { + match self { + Self::Retried => "retried", + Self::Recovered => "recovered", + Self::Exhausted => "exhausted", + Self::Permanent => "permanent", + } + } +} + +type OutcomeFn = Arc; + /// Class used to retry a job with a sleep function in between each retry. #[derive(Clone)] pub struct Retrier { sleep_fn: SleepFn, jitter_fn: JitterFn, config: Retry, + client: &'static str, + outcome_fn: Option, } impl core::fmt::Debug for Retrier { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("Retrier") .field("config", &self.config) + .field("client", &self.client) .finish_non_exhaustive() } } @@ -94,6 +127,31 @@ impl Retrier { sleep_fn, jitter_fn, config, + client: "unknown", + outcome_fn: None, + } + } + + /// Names what this retrier retries for (`azure`, `s3`, `grpc_store`, + /// ...), the `retry.client` label on its `retry.outcomes`. + #[must_use] + pub const fn with_client(mut self, client: &'static str) -> Self { + self.client = client; + self + } + + /// Also hands every outcome to `outcome_fn`, after it is recorded as a + /// metric. For tests, which cannot read the meter. + #[must_use] + pub fn with_outcome_fn(mut self, outcome_fn: OutcomeFn) -> Self { + self.outcome_fn = Some(outcome_fn); + self + } + + fn report(&self, outcome: RetryOutcome) { + record_retry_outcome(self.client, outcome.as_str()); + if let Some(outcome_fn) = &self.outcome_fn { + outcome_fn(outcome); } } @@ -156,7 +214,12 @@ impl Retrier { "Retry stream ended abruptly on attempt {attempt}", )); } - Some(RetryResult::Ok(value)) => return Ok(value), + Some(RetryResult::Ok(value)) => { + if attempt > 1 { + self.report(RetryOutcome::Recovered); + } + return Ok(value); + } Some(RetryResult::Err(e)) => { return Err(e.append(format!("On attempt {attempt}"))); } @@ -165,15 +228,18 @@ impl Retrier { if err.code == Code::NotFound { info!(?err, "Not found, not retrying"); } else { + self.report(RetryOutcome::Permanent); error!(?attempt, ?err, "Not retrying permanent error"); } return Err(err); } - (self.sleep_fn)( - iter.next() - .ok_or_else(|| err.append(format!("On attempt {attempt}")))?, - ) - .await; + let Some(delay) = iter.next() else { + self.report(RetryOutcome::Exhausted); + warn!(client = self.client, ?attempt, ?err, "Retries exhausted"); + return Err(err.append(format!("On attempt {attempt}"))); + }; + self.report(RetryOutcome::Retried); + (self.sleep_fn)(delay).await; } } } diff --git a/nativelink-util/tests/retry_test.rs b/nativelink-util/tests/retry_test.rs index cfb8b07b244..f5508e1fd37 100644 --- a/nativelink-util/tests/retry_test.rs +++ b/nativelink-util/tests/retry_test.rs @@ -14,14 +14,14 @@ use core::pin::Pin; use core::sync::atomic::{AtomicI32, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use futures::future::ready; use futures::stream::repeat_with; use nativelink_config::stores::Retry; use nativelink_error::{Code, Error, make_err}; use nativelink_macro::nativelink_test; -use nativelink_util::retry::{Retrier, RetryResult}; +use nativelink_util::retry::{Retrier, RetryOutcome, RetryResult}; use pretty_assertions::assert_eq; use tokio::time::Duration; @@ -211,3 +211,97 @@ async fn retry_calls_sleep_fn() -> Result<(), Error> { Ok(()) } + +/// A retrier whose outcomes land in the returned list, with `max_retries`. +fn observed_retrier(max_retries: usize) -> (Retrier, Arc>>) { + let outcomes = Arc::new(Mutex::new(Vec::new())); + let sink = outcomes.clone(); + let retrier = Retrier::new( + Arc::new(|_duration| Box::pin(ready(()))), + Arc::new(move |_delay| Duration::from_millis(1)), + Retry { + max_retries, + ..Default::default() + }, + ) + .with_client("test") + .with_outcome_fn(Arc::new(move |outcome| sink.lock().unwrap().push(outcome))); + (retrier, outcomes) +} + +/// Each retry is reported, and a success after them is a recovery; a success +/// on the first attempt reports nothing. +#[nativelink_test] +async fn retry_reports_retries_and_recovery() -> Result<(), Error> { + let (retrier, outcomes) = observed_retrier(5); + let run_count = AtomicI32::new(0); + Pin::new(&retrier) + .retry(repeat_with(|| { + if run_count.fetch_add(1, Ordering::Relaxed) < 2 { + RetryResult::Retry(make_err!(Code::Unavailable, "Dummy failure")) + } else { + RetryResult::Ok(true) + } + })) + .await?; + assert_eq!( + *outcomes.lock().unwrap(), + vec![ + RetryOutcome::Retried, + RetryOutcome::Retried, + RetryOutcome::Recovered + ] + ); + + let (retrier, outcomes) = observed_retrier(5); + Pin::new(&retrier) + .retry(repeat_with(|| RetryResult::Ok(true))) + .await?; + assert_eq!(*outcomes.lock().unwrap(), vec![]); + Ok(()) +} + +/// Retries that run out are reported as exhausted after the last retry. +#[nativelink_test] +async fn retry_reports_exhaustion() -> Result<(), Error> { + let (retrier, outcomes) = observed_retrier(2); + let result = Pin::new(&retrier) + .retry(repeat_with(|| { + RetryResult::::Retry(make_err!(Code::Unavailable, "Dummy failure")) + })) + .await; + assert!(result.is_err()); + assert_eq!( + *outcomes.lock().unwrap(), + vec![ + RetryOutcome::Retried, + RetryOutcome::Retried, + RetryOutcome::Exhausted + ] + ); + Ok(()) +} + +/// An error that is never retried is reported as permanent, except +/// `NotFound`, which on a store is an ordinary miss. +#[nativelink_test] +async fn retry_reports_permanent_errors_but_not_misses() -> Result<(), Error> { + let (retrier, outcomes) = observed_retrier(5); + let result = Pin::new(&retrier) + .retry(repeat_with(|| { + RetryResult::::Retry(make_err!(Code::PermissionDenied, "Dummy failure")) + })) + .await; + assert!(result.is_err()); + assert_eq!(*outcomes.lock().unwrap(), vec![RetryOutcome::Permanent]); + + let (retrier, outcomes) = observed_retrier(5); + let result = Pin::new(&retrier) + .retry(repeat_with(|| { + RetryResult::::Retry(make_err!(Code::NotFound, "Dummy miss")) + })) + .await; + assert!(result.is_err()); + assert_eq!(*outcomes.lock().unwrap(), vec![]); + Ok(()) +} diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index 3b40409eb09..9cff393377c 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,12 +5,12 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.2 (838aeade6) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (a435beba) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} -NativeLink **v1.7.2** declares 40 OpenTelemetry instruments. This page is +NativeLink **v1.7.6** declares 42 OpenTelemetry instruments. This page is generated from the declarations in -[`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs) +[`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs) and from every `.add()` and `.record()` call site across the workspace, so the **Emitted** column is not a claim, it is the result of looking. @@ -43,6 +43,7 @@ wrong. | [`worker.connected.count`](#worker-connected-count) | `nativelink_worker_connected_count` (unreferenced) | UpDownCounter | Conditionally | scheduler process only | | [`worker.connections`](#worker-connections) | `nativelink_worker_connections` (unreferenced) | Counter | Conditionally | scheduler process only | | [`worker.disconnections`](#worker-disconnections) | `nativelink_worker_disconnections` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`worker.input.relinks`](#worker-input-relinks) | `nativelink_worker_input_relinks` (unreferenced) | Counter | Yes | whenever the code path runs | | [`scheduler.dispatch.requeues`](#scheduler-dispatch-requeues) | `nativelink_scheduler_dispatch_requeues` (unreferenced) | Counter | Conditionally | scheduler process only | | [`worker.keepalives`](#worker-keepalives) | `nativelink_worker_keepalives` (unreferenced) | Counter | Conditionally | scheduler process only | | [`worker.keepalive.gap`](#worker-keepalive-gap) | `nativelink_worker_keepalive_gap` (unreferenced) | Histogram | Conditionally | scheduler process only | @@ -64,6 +65,7 @@ wrong. | [`connection.pool.available`](#connection-pool-available) | `nativelink_connection_pool_available` (unreferenced) | Histogram | Yes | whenever the code path runs | | [`connection.pool.acquisitions`](#connection-pool-acquisitions) | `nativelink_connection_pool_acquisitions` (unreferenced) | Counter | Yes | whenever the code path runs | | [`connection.reconnects`](#connection-reconnects) | `nativelink_connection_reconnects` (unreferenced) | Counter | Yes | whenever the code path runs | +| [`retry.outcomes`](#retry-outcomes) | `nativelink_retry_outcomes` (unreferenced) | Counter | Yes | whenever the code path runs | ## How an instrument becomes a Prometheus series @@ -72,7 +74,7 @@ binary. The names in the middle column above are what the shipped OpenTelemetry collector produces after two transformations: 1. The Prometheus exporter in - [`otel-collector-config.yaml`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/deployment-examples/metrics/otel-collector-config.yaml) + [`otel-collector-config.yaml`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/deployment-examples/metrics/otel-collector-config.yaml) is configured with `namespace: nativelink`, which prefixes every series with `nativelink_`. 2. The exporter rewrites the OTLP name into Prometheus form: dots become @@ -99,7 +101,7 @@ Every instrument declared in `metrics.rs` has at least one call site. Global cache metrics instruments. -Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:438`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L438), backed by the `CacheMetrics` struct. +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:438`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L438), backed by the `CacheMetrics` struct. Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. @@ -114,8 +116,8 @@ Histogram of cache operation durations in milliseconds | Unit | `ms` | | Prometheus series | `nativelink_cache_operation_duration` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L442) | -| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L73) | +| Declared at | [`nativelink-util/src/metrics.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L442) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) | **Histogram buckets:** 19 explicit boundaries, from `0.001` to `5000.0` `ms`. Anything past the last boundary lands in the @@ -133,8 +135,8 @@ Counter of cache operations by type and result | Unit | — | | Prometheus series | `nativelink_cache_operations_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:476`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L476) | -| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L260) | +| Declared at | [`nativelink-util/src/metrics.rs:476`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L476) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) | ### `cache.io` @@ -147,8 +149,8 @@ Counter of bytes read/written during cache operations | Unit | `By` | | Prometheus series | `nativelink_cache_io_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:481`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L481) | -| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L248) | +| Declared at | [`nativelink-util/src/metrics.rs:481`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L481) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) | ### `cache.size` @@ -161,8 +163,8 @@ Current total size of all cached data in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_size` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L487) | -| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/evicting_map.rs#L468) | +| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L487) | +| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.entries` @@ -175,8 +177,8 @@ Current number of entries in cache | Unit | `{entry}` | | Prometheus series | `nativelink_cache_entries` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:493`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L493) | -| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/evicting_map.rs#L468) | +| Declared at | [`nativelink-util/src/metrics.rs:493`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L493) | +| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.item.size` @@ -189,14 +191,14 @@ Histogram of individual cache entry sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_item_size` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:499`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L499) | -| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs#L83) | +| Declared at | [`nativelink-util/src/metrics.rs:499`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L499) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) | ## Execution metrics Global remote execution metrics instruments. -Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L539), backed by the `ExecutionMetrics` struct. +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L539), backed by the `ExecutionMetrics` struct. Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. @@ -211,8 +213,8 @@ Histogram of stage durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:543`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L543) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:109`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L109) | +| Declared at | [`nativelink-util/src/metrics.rs:543`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L543) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L150) | **Histogram buckets:** 15 explicit boundaries, from `0.001` to `3600.0` `s`. Anything past the last boundary lands in the @@ -230,8 +232,8 @@ Histogram of total execution durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:568`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L568) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:109`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L109) | +| Declared at | [`nativelink-util/src/metrics.rs:568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L568) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L150) | **Histogram buckets:** 13 explicit boundaries, from `0.01` to `7200.0` `s`. Anything past the last boundary lands in the @@ -249,8 +251,8 @@ Histogram of queue wait times in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:593`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L593) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:109`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L109) | +| Declared at | [`nativelink-util/src/metrics.rs:593`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L593) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L150) | **Histogram buckets:** 12 explicit boundaries, from `0.001` to `600.0` `s`. Anything past the last boundary lands in the @@ -268,8 +270,8 @@ Current number of actions in each stage | Unit | `{action}` | | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:613`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L613) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | +| Declared at | [`nativelink-util/src/metrics.rs:613`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L613) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | ### `execution.completed.count` @@ -282,8 +284,8 @@ Total number of completed executions | Unit | `{action}` | | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:619`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L619) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:108`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L108), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:118`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L118) | +| Declared at | [`nativelink-util/src/metrics.rs:619`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L619) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:149`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L149), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:159`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L159) | ### `execution.stage.transitions` @@ -296,8 +298,8 @@ Number of stage transitions | Unit | `{transition}` | | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:625`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L625) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:97`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L97) | +| Declared at | [`nativelink-util/src/metrics.rs:625`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L625) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:138`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L138) | ### `execution.cpu.time` @@ -310,8 +312,8 @@ CPU time an action used, as sampled by the worker. | Unit | `s` | | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:631`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L631) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1784`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1784) | +| Declared at | [`nativelink-util/src/metrics.rs:631`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L631) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1911`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1911) | **Histogram buckets:** 12 explicit boundaries, from `0.1` to `7200.0` `s`. Anything past the last boundary lands in the @@ -329,8 +331,8 @@ Peak memory an action used, as sampled by the worker. | Unit | `By` | | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:640`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L640) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1776`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1776) | +| Declared at | [`nativelink-util/src/metrics.rs:640`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L640) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | **Histogram buckets:** 8 explicit boundaries, from `1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the @@ -348,8 +350,8 @@ Histogram of output sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:656`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L656) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:109`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L109) | +| Declared at | [`nativelink-util/src/metrics.rs:656`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L656) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L150) | **Histogram buckets:** 8 explicit boundaries, from `1_024.0` to `10_737_418_240.0` `By`. Anything past the last boundary lands in the @@ -367,14 +369,14 @@ Counter for execution retries | Unit | `{retry}` | | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L672) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:130`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L130) | +| Declared at | [`nativelink-util/src/metrics.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L672) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:171`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L171) | ## Worker fleet metrics Global worker fleet metrics instruments. -Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L881), backed by the `WorkerMetrics` struct. +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L881), backed by the `WorkerMetrics` struct. Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. @@ -389,8 +391,8 @@ Workers currently connected. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:885`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L885) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L442), [`nativelink-scheduler/src/api_worker_scheduler.rs:1378`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1378) | +| Declared at | [`nativelink-util/src/metrics.rs:885`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L885) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464), [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | ### `worker.connections` @@ -403,8 +405,8 @@ Workers that have joined, cumulative. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L891) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L442) | +| Declared at | [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L891) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464) | ### `worker.disconnections` @@ -417,8 +419,22 @@ Workers that have left, cumulative, by reason. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:897`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L897) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1378`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1378) | +| Declared at | [`nativelink-util/src/metrics.rs:897`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L897) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | + +### `worker.input.relinks` + +Input hardlinks made again after their source was evicted mid-staging, by kind. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `input_relinks` | +| Unit | — | +| Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L903) | +| Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1029`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1029), [`nativelink-worker/src/running_actions_manager.rs:1039`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1039) | ### `scheduler.dispatch.requeues` @@ -431,8 +447,8 @@ Dispatches requeued untried, by reason. | Unit | — | | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:903`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L903) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1137`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1137), [`nativelink-scheduler/src/api_worker_scheduler.rs:1574`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1574), [`nativelink-scheduler/src/api_worker_scheduler.rs:1872`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1872), [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | +| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L908) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1187`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1187), [`nativelink-scheduler/src/api_worker_scheduler.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1381), [`nativelink-scheduler/src/api_worker_scheduler.rs:1701`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1701), [`nativelink-scheduler/src/api_worker_scheduler.rs:1999`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1999), [`nativelink-scheduler/src/api_worker_scheduler.rs:2030`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2030), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:673`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L673) | ### `worker.keepalives` @@ -445,8 +461,8 @@ Keepalives received, cumulative. | Unit | `{keepalive}` | | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L908) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:400`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L400) | +| Declared at | [`nativelink-util/src/metrics.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L913) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:422`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L422) | ### `worker.keepalive.gap` @@ -459,8 +475,8 @@ Seconds between consecutive messages from a worker. | Unit | `s` | | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L914) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L348) | +| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:356`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L356) | **Histogram buckets:** 12 explicit boundaries, from `1.0` to `60.0` `s`. Anything past the last boundary lands in the @@ -478,8 +494,8 @@ Connected workers currently paused or draining. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L925) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:383`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L383), [`nativelink-scheduler/src/api_worker_scheduler.rs:436`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L436), [`nativelink-scheduler/src/api_worker_scheduler.rs:439`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L439), [`nativelink-scheduler/src/api_worker_scheduler.rs:549`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L549), [`nativelink-scheduler/src/api_worker_scheduler.rs:1098`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1098), [`nativelink-scheduler/src/api_worker_scheduler.rs:1140`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1140), [`nativelink-scheduler/src/api_worker_scheduler.rs:1276`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/api_worker_scheduler.rs#L1276) | +| Declared at | [`nativelink-util/src/metrics.rs:930`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L930) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:405`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L405), [`nativelink-scheduler/src/api_worker_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L458), [`nativelink-scheduler/src/api_worker_scheduler.rs:461`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L461), [`nativelink-scheduler/src/api_worker_scheduler.rs:571`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L571), [`nativelink-scheduler/src/api_worker_scheduler.rs:1148`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1148), [`nativelink-scheduler/src/api_worker_scheduler.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1190), [`nativelink-scheduler/src/api_worker_scheduler.rs:1326`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1326) | ## gRPC server metrics @@ -488,7 +504,7 @@ Global gRPC serving metrics. One duration histogram covers rate, errors and latency: the count gives rate, the status attribute separates errors, and the buckets give latency. -Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1025`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1025), backed by the `RpcMetrics` struct. +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1041`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1041), backed by the `RpcMetrics` struct. Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. @@ -503,8 +519,8 @@ Duration of inbound gRPC calls, by service, method and status. | Unit | `s` | | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1029`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1029) | -| Emitted from | [`nativelink-util/src/telemetry.rs:375`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/telemetry.rs#L375) | +| Declared at | [`nativelink-util/src/metrics.rs:1045`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1045) | +| Emitted from | [`nativelink-util/src/telemetry.rs:375`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L375) | **Histogram buckets:** 15 explicit boundaries, from `0.001` to `300.0` `s`. Anything past the last boundary lands in the @@ -515,7 +531,7 @@ measurement. Global scheduler metrics. -Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1078`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1078), backed by the `SchedulerOtlpMetrics` struct. +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1094`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1094), backed by the `SchedulerOtlpMetrics` struct. Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. @@ -531,8 +547,8 @@ while the queue is non-empty means matching is the bottleneck. | Unit | `s` | | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1082`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1082) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:449`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L449) | +| Declared at | [`nativelink-util/src/metrics.rs:1098`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1098) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | **Histogram buckets:** 13 explicit boundaries, from `0.0001` to `10.0` `s`. Anything past the last boundary lands in the @@ -550,8 +566,8 @@ Matching passes, by result. | Unit | `{pass}` | | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1091`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1091) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:449`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L449) | +| Declared at | [`nativelink-util/src/metrics.rs:1107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1107) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | ### `scheduler.unsatisfiable.queued` @@ -566,8 +582,8 @@ instance, judged against that instance's own workers. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1097`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1097) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1019`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L1019) | +| Declared at | [`nativelink-util/src/metrics.rs:1113`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1113) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1028`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1028) | ### `scheduler.unsatisfiable.failed` @@ -581,8 +597,8 @@ the properties that could not be satisfied. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1105`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1105) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:551`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L551) | +| Declared at | [`nativelink-util/src/metrics.rs:1121`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1121) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L560) | ### `scheduler.queue.depth` @@ -595,8 +611,8 @@ Queued actions the last matching pass read. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1111`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1111) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1020`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L1020) | +| Declared at | [`nativelink-util/src/metrics.rs:1127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1127) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1029`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1029) | ### `scheduler.queue.retired` @@ -609,8 +625,8 @@ Queued actions retired for having no client. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1117`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1117) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:597`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L597) | +| Declared at | [`nativelink-util/src/metrics.rs:1133`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1133) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:777`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L777) | ### `scheduler.matching.parked_dispatched` @@ -624,8 +640,8 @@ opened before the pass ended, ahead of the newer actions it was reading. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1123`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1123) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:996`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L996) | +| Declared at | [`nativelink-util/src/metrics.rs:1139`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1139) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1005`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1005) | ### `scheduler.awaited_action.orphans` @@ -638,8 +654,8 @@ Listed operations whose record was gone on read, by site. | Unit | `{operation}` | | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1131`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1131) | -| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:591`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L591), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:545`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L545), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1467`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1467), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1490`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1490) | +| Declared at | [`nativelink-util/src/metrics.rs:1147`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1147) | +| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:600`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L600), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:611`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L611), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:725`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L725), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1656`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1656), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1679) | ### `scheduler.sweep.failures` @@ -652,8 +668,8 @@ Abandoned-queue sweeps that failed before finishing. | Unit | `{sweep}` | | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1139`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1139) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1241`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler.rs#L1241) | +| Declared at | [`nativelink-util/src/metrics.rs:1155`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1155) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1250`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1250), [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | ### `scheduler.hints.resolutions` @@ -666,14 +682,14 @@ Actions sized by the historical resource scheduler, by source. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1145`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1145) | -| Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:513`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/historical_resource_scheduler.rs#L513), [`nativelink-scheduler/src/historical_resource_scheduler.rs:517`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/historical_resource_scheduler.rs#L517), [`nativelink-scheduler/src/historical_resource_scheduler.rs:520`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/historical_resource_scheduler.rs#L520) | +| Declared at | [`nativelink-util/src/metrics.rs:1161`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1161) | +| Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1264`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1264), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1280`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1280), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -688,8 +704,8 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1268`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1268) | -| Emitted from | [`nativelink-store/src/fast_slow_store.rs:340`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L340), [`nativelink-store/src/fast_slow_store.rs:807`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L807), [`nativelink-store/src/fast_slow_store.rs:823`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L823), [`nativelink-store/src/fast_slow_store.rs:833`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L833), [`nativelink-store/src/fast_slow_store.rs:858`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L858), [`nativelink-store/src/fast_slow_store.rs:880`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L880) | +| Declared at | [`nativelink-util/src/metrics.rs:1284`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1284) | +| Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -702,14 +718,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1274`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1274) | -| Emitted from | [`nativelink-store/src/fast_slow_store.rs:349`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L349), [`nativelink-store/src/fast_slow_store.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L824), [`nativelink-store/src/fast_slow_store.rs:859`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L859), [`nativelink-store/src/fast_slow_store.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/fast_slow_store.rs#L881) | +| Declared at | [`nativelink-util/src/metrics.rs:1290`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1290) | +| Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1315`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1315), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1331`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1331), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -724,14 +740,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1319`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1319) | -| Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/health_utils.rs#L248) | +| Declared at | [`nativelink-util/src/metrics.rs:1335`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1335) | +| Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1346`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1346), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1362`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1362), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -746,8 +762,8 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1350`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1350) | -| Emitted from | [`nativelink-store/src/redis_store.rs:559`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/redis_store.rs#L559), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/connection_manager.rs#L351) | +| Declared at | [`nativelink-util/src/metrics.rs:1366`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1366) | +| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L352) | **Histogram buckets:** 11 explicit boundaries, from `0.0` to `512.0` `{connection}`. Anything past the last boundary lands in the @@ -765,8 +781,8 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1359`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1359) | -| Emitted from | [`nativelink-store/src/redis_store.rs:559`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/redis_store.rs#L559), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/connection_manager.rs#L351) | +| Declared at | [`nativelink-util/src/metrics.rs:1375`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1375) | +| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L352) | ### `connection.reconnects` @@ -779,8 +795,28 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1365`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs#L1365) | -| Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/redis_store.rs#L319) | +| Declared at | [`nativelink-util/src/metrics.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1381) | +| Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | + +## RETRY_METRICS + +Global metrics for operations run under a [`crate::retry::Retrier`]. + +Declared as `RETRY_METRICS` in [`nativelink-util/src/metrics.rs:1462`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1462), backed by the `RetryMetrics` struct. + +### `retry.outcomes` + +Retry events, by client and outcome. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `outcomes` | +| Unit | `{event}` | +| Prometheus series | `nativelink_retry_outcomes` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:1466`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1466) | +| Emitted from | [`nativelink-util/src/retry.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/retry.rs#L152) | ## Attributes @@ -897,10 +933,10 @@ underscores. If anything here disagrees with the binary, the source wins: -- [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/metrics.rs): every declaration -- [`nativelink-util/src/telemetry.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval -- [`nativelink-store/src/cache_metrics_store.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-store/src/cache_metrics_store.rs): every cache call site -- [`nativelink-scheduler/src/simple_scheduler_state_manager.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.2/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site +- [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs): every declaration +- [`nativelink-util/src/telemetry.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval +- [`nativelink-store/src/cache_metrics_store.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs): every cache call site +- [`nativelink-scheduler/src/simple_scheduler_state_manager.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site ## Where to go next