From ac1f618e30bace2e80f59109ff22fa1f48370836 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 13 Oct 2026 14:53:33 -0400 Subject: [PATCH 1/2] Record the input fetch retry outcome under a prefixed attribute --- nativelink-util/src/metrics.rs | 3 +- .../docs/content/docs/reference/metrics.mdx | 115 +++++++++--------- 2 files changed, 60 insertions(+), 58 deletions(-) diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index f44b024cd3..92150e99fd 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -46,6 +46,7 @@ pub const RPC_STATUS_CODE: &str = "rpc.grpc.status_code"; pub const SCHEDULER_MATCH_RESULT: &str = "scheduler.match.result"; /// The platform properties no worker could satisfy, comma separated. pub const SCHEDULER_UNSATISFIABLE_PROPERTIES: &str = "scheduler.unsatisfiable.properties"; +pub const WORKER_PREPARE_OUTCOME: &str = "worker.prepare.outcome"; // Metric attribute keys for tiered stores. pub const STORE_TIER: &str = "store.tier"; @@ -1032,7 +1033,7 @@ pub fn record_input_relink(kind: &'static str) { pub fn record_prepare_retry(outcome: &'static str) { WORKER_METRICS .prepare_retries - .add(1, &[KeyValue::new("outcome", outcome)]); + .add(1, &[KeyValue::new(WORKER_PREPARE_OUTCOME, outcome)]); } /// Records a keepalive from a worker. diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index 4b5b19ea27..d5939c96f3 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.6 (d0e6aa8e3) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (1911c9c30) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} NativeLink **v1.7.6** declares 47 OpenTelemetry instruments. This page is @@ -106,7 +106,7 @@ Every instrument declared in `metrics.rs` has at least one call site. Global cache metrics instruments. -Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:438`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L438), backed by the `CacheMetrics` struct. +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:439`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L439), backed by the `CacheMetrics` struct. Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. @@ -121,7 +121,7 @@ Histogram of cache operation durations in milliseconds | Unit | `ms` | | Prometheus series | `nativelink_cache_operation_duration` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L442) | +| Declared at | [`nativelink-util/src/metrics.rs:443`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L443) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) | **Histogram buckets:** 19 explicit boundaries, from @@ -140,7 +140,7 @@ Counter of cache operations by type and result | Unit | — | | Prometheus series | `nativelink_cache_operations_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:476`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L476) | +| Declared at | [`nativelink-util/src/metrics.rs:477`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L477) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) | ### `cache.io` @@ -154,7 +154,7 @@ Counter of bytes read/written during cache operations | Unit | `By` | | Prometheus series | `nativelink_cache_io_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:481`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L481) | +| Declared at | [`nativelink-util/src/metrics.rs:482`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L482) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) | ### `cache.size` @@ -168,7 +168,7 @@ Current total size of all cached data in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_size` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L487) | +| Declared at | [`nativelink-util/src/metrics.rs:488`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L488) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.entries` @@ -182,7 +182,7 @@ Current number of entries in cache | Unit | `{entry}` | | Prometheus series | `nativelink_cache_entries` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:493`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L493) | +| Declared at | [`nativelink-util/src/metrics.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L494) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.item.size` @@ -196,14 +196,14 @@ Histogram of individual cache entry sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_item_size` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:499`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L499) | +| Declared at | [`nativelink-util/src/metrics.rs:500`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L500) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) | ## Execution metrics Global remote execution metrics instruments. -Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L539), backed by the `ExecutionMetrics` struct. +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:540`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L540), backed by the `ExecutionMetrics` struct. Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. @@ -218,7 +218,7 @@ Histogram of stage durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:543`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L543) | +| Declared at | [`nativelink-util/src/metrics.rs:544`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L544) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 15 explicit boundaries, from @@ -237,7 +237,7 @@ Histogram of total execution durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L568) | +| Declared at | [`nativelink-util/src/metrics.rs:569`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L569) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 13 explicit boundaries, from @@ -256,7 +256,7 @@ Histogram of queue wait times in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:593`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L593) | +| Declared at | [`nativelink-util/src/metrics.rs:594`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L594) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 12 explicit boundaries, from @@ -275,7 +275,7 @@ Current number of actions in each stage | Unit | `{action}` | | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:613`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L613) | +| Declared at | [`nativelink-util/src/metrics.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L614) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | ### `execution.completed.count` @@ -289,7 +289,7 @@ Total number of completed executions | Unit | `{action}` | | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:619`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L619) | +| Declared at | [`nativelink-util/src/metrics.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L620) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L152), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L162) | ### `execution.stage.transitions` @@ -303,7 +303,7 @@ Number of stage transitions | Unit | `{transition}` | | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:625`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L625) | +| Declared at | [`nativelink-util/src/metrics.rs:626`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L626) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L141) | ### `execution.cpu.time` @@ -317,7 +317,7 @@ CPU time an action used, as sampled by the worker. | Unit | `s` | | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:631`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L631) | +| Declared at | [`nativelink-util/src/metrics.rs:632`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L632) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1911`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1911) | **Histogram buckets:** 12 explicit boundaries, from @@ -336,7 +336,7 @@ Peak memory an action used, as sampled by the worker. | Unit | `By` | | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:640`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L640) | +| Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | **Histogram buckets:** 8 explicit boundaries, from @@ -355,7 +355,7 @@ Histogram of output sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:656`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L656) | +| Declared at | [`nativelink-util/src/metrics.rs:657`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L657) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 8 explicit boundaries, from @@ -374,14 +374,14 @@ Counter for execution retries | Unit | `{retry}` | | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L672) | +| Declared at | [`nativelink-util/src/metrics.rs:673`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L673) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L174) | ## Worker fleet metrics Global worker fleet metrics instruments. -Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L881), backed by the `WorkerMetrics` struct. +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L882), backed by the `WorkerMetrics` struct. Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. @@ -396,7 +396,7 @@ Workers currently connected. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:885`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L885) | +| Declared at | [`nativelink-util/src/metrics.rs:886`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L886) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464), [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | ### `worker.connections` @@ -410,7 +410,7 @@ Workers that have joined, cumulative. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L891) | +| Declared at | [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464) | ### `worker.disconnections` @@ -424,7 +424,7 @@ Workers that have left, cumulative, by reason. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:897`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L897) | +| Declared at | [`nativelink-util/src/metrics.rs:898`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L898) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | ### `worker.input.relinks` @@ -438,7 +438,7 @@ Input hardlinks made again after their source was evicted mid-staging, by kind. | Unit | — | | Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L903) | +| Declared at | [`nativelink-util/src/metrics.rs:904`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L904) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1190) | ### `worker.prepare.retries` @@ -452,7 +452,7 @@ Input fetches tried again on the worker after a transient store failure: `retrie | Unit | — | | Prometheus series | `nativelink_worker_prepare_retries` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L908) | +| Declared at | [`nativelink-util/src/metrics.rs:909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L909) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1014) | ### `scheduler.dispatch.requeues` @@ -466,7 +466,7 @@ Dispatches requeued untried, by reason. | Unit | — | | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L913) | +| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1187`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1187), [`nativelink-scheduler/src/api_worker_scheduler.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1381), [`nativelink-scheduler/src/api_worker_scheduler.rs:1701`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1701), [`nativelink-scheduler/src/api_worker_scheduler.rs:1999`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1999), [`nativelink-scheduler/src/api_worker_scheduler.rs:2030`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2030), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | ### `worker.keepalives` @@ -480,7 +480,7 @@ Keepalives received, cumulative. | Unit | `{keepalive}` | | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:918`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L918) | +| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:422`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L422) | ### `worker.keepalive.gap` @@ -494,7 +494,7 @@ Seconds between consecutive messages from a worker. | Unit | `s` | | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:924`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L924) | +| Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L925) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:356`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L356) | **Histogram buckets:** 12 explicit boundaries, from @@ -513,7 +513,7 @@ Connected workers currently paused or draining. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L935) | +| Declared at | [`nativelink-util/src/metrics.rs:936`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L936) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:405`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L405), [`nativelink-scheduler/src/api_worker_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L458), [`nativelink-scheduler/src/api_worker_scheduler.rs:461`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L461), [`nativelink-scheduler/src/api_worker_scheduler.rs:571`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L571), [`nativelink-scheduler/src/api_worker_scheduler.rs:1148`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1148), [`nativelink-scheduler/src/api_worker_scheduler.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1190), [`nativelink-scheduler/src/api_worker_scheduler.rs:1326`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1326) | ## gRPC server metrics @@ -523,7 +523,7 @@ Global gRPC serving metrics. One duration histogram covers rate, errors and latency: the count gives rate, the status attribute separates errors, and the buckets give latency. -Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1059`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1059), backed by the `RpcMetrics` struct. +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1060`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1060), backed by the `RpcMetrics` struct. Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. @@ -538,7 +538,7 @@ Duration of inbound gRPC calls, by service, method and status. | Unit | `s` | | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1063`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1063) | +| Declared at | [`nativelink-util/src/metrics.rs:1064`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1064) | | Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L371) | **Histogram buckets:** 15 explicit boundaries, from @@ -550,7 +550,7 @@ measurement. Global scheduler metrics. -Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1112`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1112), backed by the `SchedulerOtlpMetrics` struct. +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1113`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1113), backed by the `SchedulerOtlpMetrics` struct. Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. @@ -566,7 +566,7 @@ while the queue is non-empty means matching is the bottleneck. | Unit | `s` | | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1116`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1116) | +| Declared at | [`nativelink-util/src/metrics.rs:1117`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1117) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | **Histogram buckets:** 13 explicit boundaries, from @@ -585,7 +585,7 @@ Matching passes, by result. | Unit | `{pass}` | | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1125`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1125) | +| Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | ### `scheduler.unsatisfiable.queued` @@ -601,7 +601,7 @@ instance, judged against that instance's own workers. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1131`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1131) | +| Declared at | [`nativelink-util/src/metrics.rs:1132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1132) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1036`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1036) | ### `scheduler.unsatisfiable.failed` @@ -616,7 +616,7 @@ the properties that could not be satisfied. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1139`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1139) | +| Declared at | [`nativelink-util/src/metrics.rs:1140`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1140) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L560) | ### `scheduler.queue.depth` @@ -630,7 +630,7 @@ Queued actions the last matching pass read. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1145`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1145) | +| Declared at | [`nativelink-util/src/metrics.rs:1146`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1146) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1037`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1037) | ### `scheduler.queue.age` @@ -644,7 +644,7 @@ Queued actions the last matching pass read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1151`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1151) | +| Declared at | [`nativelink-util/src/metrics.rs:1152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1152) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | ### `scheduler.queue.oldest_age` @@ -658,7 +658,7 @@ Age of the oldest action the last matching pass read, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1159`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1159) | +| Declared at | [`nativelink-util/src/metrics.rs:1160`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1160) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | ### `scheduler.executing.age` @@ -672,7 +672,7 @@ Executing actions the last executing sweep read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1167`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1167) | +| Declared at | [`nativelink-util/src/metrics.rs:1168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1168) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | ### `scheduler.executing.oldest_age` @@ -687,7 +687,7 @@ executing, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1175`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1175) | +| Declared at | [`nativelink-util/src/metrics.rs:1176`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1176) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | ### `scheduler.queue.retired` @@ -701,7 +701,7 @@ Queued actions retired for having no client. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1183`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1183) | +| Declared at | [`nativelink-util/src/metrics.rs:1184`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1184) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:783`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L783) | ### `scheduler.matching.parked_dispatched` @@ -716,7 +716,7 @@ opened before the pass ended, ahead of the newer actions it was reading. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1189`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1189) | +| Declared at | [`nativelink-util/src/metrics.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1190) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1013`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1013) | ### `scheduler.awaited_action.orphans` @@ -730,7 +730,7 @@ Listed operations whose record was gone on read, by site. | Unit | `{operation}` | | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1197`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1197) | +| Declared at | [`nativelink-util/src/metrics.rs:1198`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1198) | | Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:600`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L600), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | ### `scheduler.sweep.failures` @@ -744,7 +744,7 @@ Abandoned-queue sweeps that failed before finishing. | Unit | `{sweep}` | | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1205`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1205) | +| Declared at | [`nativelink-util/src/metrics.rs:1206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1206) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1259`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1259), [`nativelink-scheduler/src/simple_scheduler.rs:1318`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1318) | ### `scheduler.hints.resolutions` @@ -758,14 +758,14 @@ Actions sized by the historical resource scheduler, by source. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1211`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1211) | +| Declared at | [`nativelink-util/src/metrics.rs:1212`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1212) | | Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) | ## STORE_METRICS Global object-store metrics. -Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1427`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1427), backed by the `StoreMetrics` struct. +Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1428`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1428), backed by the `StoreMetrics` struct. ### `store.object.throttles` @@ -778,14 +778,14 @@ Throttled responses, by provider and operation. | Unit | `{response}` | | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1431`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1431) | +| Declared at | [`nativelink-util/src/metrics.rs:1432`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1432) | | Emitted from | [`nativelink-store/src/azure_blob_store.rs:268`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L268) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1455`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1455), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1456`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1456), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -800,7 +800,7 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1459`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1459) | +| Declared at | [`nativelink-util/src/metrics.rs:1460`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1460) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -814,14 +814,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1465`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1465) | +| Declared at | [`nativelink-util/src/metrics.rs:1466`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1466) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1506`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1506), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1507`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1507), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -836,14 +836,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1510`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1510) | +| Declared at | [`nativelink-util/src/metrics.rs:1511`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1511) | | Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1537`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1537), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1538`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1538), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -858,7 +858,7 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1541`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1541) | +| Declared at | [`nativelink-util/src/metrics.rs:1542`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1542) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | **Histogram buckets:** 11 explicit boundaries, from @@ -877,7 +877,7 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1550`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1550) | +| Declared at | [`nativelink-util/src/metrics.rs:1551`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1551) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | ### `connection.reconnects` @@ -891,7 +891,7 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1556`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1556) | +| Declared at | [`nativelink-util/src/metrics.rs:1557`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1557) | | Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | ## Attributes @@ -926,6 +926,7 @@ underscores. | Attribute | Prometheus label | Rust constant | Values | | --- | --- | --- | --- | +| `worker.prepare.outcome` | `worker_prepare_outcome` | `WORKER_PREPARE_OUTCOME` | open (free-form string) | | `worker.state` | `worker_state` | `WORKER_STATE` | open (free-form string) | | `worker.disconnect.reason` | `worker_disconnect_reason` | `WORKER_DISCONNECT_REASON` | open (free-form string) | From 3c749ecb48ac310b0ed6f56037a8d14f55f5c991 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 13 Oct 2026 16:27:43 -0400 Subject: [PATCH 2/2] Prefix the relink, throttle and action age attribute keys --- nativelink-util/src/metrics.rs | 21 +++- .../docs/content/docs/reference/metrics.mdx | 118 +++++++++--------- 2 files changed, 76 insertions(+), 63 deletions(-) diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index 92150e99fd..378f278748 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -46,7 +46,16 @@ pub const RPC_STATUS_CODE: &str = "rpc.grpc.status_code"; pub const SCHEDULER_MATCH_RESULT: &str = "scheduler.match.result"; /// The platform properties no worker could satisfy, comma separated. pub const SCHEDULER_UNSATISFIABLE_PROPERTIES: &str = "scheduler.unsatisfiable.properties"; +/// The age bucket of a queued or executing action. +pub const SCHEDULER_ACTION_AGE: &str = "scheduler.action.age"; + +// Metric attribute keys for the worker. pub const WORKER_PREPARE_OUTCOME: &str = "worker.prepare.outcome"; +pub const WORKER_RELINK_KIND: &str = "worker.relink.kind"; + +// Metric attribute keys for object stores. +pub const STORE_OBJECT_PROVIDER: &str = "store.object.provider"; +pub const STORE_OBJECT_OPERATION: &str = "store.object.operation"; // Metric attribute keys for tiered stores. pub const STORE_TIER: &str = "store.tier"; @@ -1022,7 +1031,7 @@ pub fn record_dispatch_requeue(reason: &'static str) { pub fn record_input_relink(kind: &'static str) { WORKER_METRICS .input_relinks - .add(1, &[KeyValue::new("kind", kind)]); + .add(1, &[KeyValue::new(WORKER_RELINK_KIND, kind)]); } /// Records an input fetch the worker tried again after a transient store @@ -1285,9 +1294,6 @@ pub fn record_queue_depth(count: u64) { SCHEDULER_METRICS.queue_depth.record(count, &[]); } -/// Attribute key for the age bucket of a queued or executing action. -pub const ACTION_AGE_BUCKET: &str = "age"; - /// Upper bounds of the action age buckets; anything older falls in the last, /// open bucket. Matches how an operator asks about a backlog: how much has /// waited over a minute, five, fifteen, thirty, an hour. @@ -1368,7 +1374,7 @@ fn record_age_buckets( ages: &AgeBuckets, ) { for (label, count) in AGE_BUCKET_LABELS.iter().zip(ages.counts()) { - by_bucket.record(count, &[KeyValue::new(ACTION_AGE_BUCKET, *label)]); + by_bucket.record(count, &[KeyValue::new(SCHEDULER_ACTION_AGE, *label)]); } oldest.record(ages.oldest().as_secs_f64(), &[]); } @@ -1448,7 +1454,10 @@ pub struct StoreMetrics { pub fn record_object_store_throttle(provider: &'static str, op: &'static str) { STORE_METRICS.object_throttles.add( 1, - &[KeyValue::new("provider", provider), KeyValue::new("op", op)], + &[ + KeyValue::new(STORE_OBJECT_PROVIDER, provider), + KeyValue::new(STORE_OBJECT_OPERATION, op), + ], ); } diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index d5939c96f3..cd7ed6fc12 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.6 (1911c9c30) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (3d614c607) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} NativeLink **v1.7.6** declares 47 OpenTelemetry instruments. This page is @@ -106,7 +106,7 @@ Every instrument declared in `metrics.rs` has at least one call site. Global cache metrics instruments. -Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:439`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L439), backed by the `CacheMetrics` struct. +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:448`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L448), backed by the `CacheMetrics` struct. Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. @@ -121,7 +121,7 @@ Histogram of cache operation durations in milliseconds | Unit | `ms` | | Prometheus series | `nativelink_cache_operation_duration` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:443`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L443) | +| Declared at | [`nativelink-util/src/metrics.rs:452`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L452) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) | **Histogram buckets:** 19 explicit boundaries, from @@ -140,7 +140,7 @@ Counter of cache operations by type and result | Unit | — | | Prometheus series | `nativelink_cache_operations_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:477`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L477) | +| Declared at | [`nativelink-util/src/metrics.rs:486`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L486) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) | ### `cache.io` @@ -154,7 +154,7 @@ Counter of bytes read/written during cache operations | Unit | `By` | | Prometheus series | `nativelink_cache_io_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:482`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L482) | +| Declared at | [`nativelink-util/src/metrics.rs:491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L491) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) | ### `cache.size` @@ -168,7 +168,7 @@ Current total size of all cached data in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_size` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:488`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L488) | +| Declared at | [`nativelink-util/src/metrics.rs:497`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L497) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.entries` @@ -182,7 +182,7 @@ Current number of entries in cache | Unit | `{entry}` | | Prometheus series | `nativelink_cache_entries` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L494) | +| Declared at | [`nativelink-util/src/metrics.rs:503`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L503) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.item.size` @@ -196,14 +196,14 @@ Histogram of individual cache entry sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_item_size` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:500`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L500) | +| Declared at | [`nativelink-util/src/metrics.rs:509`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L509) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) | ## Execution metrics Global remote execution metrics instruments. -Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:540`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L540), backed by the `ExecutionMetrics` struct. +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:549`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L549), backed by the `ExecutionMetrics` struct. Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. @@ -218,7 +218,7 @@ Histogram of stage durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:544`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L544) | +| Declared at | [`nativelink-util/src/metrics.rs:553`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L553) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 15 explicit boundaries, from @@ -237,7 +237,7 @@ Histogram of total execution durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:569`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L569) | +| Declared at | [`nativelink-util/src/metrics.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L578) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 13 explicit boundaries, from @@ -256,7 +256,7 @@ Histogram of queue wait times in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:594`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L594) | +| Declared at | [`nativelink-util/src/metrics.rs:603`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L603) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 12 explicit boundaries, from @@ -275,7 +275,7 @@ Current number of actions in each stage | Unit | `{action}` | | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L614) | +| Declared at | [`nativelink-util/src/metrics.rs:623`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L623) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | ### `execution.completed.count` @@ -289,7 +289,7 @@ Total number of completed executions | Unit | `{action}` | | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L620) | +| Declared at | [`nativelink-util/src/metrics.rs:629`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L629) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L152), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L162) | ### `execution.stage.transitions` @@ -303,7 +303,7 @@ Number of stage transitions | Unit | `{transition}` | | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:626`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L626) | +| Declared at | [`nativelink-util/src/metrics.rs:635`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L635) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L141) | ### `execution.cpu.time` @@ -317,7 +317,7 @@ CPU time an action used, as sampled by the worker. | Unit | `s` | | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:632`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L632) | +| Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1911`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1911) | **Histogram buckets:** 12 explicit boundaries, from @@ -336,7 +336,7 @@ Peak memory an action used, as sampled by the worker. | Unit | `By` | | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | +| Declared at | [`nativelink-util/src/metrics.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L650) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | **Histogram buckets:** 8 explicit boundaries, from @@ -355,7 +355,7 @@ Histogram of output sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:657`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L657) | +| Declared at | [`nativelink-util/src/metrics.rs:666`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L666) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 8 explicit boundaries, from @@ -374,14 +374,14 @@ Counter for execution retries | Unit | `{retry}` | | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:673`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L673) | +| Declared at | [`nativelink-util/src/metrics.rs:682`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L682) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L174) | ## Worker fleet metrics Global worker fleet metrics instruments. -Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L882), backed by the `WorkerMetrics` struct. +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L891), backed by the `WorkerMetrics` struct. Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. @@ -396,7 +396,7 @@ Workers currently connected. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:886`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L886) | +| Declared at | [`nativelink-util/src/metrics.rs:895`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L895) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464), [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | ### `worker.connections` @@ -410,7 +410,7 @@ Workers that have joined, cumulative. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892) | +| Declared at | [`nativelink-util/src/metrics.rs:901`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L901) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464) | ### `worker.disconnections` @@ -424,7 +424,7 @@ Workers that have left, cumulative, by reason. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:898`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L898) | +| Declared at | [`nativelink-util/src/metrics.rs:907`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L907) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | ### `worker.input.relinks` @@ -438,7 +438,7 @@ Input hardlinks made again after their source was evicted mid-staging, by kind. | Unit | — | | Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:904`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L904) | +| Declared at | [`nativelink-util/src/metrics.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L913) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1190) | ### `worker.prepare.retries` @@ -452,7 +452,7 @@ Input fetches tried again on the worker after a transient store failure: `retrie | Unit | — | | Prometheus series | `nativelink_worker_prepare_retries` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L909) | +| Declared at | [`nativelink-util/src/metrics.rs:918`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L918) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1014) | ### `scheduler.dispatch.requeues` @@ -466,7 +466,7 @@ Dispatches requeued untried, by reason. | Unit | — | | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) | +| Declared at | [`nativelink-util/src/metrics.rs:923`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L923) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1187`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1187), [`nativelink-scheduler/src/api_worker_scheduler.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1381), [`nativelink-scheduler/src/api_worker_scheduler.rs:1701`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1701), [`nativelink-scheduler/src/api_worker_scheduler.rs:1999`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1999), [`nativelink-scheduler/src/api_worker_scheduler.rs:2030`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2030), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | ### `worker.keepalives` @@ -480,7 +480,7 @@ Keepalives received, cumulative. | Unit | `{keepalive}` | | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | +| Declared at | [`nativelink-util/src/metrics.rs:928`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L928) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:422`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L422) | ### `worker.keepalive.gap` @@ -494,7 +494,7 @@ Seconds between consecutive messages from a worker. | Unit | `s` | | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L925) | +| Declared at | [`nativelink-util/src/metrics.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L934) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:356`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L356) | **Histogram buckets:** 12 explicit boundaries, from @@ -513,7 +513,7 @@ Connected workers currently paused or draining. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:936`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L936) | +| Declared at | [`nativelink-util/src/metrics.rs:945`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L945) | | Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:405`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L405), [`nativelink-scheduler/src/api_worker_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L458), [`nativelink-scheduler/src/api_worker_scheduler.rs:461`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L461), [`nativelink-scheduler/src/api_worker_scheduler.rs:571`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L571), [`nativelink-scheduler/src/api_worker_scheduler.rs:1148`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1148), [`nativelink-scheduler/src/api_worker_scheduler.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1190), [`nativelink-scheduler/src/api_worker_scheduler.rs:1326`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1326) | ## gRPC server metrics @@ -523,7 +523,7 @@ Global gRPC serving metrics. One duration histogram covers rate, errors and latency: the count gives rate, the status attribute separates errors, and the buckets give latency. -Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1060`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1060), backed by the `RpcMetrics` struct. +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1069`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1069), backed by the `RpcMetrics` struct. Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. @@ -538,7 +538,7 @@ Duration of inbound gRPC calls, by service, method and status. | Unit | `s` | | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1064`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1064) | +| Declared at | [`nativelink-util/src/metrics.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1073) | | Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L371) | **Histogram buckets:** 15 explicit boundaries, from @@ -550,7 +550,7 @@ measurement. Global scheduler metrics. -Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1113`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1113), backed by the `SchedulerOtlpMetrics` struct. +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1122`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1122), backed by the `SchedulerOtlpMetrics` struct. Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. @@ -566,7 +566,7 @@ while the queue is non-empty means matching is the bottleneck. | Unit | `s` | | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1117`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1117) | +| Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | **Histogram buckets:** 13 explicit boundaries, from @@ -585,7 +585,7 @@ Matching passes, by result. | Unit | `{pass}` | | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | +| Declared at | [`nativelink-util/src/metrics.rs:1135`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1135) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | ### `scheduler.unsatisfiable.queued` @@ -601,7 +601,7 @@ instance, judged against that instance's own workers. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1132) | +| Declared at | [`nativelink-util/src/metrics.rs:1141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1141) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1036`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1036) | ### `scheduler.unsatisfiable.failed` @@ -616,7 +616,7 @@ the properties that could not be satisfied. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1140`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1140) | +| Declared at | [`nativelink-util/src/metrics.rs:1149`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1149) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L560) | ### `scheduler.queue.depth` @@ -630,7 +630,7 @@ Queued actions the last matching pass read. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1146`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1146) | +| Declared at | [`nativelink-util/src/metrics.rs:1155`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1155) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1037`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1037) | ### `scheduler.queue.age` @@ -644,7 +644,7 @@ Queued actions the last matching pass read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1152) | +| Declared at | [`nativelink-util/src/metrics.rs:1161`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1161) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | ### `scheduler.queue.oldest_age` @@ -658,7 +658,7 @@ Age of the oldest action the last matching pass read, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1160`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1160) | +| Declared at | [`nativelink-util/src/metrics.rs:1169`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1169) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | ### `scheduler.executing.age` @@ -672,7 +672,7 @@ Executing actions the last executing sweep read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1168) | +| Declared at | [`nativelink-util/src/metrics.rs:1177`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1177) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | ### `scheduler.executing.oldest_age` @@ -687,7 +687,7 @@ executing, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1176`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1176) | +| Declared at | [`nativelink-util/src/metrics.rs:1185`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1185) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | ### `scheduler.queue.retired` @@ -701,7 +701,7 @@ Queued actions retired for having no client. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1184`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1184) | +| Declared at | [`nativelink-util/src/metrics.rs:1193`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1193) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:783`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L783) | ### `scheduler.matching.parked_dispatched` @@ -716,7 +716,7 @@ opened before the pass ended, ahead of the newer actions it was reading. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1190) | +| Declared at | [`nativelink-util/src/metrics.rs:1199`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1199) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1013`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1013) | ### `scheduler.awaited_action.orphans` @@ -730,7 +730,7 @@ Listed operations whose record was gone on read, by site. | Unit | `{operation}` | | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1198`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1198) | +| Declared at | [`nativelink-util/src/metrics.rs:1207`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1207) | | Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:600`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L600), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | ### `scheduler.sweep.failures` @@ -744,7 +744,7 @@ Abandoned-queue sweeps that failed before finishing. | Unit | `{sweep}` | | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1206) | +| Declared at | [`nativelink-util/src/metrics.rs:1215`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1215) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1259`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1259), [`nativelink-scheduler/src/simple_scheduler.rs:1318`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1318) | ### `scheduler.hints.resolutions` @@ -758,14 +758,14 @@ Actions sized by the historical resource scheduler, by source. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1212`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1212) | +| Declared at | [`nativelink-util/src/metrics.rs:1221`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1221) | | Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) | ## STORE_METRICS Global object-store metrics. -Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1428`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1428), backed by the `StoreMetrics` struct. +Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1434`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1434), backed by the `StoreMetrics` struct. ### `store.object.throttles` @@ -778,14 +778,14 @@ Throttled responses, by provider and operation. | Unit | `{response}` | | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1432`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1432) | +| Declared at | [`nativelink-util/src/metrics.rs:1438`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1438) | | Emitted from | [`nativelink-store/src/azure_blob_store.rs:268`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L268) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1456`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1456), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1465`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1465), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -800,7 +800,7 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1460`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1460) | +| Declared at | [`nativelink-util/src/metrics.rs:1469`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1469) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -814,14 +814,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1466`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1466) | +| Declared at | [`nativelink-util/src/metrics.rs:1475`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1475) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1507`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1507), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1516`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1516), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -836,14 +836,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1511`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1511) | +| Declared at | [`nativelink-util/src/metrics.rs:1520`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1520) | | Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1538`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1538), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1547`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1547), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -858,7 +858,7 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1542`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1542) | +| Declared at | [`nativelink-util/src/metrics.rs:1551`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1551) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | **Histogram buckets:** 11 explicit boundaries, from @@ -877,7 +877,7 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1551`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1551) | +| Declared at | [`nativelink-util/src/metrics.rs:1560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1560) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | ### `connection.reconnects` @@ -891,7 +891,7 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1557`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1557) | +| Declared at | [`nativelink-util/src/metrics.rs:1566`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1566) | | Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | ## Attributes @@ -927,6 +927,7 @@ underscores. | Attribute | Prometheus label | Rust constant | Values | | --- | --- | --- | --- | | `worker.prepare.outcome` | `worker_prepare_outcome` | `WORKER_PREPARE_OUTCOME` | open (free-form string) | +| `worker.relink.kind` | `worker_relink_kind` | `WORKER_RELINK_KIND` | open (free-form string) | | `worker.state` | `worker_state` | `WORKER_STATE` | open (free-form string) | | `worker.disconnect.reason` | `worker_disconnect_reason` | `WORKER_DISCONNECT_REASON` | open (free-form string) | @@ -944,11 +945,14 @@ underscores. | --- | --- | --- | --- | | `scheduler.match.result` | `scheduler_match_result` | `SCHEDULER_MATCH_RESULT` | open (free-form string) | | `scheduler.unsatisfiable.properties` | `scheduler_unsatisfiable_properties` | `SCHEDULER_UNSATISFIABLE_PROPERTIES` | open (free-form string) | +| `scheduler.action.age` | `scheduler_action_age` | `SCHEDULER_ACTION_AGE` | open (free-form string) | ### Store tier attributes | Attribute | Prometheus label | Rust constant | Values | | --- | --- | --- | --- | +| `store.object.provider` | `store_object_provider` | `STORE_OBJECT_PROVIDER` | open (free-form string) | +| `store.object.operation` | `store_object_operation` | `STORE_OBJECT_OPERATION` | open (free-form string) | | `store.tier` | `store_tier` | `STORE_TIER` | open (free-form string) | | `store.result` | `store_result` | `STORE_RESULT` | open (free-form string) | | `store.direction` | `store_direction` | `STORE_DIRECTION` | open (free-form string) |