Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions nativelink-util/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ pub const RPC_STATUS_CODE: &str = "rpc.grpc.status_code";
pub const SCHEDULER_MATCH_RESULT: &str = "scheduler.match.result";
/// The platform properties no worker could satisfy, comma separated.
pub const SCHEDULER_UNSATISFIABLE_PROPERTIES: &str = "scheduler.unsatisfiable.properties";
/// The age bucket of a queued or executing action.
pub const SCHEDULER_ACTION_AGE: &str = "scheduler.action.age";

// Metric attribute keys for the worker.
pub const WORKER_PREPARE_OUTCOME: &str = "worker.prepare.outcome";

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 on the convention. A few other unreleased labels are still bare, though, and since none of them has shipped yet, renaming them now costs nothing:

Metric Current label Suggested
worker.input.relinks (#2926) kind worker.relink.kind
store.object.throttles (#2931) provider, op store.object.provider, store.object.operation
scheduler.queue.age, scheduler.executing.age (#2929) age scheduler.action.age

pub const WORKER_RELINK_KIND: &str = "worker.relink.kind";

// Metric attribute keys for object stores.
pub const STORE_OBJECT_PROVIDER: &str = "store.object.provider";
pub const STORE_OBJECT_OPERATION: &str = "store.object.operation";

// Metric attribute keys for tiered stores.
pub const STORE_TIER: &str = "store.tier";
Expand Down Expand Up @@ -1021,7 +1031,7 @@ pub fn record_dispatch_requeue(reason: &'static str) {
pub fn record_input_relink(kind: &'static str) {
WORKER_METRICS
.input_relinks
.add(1, &[KeyValue::new("kind", kind)]);
.add(1, &[KeyValue::new(WORKER_RELINK_KIND, kind)]);
}

/// Records an input fetch the worker tried again after a transient store
Expand All @@ -1032,7 +1042,7 @@ pub fn record_input_relink(kind: &'static str) {
pub fn record_prepare_retry(outcome: &'static str) {
WORKER_METRICS
.prepare_retries
.add(1, &[KeyValue::new("outcome", outcome)]);
.add(1, &[KeyValue::new(WORKER_PREPARE_OUTCOME, outcome)]);
}

/// Records a keepalive from a worker.
Expand Down Expand Up @@ -1284,9 +1294,6 @@ pub fn record_queue_depth(count: u64) {
SCHEDULER_METRICS.queue_depth.record(count, &[]);
}

/// Attribute key for the age bucket of a queued or executing action.
pub const ACTION_AGE_BUCKET: &str = "age";

/// Upper bounds of the action age buckets; anything older falls in the last,
/// open bucket. Matches how an operator asks about a backlog: how much has
/// waited over a minute, five, fifteen, thirty, an hour.
Expand Down Expand Up @@ -1367,7 +1374,7 @@ fn record_age_buckets(
ages: &AgeBuckets,
) {
for (label, count) in AGE_BUCKET_LABELS.iter().zip(ages.counts()) {
by_bucket.record(count, &[KeyValue::new(ACTION_AGE_BUCKET, *label)]);
by_bucket.record(count, &[KeyValue::new(SCHEDULER_ACTION_AGE, *label)]);
}
oldest.record(ages.oldest().as_secs_f64(), &[]);
}
Expand Down Expand Up @@ -1447,7 +1454,10 @@ pub struct StoreMetrics {
pub fn record_object_store_throttle(provider: &'static str, op: &'static str) {
STORE_METRICS.object_throttles.add(
1,
&[KeyValue::new("provider", provider), KeyValue::new("op", op)],
&[
KeyValue::new(STORE_OBJECT_PROVIDER, provider),
KeyValue::new(STORE_OBJECT_OPERATION, op),
],
);
}

Expand Down
Loading
Loading