diff --git a/scripts/check_dashboards.py b/scripts/check_dashboards.py index 782d85c..0a30e7d 100755 --- a/scripts/check_dashboards.py +++ b/scripts/check_dashboards.py @@ -9,7 +9,7 @@ * every datasource UID resolves to one declared in provisioning; * every dashboard UID is unique across the folder; * panels fit the 24-column grid and do not overlap; - * every panel has at least one target. + * every panel that queries data has at least one target. Additionally, every PromQL expression is emitted to stdout in Prometheus recording-rule form when --emit-promql is passed, so promtool can parse them. @@ -28,6 +28,16 @@ # UIDs Grafana provides itself. BUILTIN_UIDS = {"-- Grafana --", "-- Mixed --", "-- Dashboard --", "grafana"} +# Panel types that legitimately carry no targets. +# +# row and text render no data at all. alertlist is different and worth naming: +# it does read from a datasource, but not through a query — it pulls alerts from +# the Alertmanager datasource named in its own options, so a targets array would +# be meaningless. Requiring one here would have forced a fake target onto every +# alert panel, which is exactly the kind of thing that teaches people to work +# around this script rather than trust it. +TARGETLESS_PANEL_TYPES = {"row", "text", "alertlist"} + def declared_datasource_uids() -> set[str]: """Read provisioned UIDs without requiring PyYAML.""" @@ -116,7 +126,7 @@ def main() -> int: f"{name}: panel '{title}' overflows the 24-column grid " f"(x={grid['x']} w={grid['w']})" ) - if panel.get("type") not in ("row", "text") and not panel.get("targets"): + if panel.get("type") not in TARGETLESS_PANEL_TYPES and not panel.get("targets"): problems.append(f"{name}: panel '{title}' has no targets") for other in panels[i + 1:]: diff --git a/stacks/observability/grafana/dashboards/docker-containers.json b/stacks/observability/grafana/dashboards/docker-containers.json index 3b5bc93..f55dbe7 100644 --- a/stacks/observability/grafana/dashboards/docker-containers.json +++ b/stacks/observability/grafana/dashboards/docker-containers.json @@ -83,6 +83,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Container alerts firing", + "description": "Firing and pending alerts carrying component=containers, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"containers\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Containers running", @@ -95,7 +128,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -161,7 +194,7 @@ "h": 4, "w": 5, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -223,7 +256,7 @@ "h": 4, "w": 5, "x": 9, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -286,7 +319,7 @@ "h": 4, "w": 5, "x": 14, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -356,7 +389,7 @@ "h": 4, "w": 5, "x": 19, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -418,7 +451,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 10 }, "panels": [] }, @@ -434,7 +467,7 @@ "h": 9, "w": 12, "x": 0, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -515,7 +548,7 @@ "h": 9, "w": 12, "x": 12, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -596,7 +629,7 @@ "h": 9, "w": 12, "x": 0, - "y": 14 + "y": 20 }, "fieldConfig": { "defaults": { @@ -684,7 +717,7 @@ "h": 9, "w": 12, "x": 12, - "y": 14 + "y": 20 }, "fieldConfig": { "defaults": { @@ -757,7 +790,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 29 }, "panels": [] }, @@ -773,7 +806,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 30 }, "fieldConfig": { "defaults": { @@ -850,7 +883,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 30 }, "fieldConfig": { "defaults": { @@ -927,7 +960,7 @@ "h": 9, "w": 24, "x": 0, - "y": 32 + "y": 38 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/host-overview.json b/stacks/observability/grafana/dashboards/host-overview.json index 57dd840..e558f97 100644 --- a/stacks/observability/grafana/dashboards/host-overview.json +++ b/stacks/observability/grafana/dashboards/host-overview.json @@ -61,6 +61,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Host alerts firing", + "description": "Firing and pending alerts carrying component=host, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"host\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Hosts reporting", @@ -73,7 +106,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -139,7 +172,7 @@ "h": 8, "w": 5, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -207,7 +240,7 @@ "h": 8, "w": 5, "x": 9, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -275,7 +308,7 @@ "h": 8, "w": 5, "x": 14, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -343,7 +376,7 @@ "h": 8, "w": 5, "x": 19, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -405,7 +438,7 @@ "h": 4, "w": 4, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -472,7 +505,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 14 }, "panels": [] }, @@ -488,7 +521,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -569,7 +602,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -658,7 +691,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -759,7 +792,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -844,7 +877,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 31 }, "panels": [] }, @@ -860,7 +893,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -938,7 +971,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -1026,7 +1059,7 @@ "h": 8, "w": 24, "x": 0, - "y": 34 + "y": 40 }, "fieldConfig": { "defaults": { @@ -1154,7 +1187,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 48 }, "panels": [] }, @@ -1170,7 +1203,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 49 }, "fieldConfig": { "defaults": { @@ -1258,7 +1291,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 49 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/logs-explorer.json b/stacks/observability/grafana/dashboards/logs-explorer.json index 877415a..da6d80a 100644 --- a/stacks/observability/grafana/dashboards/logs-explorer.json +++ b/stacks/observability/grafana/dashboards/logs-explorer.json @@ -84,6 +84,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Log-based alerts firing", + "description": "Firing and pending alerts carrying component=logs, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"logs\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Lines/s (all hosts)", @@ -96,7 +129,7 @@ "h": 4, "w": 5, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -159,7 +192,7 @@ "h": 4, "w": 5, "x": 5, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -229,7 +262,7 @@ "h": 4, "w": 5, "x": 10, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -295,7 +328,7 @@ "h": 4, "w": 9, "x": 15, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -361,7 +394,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 10 }, "panels": [] }, @@ -377,7 +410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -458,7 +491,7 @@ "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -539,7 +572,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -616,7 +649,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -693,7 +726,7 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 27 }, "panels": [] }, @@ -709,7 +742,7 @@ "h": 11, "w": 24, "x": 0, - "y": 22 + "y": 28 }, "options": { "showTime": true, @@ -746,7 +779,7 @@ "h": 11, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "options": { "showTime": true, @@ -783,7 +816,7 @@ "h": 12, "w": 24, "x": 0, - "y": 44 + "y": 50 }, "options": { "showTime": true, diff --git a/stacks/observability/grafana/dashboards/network-snmp.json b/stacks/observability/grafana/dashboards/network-snmp.json index a48fb4b..0b099ef 100644 --- a/stacks/observability/grafana/dashboards/network-snmp.json +++ b/stacks/observability/grafana/dashboards/network-snmp.json @@ -39,6 +39,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Network & firewall alerts firing", + "description": "Firing and pending alerts carrying component=network, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"network\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Packet filter", @@ -51,7 +84,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -133,7 +166,7 @@ "h": 4, "w": 4, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -203,7 +236,7 @@ "h": 8, "w": 5, "x": 8, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -271,7 +304,7 @@ "h": 4, "w": 5, "x": 13, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -333,7 +366,7 @@ "h": 4, "w": 6, "x": 18, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -395,7 +428,7 @@ "h": 4, "w": 4, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -457,7 +490,7 @@ "h": 4, "w": 4, "x": 4, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -520,7 +553,7 @@ "h": 4, "w": 5, "x": 13, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -586,7 +619,7 @@ "h": 4, "w": 6, "x": 18, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -649,7 +682,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 14 }, "panels": [] }, @@ -665,7 +698,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -754,7 +787,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -855,7 +888,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -980,7 +1013,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -1065,7 +1098,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 31 }, "panels": [] }, @@ -1081,7 +1114,7 @@ "h": 9, "w": 24, "x": 0, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -1169,7 +1202,7 @@ "h": 9, "w": 24, "x": 0, - "y": 35 + "y": 41 }, "fieldConfig": { "defaults": { @@ -1286,7 +1319,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 50 }, "panels": [] }, @@ -1302,7 +1335,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 51 }, "fieldConfig": { "defaults": { @@ -1379,7 +1412,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 51 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/ups-power.json b/stacks/observability/grafana/dashboards/ups-power.json index 7609f2f..3a8af73 100644 --- a/stacks/observability/grafana/dashboards/ups-power.json +++ b/stacks/observability/grafana/dashboards/ups-power.json @@ -39,12 +39,45 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Power alerts firing", + "description": "Firing and pending alerts carrying component=power, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"power\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "text", "title": "No battery is installed in this UPS", "gridPos": { "x": 0, - "y": 0, + "y": 6, "w": 24, "h": 4 }, @@ -65,7 +98,7 @@ "h": 4, "w": 5, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -154,7 +187,7 @@ "h": 8, "w": 5, "x": 5, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -222,7 +255,7 @@ "h": 8, "w": 5, "x": 10, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -290,7 +323,7 @@ "h": 4, "w": 5, "x": 15, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -360,7 +393,7 @@ "h": 4, "w": 4, "x": 20, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -456,7 +489,7 @@ "h": 4, "w": 5, "x": 0, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -518,7 +551,7 @@ "h": 4, "w": 5, "x": 15, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -584,7 +617,7 @@ "h": 4, "w": 4, "x": 20, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -650,7 +683,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "panels": [] }, @@ -666,7 +699,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -755,7 +788,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -844,7 +877,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 27 }, "fieldConfig": { "defaults": { @@ -932,7 +965,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 27 }, "fieldConfig": { "defaults": { @@ -1020,7 +1053,7 @@ "h": 8, "w": 24, "x": 0, - "y": 29 + "y": 35 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/loki/rules/security.rules.yaml b/stacks/observability/loki/rules/security.rules.yaml index 5e0f624..76e9a52 100644 --- a/stacks/observability/loki/rules/security.rules.yaml +++ b/stacks/observability/loki/rules/security.rules.yaml @@ -20,6 +20,7 @@ groups: ) > 20 for: 2m labels: + component: logs severity: warning category: security annotations: @@ -35,6 +36,7 @@ groups: ) > 100 for: 1m labels: + component: logs severity: critical category: security annotations: @@ -56,6 +58,7 @@ groups: ) > 0 for: 1m labels: + component: logs severity: critical category: security annotations: @@ -71,6 +74,7 @@ groups: ) > 5 for: 5m labels: + component: logs severity: warning category: security annotations: @@ -82,6 +86,7 @@ groups: count_over_time({log_type="authlog"} |~ "(useradd|groupadd|usermod).*new (user|group)" [10m]) ) > 0 labels: + component: logs severity: warning category: security annotations: @@ -100,6 +105,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "Out of memory: Kill(ed)? process" [15m]) ) > 0 labels: + component: logs severity: warning category: capacity annotations: @@ -111,6 +117,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "Remounting filesystem read-only" [15m]) ) > 0 labels: + component: logs severity: critical category: hardware annotations: @@ -125,6 +132,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "(I/O error|ata[0-9]+.*failed|SMART.*FAILED)" [15m]) ) > 0 labels: + component: logs severity: critical category: hardware annotations: diff --git a/stacks/observability/prometheus/rules/containers.rules.yaml b/stacks/observability/prometheus/rules/containers.rules.yaml index f9988d0..07a7fde 100644 --- a/stacks/observability/prometheus/rules/containers.rules.yaml +++ b/stacks/observability/prometheus/rules/containers.rules.yaml @@ -10,6 +10,7 @@ groups: expr: changes(container_start_time_seconds{name!=""}[15m]) > 3 for: 5m labels: + component: containers severity: warning category: availability annotations: @@ -19,6 +20,7 @@ groups: - alert: ContainerOomKilled expr: increase(container_oom_events_total{name!=""}[15m]) > 0 labels: + component: containers severity: warning category: capacity annotations: @@ -33,6 +35,7 @@ groups: and container_spec_memory_limit_bytes{name!=""} > 0 for: 15m labels: + component: containers severity: warning category: capacity annotations: @@ -44,6 +47,7 @@ groups: / clamp_min(rate(container_cpu_cfs_periods_total{name!=""}[15m]), 1) > 0.25 for: 30m labels: + component: containers severity: info category: saturation annotations: @@ -54,6 +58,7 @@ groups: expr: prometheus_config_last_reload_successful == 0 for: 5m labels: + component: containers severity: critical category: correctness annotations: @@ -63,6 +68,7 @@ groups: expr: increase(prometheus_rule_evaluation_failures_total[15m]) > 0 for: 5m labels: + component: containers severity: warning category: correctness annotations: @@ -72,6 +78,7 @@ groups: expr: increase(alertmanager_notifications_failed_total[15m]) > 0 for: 5m labels: + component: containers severity: critical category: correctness annotations: @@ -85,6 +92,7 @@ groups: sum(rate(loki_distributor_lines_received_total[30m])) == 0 for: 30m labels: + component: containers severity: warning category: availability annotations: diff --git a/stacks/observability/prometheus/rules/host.rules.yaml b/stacks/observability/prometheus/rules/host.rules.yaml index 7776db6..35c06f0 100644 --- a/stacks/observability/prometheus/rules/host.rules.yaml +++ b/stacks/observability/prometheus/rules/host.rules.yaml @@ -7,6 +7,7 @@ groups: expr: up == 0 for: 5m labels: + component: host severity: critical category: availability annotations: @@ -24,6 +25,7 @@ groups: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.30 for: 1h labels: + component: host severity: warning category: capacity annotations: @@ -39,6 +41,7 @@ groups: / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} < 0.10 for: 15m labels: + component: host severity: critical category: capacity annotations: @@ -49,6 +52,7 @@ groups: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10 for: 15m labels: + component: host severity: warning category: capacity annotations: @@ -61,6 +65,7 @@ groups: node_load15 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 2 for: 30m labels: + component: host severity: warning category: saturation annotations: @@ -70,6 +75,7 @@ groups: expr: abs(node_timex_offset_seconds) > 0.5 for: 10m labels: + component: host severity: warning category: correctness annotations: @@ -81,6 +87,7 @@ groups: - alert: HostRebooted expr: time() - node_boot_time_seconds < 600 labels: + component: host severity: info category: availability annotations: diff --git a/stacks/observability/prometheus/rules/network.rules.yaml b/stacks/observability/prometheus/rules/network.rules.yaml index 81fbbc4..9188de1 100644 --- a/stacks/observability/prometheus/rules/network.rules.yaml +++ b/stacks/observability/prometheus/rules/network.rules.yaml @@ -7,6 +7,7 @@ groups: expr: up{job="snmp-exporter"} == 0 for: 5m labels: + component: network severity: critical category: availability annotations: @@ -16,6 +17,7 @@ groups: expr: up{job="snmp"} == 0 for: 10m labels: + component: network severity: warning category: availability annotations: @@ -29,6 +31,7 @@ groups: expr: scrape_duration_seconds{job="snmp"} > 30 for: 15m labels: + component: network severity: warning category: performance annotations: @@ -42,6 +45,7 @@ groups: expr: pfStatusRunning == 0 for: 2m labels: + component: network severity: critical category: security annotations: @@ -54,6 +58,7 @@ groups: expr: pfStateTableCount / pfLimitsStates > 0.80 for: 10m labels: + component: network severity: warning category: saturation annotations: @@ -65,6 +70,7 @@ groups: expr: rate(pfCounterMemDrop[15m]) > 0 for: 15m labels: + component: network severity: warning category: saturation annotations: @@ -79,6 +85,7 @@ groups: and on (instance, ifIndex) (max_over_time(ifOperStatus[1h]) == 1) for: 5m labels: + component: network severity: warning category: availability annotations: @@ -92,6 +99,7 @@ groups: expr: rate(ifInOctets[5m]) < 0 or rate(ifOutOctets[5m]) < 0 for: 10m labels: + component: network severity: info category: correctness annotations: @@ -107,6 +115,7 @@ groups: or cpqDaPhyDrvCondition > 2 for: 5m labels: + component: network severity: critical category: hardware annotations: @@ -119,6 +128,7 @@ groups: expr: cpqHeSysBackupBatteryCondition > 2 for: 30m labels: + component: network severity: warning category: hardware annotations: diff --git a/stacks/observability/prometheus/rules/ups.rules.yaml b/stacks/observability/prometheus/rules/ups.rules.yaml index 701d58f..cee7003 100644 --- a/stacks/observability/prometheus/rules/ups.rules.yaml +++ b/stacks/observability/prometheus/rules/ups.rules.yaml @@ -40,6 +40,7 @@ groups: expr: upsSecondsOnBattery > 0 for: 30s labels: + component: power severity: critical category: power annotations: @@ -53,6 +54,7 @@ groups: expr: upsBatteryStatus > 2 for: 1m labels: + component: power severity: critical category: power annotations: @@ -62,6 +64,7 @@ groups: expr: upsEstimatedMinutesRemaining < 5 for: 1m labels: + component: power severity: critical category: power annotations: @@ -71,6 +74,7 @@ groups: expr: upsEstimatedChargeRemaining < 50 for: 10m labels: + component: power severity: warning category: power annotations: @@ -80,6 +84,7 @@ groups: expr: upsOutputPercentLoad > 80 for: 15m labels: + component: power severity: warning category: power annotations: @@ -91,6 +96,7 @@ groups: expr: upsAlarmsPresent > 0 for: 5m labels: + component: power severity: warning category: power annotations: @@ -100,6 +106,7 @@ groups: expr: upsBatteryTemperature > 40 for: 15m labels: + component: power severity: warning category: power annotations: