From 66424ffa3f0a371aa63f3c671c1e74a9226d48dd Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Aug 2026 22:09:34 +0000 Subject: [PATCH] feat(observability): show firing alerts on every dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forty alert rules, one routing tree, and not one dashboard showed any of them. The only ways to see what was firing were the Alertmanager UI or curl, neither of which is where anyone looks. That hid IloBatteryCondition — the ProLiant's failed system battery — for over a day. It matters more here than it looks, because delivery is fragile by design: the webhook pointed at ntfy.example.invalid for the entire life of the stack, so no alert had ever been delivered until #31. A panel is the fallback that does not depend on a receiver being right. Each dashboard gains an alertlist panel at the top, filtered to its own concern and backed by the Alertmanager datasource rather than the ALERTS metric — so a silenced alert shows as silenced instead of simply vanishing. An alert that disappears when silenced is how a silence outlives the reason for it. Filtering needed a label that maps onto dashboards, and neither existing one does: severity cuts across everything, and category splits the same dashboard across availability, capacity, correctness and saturation. So every rule gains component, one value per rule file — network, host, containers, power, logs. It is additive, and alertmanager.yaml routes on category and severity, so routing is untouched. check_dashboards.py learns that alertlist carries no targets. It does read from a datasource, just not through a query — it pulls from the Alertmanager datasource named in its own options. Requiring a target would have forced a fake one onto every alert panel, which teaches people to work around the script rather than trust it. One-time churn worth expecting: adding a label changes an alert's fingerprint, so currently-firing alerts resolve and re-fire, and their `for` timers restart. IloBatteryCondition is pending again and returns to firing 30 minutes after the reload. Closes #36 --- scripts/check_dashboards.py | 14 +++- .../grafana/dashboards/docker-containers.json | 61 ++++++++++++---- .../grafana/dashboards/host-overview.json | 69 +++++++++++++----- .../grafana/dashboards/logs-explorer.json | 59 +++++++++++---- .../grafana/dashboards/network-snmp.json | 73 ++++++++++++++----- .../grafana/dashboards/ups-power.json | 63 ++++++++++++---- .../loki/rules/security.rules.yaml | 8 ++ .../prometheus/rules/containers.rules.yaml | 8 ++ .../prometheus/rules/host.rules.yaml | 7 ++ .../prometheus/rules/network.rules.yaml | 10 +++ .../prometheus/rules/ups.rules.yaml | 7 ++ 11 files changed, 297 insertions(+), 82 deletions(-) diff --git a/scripts/check_dashboards.py b/scripts/check_dashboards.py index 782d85c..0a30e7d 100755 --- a/scripts/check_dashboards.py +++ b/scripts/check_dashboards.py @@ -9,7 +9,7 @@ * every datasource UID resolves to one declared in provisioning; * every dashboard UID is unique across the folder; * panels fit the 24-column grid and do not overlap; - * every panel has at least one target. + * every panel that queries data has at least one target. Additionally, every PromQL expression is emitted to stdout in Prometheus recording-rule form when --emit-promql is passed, so promtool can parse them. @@ -28,6 +28,16 @@ # UIDs Grafana provides itself. BUILTIN_UIDS = {"-- Grafana --", "-- Mixed --", "-- Dashboard --", "grafana"} +# Panel types that legitimately carry no targets. +# +# row and text render no data at all. alertlist is different and worth naming: +# it does read from a datasource, but not through a query — it pulls alerts from +# the Alertmanager datasource named in its own options, so a targets array would +# be meaningless. Requiring one here would have forced a fake target onto every +# alert panel, which is exactly the kind of thing that teaches people to work +# around this script rather than trust it. +TARGETLESS_PANEL_TYPES = {"row", "text", "alertlist"} + def declared_datasource_uids() -> set[str]: """Read provisioned UIDs without requiring PyYAML.""" @@ -116,7 +126,7 @@ def main() -> int: f"{name}: panel '{title}' overflows the 24-column grid " f"(x={grid['x']} w={grid['w']})" ) - if panel.get("type") not in ("row", "text") and not panel.get("targets"): + if panel.get("type") not in TARGETLESS_PANEL_TYPES and not panel.get("targets"): problems.append(f"{name}: panel '{title}' has no targets") for other in panels[i + 1:]: diff --git a/stacks/observability/grafana/dashboards/docker-containers.json b/stacks/observability/grafana/dashboards/docker-containers.json index 3b5bc93..f55dbe7 100644 --- a/stacks/observability/grafana/dashboards/docker-containers.json +++ b/stacks/observability/grafana/dashboards/docker-containers.json @@ -83,6 +83,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Container alerts firing", + "description": "Firing and pending alerts carrying component=containers, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"containers\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Containers running", @@ -95,7 +128,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -161,7 +194,7 @@ "h": 4, "w": 5, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -223,7 +256,7 @@ "h": 4, "w": 5, "x": 9, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -286,7 +319,7 @@ "h": 4, "w": 5, "x": 14, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -356,7 +389,7 @@ "h": 4, "w": 5, "x": 19, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -418,7 +451,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 10 }, "panels": [] }, @@ -434,7 +467,7 @@ "h": 9, "w": 12, "x": 0, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -515,7 +548,7 @@ "h": 9, "w": 12, "x": 12, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -596,7 +629,7 @@ "h": 9, "w": 12, "x": 0, - "y": 14 + "y": 20 }, "fieldConfig": { "defaults": { @@ -684,7 +717,7 @@ "h": 9, "w": 12, "x": 12, - "y": 14 + "y": 20 }, "fieldConfig": { "defaults": { @@ -757,7 +790,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 29 }, "panels": [] }, @@ -773,7 +806,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 30 }, "fieldConfig": { "defaults": { @@ -850,7 +883,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 30 }, "fieldConfig": { "defaults": { @@ -927,7 +960,7 @@ "h": 9, "w": 24, "x": 0, - "y": 32 + "y": 38 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/host-overview.json b/stacks/observability/grafana/dashboards/host-overview.json index 57dd840..e558f97 100644 --- a/stacks/observability/grafana/dashboards/host-overview.json +++ b/stacks/observability/grafana/dashboards/host-overview.json @@ -61,6 +61,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Host alerts firing", + "description": "Firing and pending alerts carrying component=host, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"host\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Hosts reporting", @@ -73,7 +106,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -139,7 +172,7 @@ "h": 8, "w": 5, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -207,7 +240,7 @@ "h": 8, "w": 5, "x": 9, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -275,7 +308,7 @@ "h": 8, "w": 5, "x": 14, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -343,7 +376,7 @@ "h": 8, "w": 5, "x": 19, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -405,7 +438,7 @@ "h": 4, "w": 4, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -472,7 +505,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 14 }, "panels": [] }, @@ -488,7 +521,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -569,7 +602,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -658,7 +691,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -759,7 +792,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -844,7 +877,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 31 }, "panels": [] }, @@ -860,7 +893,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -938,7 +971,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -1026,7 +1059,7 @@ "h": 8, "w": 24, "x": 0, - "y": 34 + "y": 40 }, "fieldConfig": { "defaults": { @@ -1154,7 +1187,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 48 }, "panels": [] }, @@ -1170,7 +1203,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 49 }, "fieldConfig": { "defaults": { @@ -1258,7 +1291,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 49 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/logs-explorer.json b/stacks/observability/grafana/dashboards/logs-explorer.json index 877415a..da6d80a 100644 --- a/stacks/observability/grafana/dashboards/logs-explorer.json +++ b/stacks/observability/grafana/dashboards/logs-explorer.json @@ -84,6 +84,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Log-based alerts firing", + "description": "Firing and pending alerts carrying component=logs, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"logs\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Lines/s (all hosts)", @@ -96,7 +129,7 @@ "h": 4, "w": 5, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -159,7 +192,7 @@ "h": 4, "w": 5, "x": 5, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -229,7 +262,7 @@ "h": 4, "w": 5, "x": 10, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -295,7 +328,7 @@ "h": 4, "w": 9, "x": 15, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -361,7 +394,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 10 }, "panels": [] }, @@ -377,7 +410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -458,7 +491,7 @@ "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 11 }, "fieldConfig": { "defaults": { @@ -539,7 +572,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -616,7 +649,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -693,7 +726,7 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 27 }, "panels": [] }, @@ -709,7 +742,7 @@ "h": 11, "w": 24, "x": 0, - "y": 22 + "y": 28 }, "options": { "showTime": true, @@ -746,7 +779,7 @@ "h": 11, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "options": { "showTime": true, @@ -783,7 +816,7 @@ "h": 12, "w": 24, "x": 0, - "y": 44 + "y": 50 }, "options": { "showTime": true, diff --git a/stacks/observability/grafana/dashboards/network-snmp.json b/stacks/observability/grafana/dashboards/network-snmp.json index a48fb4b..0b099ef 100644 --- a/stacks/observability/grafana/dashboards/network-snmp.json +++ b/stacks/observability/grafana/dashboards/network-snmp.json @@ -39,6 +39,39 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Network & firewall alerts firing", + "description": "Firing and pending alerts carrying component=network, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"network\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "stat", "title": "Packet filter", @@ -51,7 +84,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -133,7 +166,7 @@ "h": 4, "w": 4, "x": 4, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -203,7 +236,7 @@ "h": 8, "w": 5, "x": 8, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -271,7 +304,7 @@ "h": 4, "w": 5, "x": 13, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -333,7 +366,7 @@ "h": 4, "w": 6, "x": 18, - "y": 0 + "y": 6 }, "fieldConfig": { "defaults": { @@ -395,7 +428,7 @@ "h": 4, "w": 4, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -457,7 +490,7 @@ "h": 4, "w": 4, "x": 4, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -520,7 +553,7 @@ "h": 4, "w": 5, "x": 13, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -586,7 +619,7 @@ "h": 4, "w": 6, "x": 18, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -649,7 +682,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 14 }, "panels": [] }, @@ -665,7 +698,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -754,7 +787,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 15 }, "fieldConfig": { "defaults": { @@ -855,7 +888,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -980,7 +1013,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 23 }, "fieldConfig": { "defaults": { @@ -1065,7 +1098,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 31 }, "panels": [] }, @@ -1081,7 +1114,7 @@ "h": 9, "w": 24, "x": 0, - "y": 26 + "y": 32 }, "fieldConfig": { "defaults": { @@ -1169,7 +1202,7 @@ "h": 9, "w": 24, "x": 0, - "y": 35 + "y": 41 }, "fieldConfig": { "defaults": { @@ -1286,7 +1319,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 50 }, "panels": [] }, @@ -1302,7 +1335,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 51 }, "fieldConfig": { "defaults": { @@ -1379,7 +1412,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 51 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/grafana/dashboards/ups-power.json b/stacks/observability/grafana/dashboards/ups-power.json index 7609f2f..3a8af73 100644 --- a/stacks/observability/grafana/dashboards/ups-power.json +++ b/stacks/observability/grafana/dashboards/ups-power.json @@ -39,12 +39,45 @@ ] }, "panels": [ + { + "type": "alertlist", + "title": "Power alerts firing", + "description": "Firing and pending alerts carrying component=power, from the Alertmanager datasource — so a silenced alert shows as silenced rather than simply vanishing. Empty is the good state.", + "datasource": { + "type": "alertmanager", + "uid": "alertmanager" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "alertInstanceLabelFilter": "{component=\"power\"}", + "alertName": "", + "dashboardAlerts": false, + "datasource": "alertmanager", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + } + }, { "type": "text", "title": "No battery is installed in this UPS", "gridPos": { "x": 0, - "y": 0, + "y": 6, "w": 24, "h": 4 }, @@ -65,7 +98,7 @@ "h": 4, "w": 5, "x": 0, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -154,7 +187,7 @@ "h": 8, "w": 5, "x": 5, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -222,7 +255,7 @@ "h": 8, "w": 5, "x": 10, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -290,7 +323,7 @@ "h": 4, "w": 5, "x": 15, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -360,7 +393,7 @@ "h": 4, "w": 4, "x": 20, - "y": 4 + "y": 10 }, "fieldConfig": { "defaults": { @@ -456,7 +489,7 @@ "h": 4, "w": 5, "x": 0, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -518,7 +551,7 @@ "h": 4, "w": 5, "x": 15, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -584,7 +617,7 @@ "h": 4, "w": 4, "x": 20, - "y": 8 + "y": 14 }, "fieldConfig": { "defaults": { @@ -650,7 +683,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "panels": [] }, @@ -666,7 +699,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -755,7 +788,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 19 }, "fieldConfig": { "defaults": { @@ -844,7 +877,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 27 }, "fieldConfig": { "defaults": { @@ -932,7 +965,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 27 }, "fieldConfig": { "defaults": { @@ -1020,7 +1053,7 @@ "h": 8, "w": 24, "x": 0, - "y": 29 + "y": 35 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/loki/rules/security.rules.yaml b/stacks/observability/loki/rules/security.rules.yaml index 5e0f624..76e9a52 100644 --- a/stacks/observability/loki/rules/security.rules.yaml +++ b/stacks/observability/loki/rules/security.rules.yaml @@ -20,6 +20,7 @@ groups: ) > 20 for: 2m labels: + component: logs severity: warning category: security annotations: @@ -35,6 +36,7 @@ groups: ) > 100 for: 1m labels: + component: logs severity: critical category: security annotations: @@ -56,6 +58,7 @@ groups: ) > 0 for: 1m labels: + component: logs severity: critical category: security annotations: @@ -71,6 +74,7 @@ groups: ) > 5 for: 5m labels: + component: logs severity: warning category: security annotations: @@ -82,6 +86,7 @@ groups: count_over_time({log_type="authlog"} |~ "(useradd|groupadd|usermod).*new (user|group)" [10m]) ) > 0 labels: + component: logs severity: warning category: security annotations: @@ -100,6 +105,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "Out of memory: Kill(ed)? process" [15m]) ) > 0 labels: + component: logs severity: warning category: capacity annotations: @@ -111,6 +117,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "Remounting filesystem read-only" [15m]) ) > 0 labels: + component: logs severity: critical category: hardware annotations: @@ -125,6 +132,7 @@ groups: count_over_time({log_type=~"syslog|varlog"} |~ "(I/O error|ata[0-9]+.*failed|SMART.*FAILED)" [15m]) ) > 0 labels: + component: logs severity: critical category: hardware annotations: diff --git a/stacks/observability/prometheus/rules/containers.rules.yaml b/stacks/observability/prometheus/rules/containers.rules.yaml index f9988d0..07a7fde 100644 --- a/stacks/observability/prometheus/rules/containers.rules.yaml +++ b/stacks/observability/prometheus/rules/containers.rules.yaml @@ -10,6 +10,7 @@ groups: expr: changes(container_start_time_seconds{name!=""}[15m]) > 3 for: 5m labels: + component: containers severity: warning category: availability annotations: @@ -19,6 +20,7 @@ groups: - alert: ContainerOomKilled expr: increase(container_oom_events_total{name!=""}[15m]) > 0 labels: + component: containers severity: warning category: capacity annotations: @@ -33,6 +35,7 @@ groups: and container_spec_memory_limit_bytes{name!=""} > 0 for: 15m labels: + component: containers severity: warning category: capacity annotations: @@ -44,6 +47,7 @@ groups: / clamp_min(rate(container_cpu_cfs_periods_total{name!=""}[15m]), 1) > 0.25 for: 30m labels: + component: containers severity: info category: saturation annotations: @@ -54,6 +58,7 @@ groups: expr: prometheus_config_last_reload_successful == 0 for: 5m labels: + component: containers severity: critical category: correctness annotations: @@ -63,6 +68,7 @@ groups: expr: increase(prometheus_rule_evaluation_failures_total[15m]) > 0 for: 5m labels: + component: containers severity: warning category: correctness annotations: @@ -72,6 +78,7 @@ groups: expr: increase(alertmanager_notifications_failed_total[15m]) > 0 for: 5m labels: + component: containers severity: critical category: correctness annotations: @@ -85,6 +92,7 @@ groups: sum(rate(loki_distributor_lines_received_total[30m])) == 0 for: 30m labels: + component: containers severity: warning category: availability annotations: diff --git a/stacks/observability/prometheus/rules/host.rules.yaml b/stacks/observability/prometheus/rules/host.rules.yaml index 7776db6..35c06f0 100644 --- a/stacks/observability/prometheus/rules/host.rules.yaml +++ b/stacks/observability/prometheus/rules/host.rules.yaml @@ -7,6 +7,7 @@ groups: expr: up == 0 for: 5m labels: + component: host severity: critical category: availability annotations: @@ -24,6 +25,7 @@ groups: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.30 for: 1h labels: + component: host severity: warning category: capacity annotations: @@ -39,6 +41,7 @@ groups: / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} < 0.10 for: 15m labels: + component: host severity: critical category: capacity annotations: @@ -49,6 +52,7 @@ groups: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10 for: 15m labels: + component: host severity: warning category: capacity annotations: @@ -61,6 +65,7 @@ groups: node_load15 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 2 for: 30m labels: + component: host severity: warning category: saturation annotations: @@ -70,6 +75,7 @@ groups: expr: abs(node_timex_offset_seconds) > 0.5 for: 10m labels: + component: host severity: warning category: correctness annotations: @@ -81,6 +87,7 @@ groups: - alert: HostRebooted expr: time() - node_boot_time_seconds < 600 labels: + component: host severity: info category: availability annotations: diff --git a/stacks/observability/prometheus/rules/network.rules.yaml b/stacks/observability/prometheus/rules/network.rules.yaml index 81fbbc4..9188de1 100644 --- a/stacks/observability/prometheus/rules/network.rules.yaml +++ b/stacks/observability/prometheus/rules/network.rules.yaml @@ -7,6 +7,7 @@ groups: expr: up{job="snmp-exporter"} == 0 for: 5m labels: + component: network severity: critical category: availability annotations: @@ -16,6 +17,7 @@ groups: expr: up{job="snmp"} == 0 for: 10m labels: + component: network severity: warning category: availability annotations: @@ -29,6 +31,7 @@ groups: expr: scrape_duration_seconds{job="snmp"} > 30 for: 15m labels: + component: network severity: warning category: performance annotations: @@ -42,6 +45,7 @@ groups: expr: pfStatusRunning == 0 for: 2m labels: + component: network severity: critical category: security annotations: @@ -54,6 +58,7 @@ groups: expr: pfStateTableCount / pfLimitsStates > 0.80 for: 10m labels: + component: network severity: warning category: saturation annotations: @@ -65,6 +70,7 @@ groups: expr: rate(pfCounterMemDrop[15m]) > 0 for: 15m labels: + component: network severity: warning category: saturation annotations: @@ -79,6 +85,7 @@ groups: and on (instance, ifIndex) (max_over_time(ifOperStatus[1h]) == 1) for: 5m labels: + component: network severity: warning category: availability annotations: @@ -92,6 +99,7 @@ groups: expr: rate(ifInOctets[5m]) < 0 or rate(ifOutOctets[5m]) < 0 for: 10m labels: + component: network severity: info category: correctness annotations: @@ -107,6 +115,7 @@ groups: or cpqDaPhyDrvCondition > 2 for: 5m labels: + component: network severity: critical category: hardware annotations: @@ -119,6 +128,7 @@ groups: expr: cpqHeSysBackupBatteryCondition > 2 for: 30m labels: + component: network severity: warning category: hardware annotations: diff --git a/stacks/observability/prometheus/rules/ups.rules.yaml b/stacks/observability/prometheus/rules/ups.rules.yaml index 701d58f..cee7003 100644 --- a/stacks/observability/prometheus/rules/ups.rules.yaml +++ b/stacks/observability/prometheus/rules/ups.rules.yaml @@ -40,6 +40,7 @@ groups: expr: upsSecondsOnBattery > 0 for: 30s labels: + component: power severity: critical category: power annotations: @@ -53,6 +54,7 @@ groups: expr: upsBatteryStatus > 2 for: 1m labels: + component: power severity: critical category: power annotations: @@ -62,6 +64,7 @@ groups: expr: upsEstimatedMinutesRemaining < 5 for: 1m labels: + component: power severity: critical category: power annotations: @@ -71,6 +74,7 @@ groups: expr: upsEstimatedChargeRemaining < 50 for: 10m labels: + component: power severity: warning category: power annotations: @@ -80,6 +84,7 @@ groups: expr: upsOutputPercentLoad > 80 for: 15m labels: + component: power severity: warning category: power annotations: @@ -91,6 +96,7 @@ groups: expr: upsAlarmsPresent > 0 for: 5m labels: + component: power severity: warning category: power annotations: @@ -100,6 +106,7 @@ groups: expr: upsBatteryTemperature > 40 for: 15m labels: + component: power severity: warning category: power annotations: