From c96954d3466a29f16639064e3ad01cbd5eca5a51 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Aug 2026 03:50:45 +0000 Subject: [PATCH] fix(observability): stop the UPS dashboard reporting a battery that isn't there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mjolnir has no battery pack fitted. Its management card does not report that — it fabricates a healthy one: upsBatteryStatus 2 (batteryNormal) upsEstimatedChargeRemaining 100 upsEstimatedMinutesRemaining 63 upsBatteryVoltage 480 (48.0 V) upsAlarmsPresent 0 So the dashboard shows a green battery status and 63 minutes of runtime that do not exist, and UpsBatteryLow, UpsChargeLow, UpsRuntimeCritical and UpsAlarmPresent all sit comfortably inside their thresholds and will stay there. The missing battery is known and tracked. The hazard this fixes is different: the monitoring currently gives positive reassurance about it. A green UPS panel reads as "an hour of runtime", at the exact moment that is most wrong. Adds a banner to the dashboard, marks the four fabricated panels in their titles and descriptions, and rewrites the rules-file header to say plainly that those rules cannot fire for this condition. Nothing is disabled: every rule becomes correct the moment a pack is fitted. Also marks Output power, which this NMC firmware reports as 0 regardless of actual draw while Output load reads 15%. Records the trap too: do not run a self-test to force an honest reading. A self-test transfers the load to battery, and with none fitted that is the outage it exists to warn about. Refs docs/roadmap.md Co-Authored-By: Claude Opus 5 --- .../grafana/dashboards/ups-power.json | 66 +++++++++++-------- .../prometheus/rules/ups.rules.yaml | 35 ++++++++-- 2 files changed, 71 insertions(+), 30 deletions(-) diff --git a/stacks/observability/grafana/dashboards/ups-power.json b/stacks/observability/grafana/dashboards/ups-power.json index 807423a..7609f2f 100644 --- a/stacks/observability/grafana/dashboards/ups-power.json +++ b/stacks/observability/grafana/dashboards/ups-power.json @@ -39,6 +39,20 @@ ] }, "panels": [ + { + "type": "text", + "title": "No battery is installed in this UPS", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 4 + }, + "options": { + "mode": "markdown", + "content": "**Every battery figure on this dashboard is fabricated.** `mjolnir` has no battery pack fitted, yet its management card still reports `batteryNormal`, 100% charge, 48.0 V and a runtime estimate. It raises no alarm (`upsAlarmsPresent = 0`).\n\nA mains loss is therefore an **immediate hard shutdown of the rack**, with no warning from here. Treat charge, runtime, battery status and battery temperature below as decoration until a pack is fitted — every alert in `ups.rules.yaml` is dead for the same reason.\n\nDo **not** run a UPS self-test to get an honest reading: a self-test transfers the load to battery, and with none fitted that is the outage it is meant to warn about. Tracked in `docs/roadmap.md`." + } + }, { "type": "stat", "title": "Power source", @@ -51,7 +65,7 @@ "h": 4, "w": 5, "x": 0, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -130,8 +144,8 @@ }, { "type": "gauge", - "title": "Battery charge", - "description": "", + "title": "Battery charge (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -140,7 +154,7 @@ "h": 8, "w": 5, "x": 5, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -208,7 +222,7 @@ "h": 8, "w": 5, "x": 10, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -266,8 +280,8 @@ }, { "type": "stat", - "title": "Runtime remaining", - "description": "", + "title": "Runtime remaining (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -276,7 +290,7 @@ "h": 4, "w": 5, "x": 15, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -336,8 +350,8 @@ }, { "type": "stat", - "title": "Battery status", - "description": "UPS-MIB: 1 unknown, 2 normal, 3 low, 4 depleted.", + "title": "Battery status (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -346,7 +360,7 @@ "h": 4, "w": 4, "x": 20, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -432,8 +446,8 @@ }, { "type": "stat", - "title": "Output power", - "description": "", + "title": "Output power (not reported by this NMC)", + "description": "This NMC firmware reports upsOutputPower as 0 regardless of actual draw; use Output load instead.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -442,7 +456,7 @@ "h": 4, "w": 5, "x": 0, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -504,7 +518,7 @@ "h": 4, "w": 5, "x": 15, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -560,8 +574,8 @@ }, { "type": "stat", - "title": "Battery temp", - "description": "", + "title": "Battery temp (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -570,7 +584,7 @@ "h": 4, "w": 4, "x": 20, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -636,14 +650,14 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "panels": [] }, { "type": "timeseries", - "title": "Battery charge and runtime", - "description": "", + "title": "Battery charge and runtime (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -652,7 +666,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 13 }, "fieldConfig": { "defaults": { @@ -741,7 +755,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 13 }, "fieldConfig": { "defaults": { @@ -830,7 +844,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 21 }, "fieldConfig": { "defaults": { @@ -918,7 +932,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 21 }, "fieldConfig": { "defaults": { @@ -1006,7 +1020,7 @@ "h": 8, "w": 24, "x": 0, - "y": 25 + "y": 29 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/prometheus/rules/ups.rules.yaml b/stacks/observability/prometheus/rules/ups.rules.yaml index 7ac59c5..701d58f 100644 --- a/stacks/observability/prometheus/rules/ups.rules.yaml +++ b/stacks/observability/prometheus/rules/ups.rules.yaml @@ -1,10 +1,37 @@ --- # APC Smart-UPS (mjolnir, 10.0.99.10) via UPS-MIB. # -# The UPS currently has no working battery — see docs/roadmap.md. Until that is -# fixed a mains loss is an immediate hard shutdown of the whole rack, which is -# why every rule here is deliberately loud and routed to the `urgent` receiver -# by category=power in alertmanager.yaml. +# READ THIS BEFORE TRUSTING ANY ALERT BELOW. +# +# The UPS has no battery pack fitted — see docs/roadmap.md. A mains loss is an +# immediate hard shutdown of the whole rack. Every rule here is deliberately +# loud and routed to the `urgent` receiver by category=power in +# alertmanager.yaml. +# +# None of them can fire for that condition. The management card does not report +# the missing pack; it fabricates a healthy one. Observed on a UPS with no +# battery physically present: +# +# upsBatteryStatus 2 (batteryNormal) +# upsEstimatedChargeRemaining 100 (percent) +# upsEstimatedMinutesRemaining 63 (minutes of runtime it does not have) +# upsBatteryVoltage 480 (48.0 V) +# upsAlarmsPresent 0 (no alarm raised) +# +# So UpsBatteryLow, UpsChargeLow, UpsRuntimeCritical and UpsAlarmPresent are +# all comfortably inside their thresholds and will stay there. These rules are +# kept because they become correct the moment a pack is fitted — not because +# they are protecting anything today. A quiet `ups` group is not evidence of +# runtime. +# +# UpsOnBattery, UpsOverloaded and UpsBatteryTemperatureHigh read mains-side or +# load-side values and are unaffected. UpsOnBattery in particular would still +# fire on a transfer — it would just be reporting an outage already in progress +# rather than warning of one. +# +# Do not run a UPS self-test to force an honest reading. A self-test transfers +# the load to battery; with none fitted, that is the outage it exists to warn +# about. upsTestResultsSummary is 6 (noTestsInitiated) and should stay there. groups: - name: ups interval: 60s