diff --git a/stacks/observability/grafana/dashboards/ups-power.json b/stacks/observability/grafana/dashboards/ups-power.json index 807423a..7609f2f 100644 --- a/stacks/observability/grafana/dashboards/ups-power.json +++ b/stacks/observability/grafana/dashboards/ups-power.json @@ -39,6 +39,20 @@ ] }, "panels": [ + { + "type": "text", + "title": "No battery is installed in this UPS", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 4 + }, + "options": { + "mode": "markdown", + "content": "**Every battery figure on this dashboard is fabricated.** `mjolnir` has no battery pack fitted, yet its management card still reports `batteryNormal`, 100% charge, 48.0 V and a runtime estimate. It raises no alarm (`upsAlarmsPresent = 0`).\n\nA mains loss is therefore an **immediate hard shutdown of the rack**, with no warning from here. Treat charge, runtime, battery status and battery temperature below as decoration until a pack is fitted — every alert in `ups.rules.yaml` is dead for the same reason.\n\nDo **not** run a UPS self-test to get an honest reading: a self-test transfers the load to battery, and with none fitted that is the outage it is meant to warn about. Tracked in `docs/roadmap.md`." + } + }, { "type": "stat", "title": "Power source", @@ -51,7 +65,7 @@ "h": 4, "w": 5, "x": 0, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -130,8 +144,8 @@ }, { "type": "gauge", - "title": "Battery charge", - "description": "", + "title": "Battery charge (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -140,7 +154,7 @@ "h": 8, "w": 5, "x": 5, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -208,7 +222,7 @@ "h": 8, "w": 5, "x": 10, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -266,8 +280,8 @@ }, { "type": "stat", - "title": "Runtime remaining", - "description": "", + "title": "Runtime remaining (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -276,7 +290,7 @@ "h": 4, "w": 5, "x": 15, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -336,8 +350,8 @@ }, { "type": "stat", - "title": "Battery status", - "description": "UPS-MIB: 1 unknown, 2 normal, 3 low, 4 depleted.", + "title": "Battery status (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -346,7 +360,7 @@ "h": 4, "w": 4, "x": 20, - "y": 0 + "y": 4 }, "fieldConfig": { "defaults": { @@ -432,8 +446,8 @@ }, { "type": "stat", - "title": "Output power", - "description": "", + "title": "Output power (not reported by this NMC)", + "description": "This NMC firmware reports upsOutputPower as 0 regardless of actual draw; use Output load instead.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -442,7 +456,7 @@ "h": 4, "w": 5, "x": 0, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -504,7 +518,7 @@ "h": 4, "w": 5, "x": 15, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -560,8 +574,8 @@ }, { "type": "stat", - "title": "Battery temp", - "description": "", + "title": "Battery temp (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -570,7 +584,7 @@ "h": 4, "w": 4, "x": 20, - "y": 4 + "y": 8 }, "fieldConfig": { "defaults": { @@ -636,14 +650,14 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "panels": [] }, { "type": "timeseries", - "title": "Battery charge and runtime", - "description": "", + "title": "Battery charge and runtime (fabricated — no battery fitted)", + "description": "No battery pack is fitted to mjolnir. The management card reports this value anyway; it does not reflect any hardware. See the banner at the top of this dashboard.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -652,7 +666,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 13 }, "fieldConfig": { "defaults": { @@ -741,7 +755,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 13 }, "fieldConfig": { "defaults": { @@ -830,7 +844,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 21 }, "fieldConfig": { "defaults": { @@ -918,7 +932,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 21 }, "fieldConfig": { "defaults": { @@ -1006,7 +1020,7 @@ "h": 8, "w": 24, "x": 0, - "y": 25 + "y": 29 }, "fieldConfig": { "defaults": { diff --git a/stacks/observability/prometheus/rules/ups.rules.yaml b/stacks/observability/prometheus/rules/ups.rules.yaml index 7ac59c5..701d58f 100644 --- a/stacks/observability/prometheus/rules/ups.rules.yaml +++ b/stacks/observability/prometheus/rules/ups.rules.yaml @@ -1,10 +1,37 @@ --- # APC Smart-UPS (mjolnir, 10.0.99.10) via UPS-MIB. # -# The UPS currently has no working battery — see docs/roadmap.md. Until that is -# fixed a mains loss is an immediate hard shutdown of the whole rack, which is -# why every rule here is deliberately loud and routed to the `urgent` receiver -# by category=power in alertmanager.yaml. +# READ THIS BEFORE TRUSTING ANY ALERT BELOW. +# +# The UPS has no battery pack fitted — see docs/roadmap.md. A mains loss is an +# immediate hard shutdown of the whole rack. Every rule here is deliberately +# loud and routed to the `urgent` receiver by category=power in +# alertmanager.yaml. +# +# None of them can fire for that condition. The management card does not report +# the missing pack; it fabricates a healthy one. Observed on a UPS with no +# battery physically present: +# +# upsBatteryStatus 2 (batteryNormal) +# upsEstimatedChargeRemaining 100 (percent) +# upsEstimatedMinutesRemaining 63 (minutes of runtime it does not have) +# upsBatteryVoltage 480 (48.0 V) +# upsAlarmsPresent 0 (no alarm raised) +# +# So UpsBatteryLow, UpsChargeLow, UpsRuntimeCritical and UpsAlarmPresent are +# all comfortably inside their thresholds and will stay there. These rules are +# kept because they become correct the moment a pack is fitted — not because +# they are protecting anything today. A quiet `ups` group is not evidence of +# runtime. +# +# UpsOnBattery, UpsOverloaded and UpsBatteryTemperatureHigh read mains-side or +# load-side values and are unaffected. UpsOnBattery in particular would still +# fire on a transfer — it would just be reporting an outage already in progress +# rather than warning of one. +# +# Do not run a UPS self-test to force an honest reading. A self-test transfers +# the load to battery; with none fitted, that is the outage it exists to warn +# about. upsTestResultsSummary is 6 (noTestsInitiated) and should stay there. groups: - name: ups interval: 60s