From b95fbba2b8504f6b3d50579ac8e2aa69a61f5c48 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Aug 2026 03:56:57 +0000 Subject: [PATCH] chore(secrets): point Alertmanager at a real receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ALERTMANAGER_WEBHOOK_URL was still the placeholder from observability.example.yaml — ntfy.example.invalid. Alertmanager had been resolving it on every notification, failing DNS, and giving up after 15 retries. No alert had ever been delivered since the stack was built: all 40 rules were firing into nothing, and AlertmanagerNotificationsFailing was itself undeliverable for the same reason. Now an ntfy.sh topic. External on purpose: the alerting stack runs on the rack it monitors, so a self-hosted receiver would die with the thing it is meant to warn about — including a mains loss, which is the scenario the missing UPS battery makes unrecoverable. The topic is a 20-character random string and functions as a bearer secret, which is why it lives here rather than in a config file. The trade-off is that alert text — hostnames, addresses, roles — transits a third-party server. Switching to a self-hosted or private receiver is one `make secrets-edit` away and needs no other change. Verified end to end: both alerts that had been stuck in the retry queue were delivered on reload. Co-Authored-By: Claude Opus 5 --- secrets/observability.sops.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/secrets/observability.sops.yaml b/secrets/observability.sops.yaml index 19b57e9..192d3aa 100644 --- a/secrets/observability.sops.yaml +++ b/secrets/observability.sops.yaml @@ -27,7 +27,7 @@ SNMP_COMMUNITY_MOKERLINK: ENC[AES256_GCM,data:v+SIe7w9gpQkHYFTiHWB,iv:B4r0MxOESo SNMP_COMMUNITY_ILO: ENC[AES256_GCM,data:RMutxt3mfxMlthO8IQp2wA9x54+SLQuF,iv:fPtLWUwAAojRjX1e7bN6z5f75GNKwy0GTWyr/bFZorM=,tag:18OmJT6h6MiK6UZdwxkfZA==,type:str] #ENC[AES256_GCM,data:KYm60Zx9WHuw0PObbmuV5uOxqGJG51YHztUtmrAPocdqvPmnbSQJthKe3+PngPrfFfn7G8RfRANrDTgboVgQOWbwtvIlhK8CutPr16gq2rM=,iv:elgWymJ/VyQzT63VSPBtWYIsJY26GeG2T4MYZCmInWY=,tag:Px8SEziU2qyNYMMI0B1WAg==,type:comment] #ENC[AES256_GCM,data:c+QE2sE9agdh20ZSaWSTdnENoeFagYWbpy9D7oVKofDsi9oBM8rxxrlp4s8CfB3KcNv7uFBD2SZdMBVDltjfkJICEjKgfJCBQ8A=,iv:PQBt/eUj95BdLusTJc5o3ZwyqIZGy9/XEG/AQTHk/IU=,tag:spc+wLic1oKSlHgvBjiLSg==,type:comment] -ALERTMANAGER_WEBHOOK_URL: ENC[AES256_GCM,data:9RCG8XSICAFemb4BuFVMyE6WO6SdIpikzX4l8E4JJVPxv107yJaXHwHE9g==,iv:N1MhJ7V8wLr8oFuDjI2RiWk/QOTshznPupXLLqJVoTM=,tag:61DcbSMlu+SuIPmeJfEbyA==,type:str] +ALERTMANAGER_WEBHOOK_URL: ENC[AES256_GCM,data:w1Wf/HOjRiWeEPykftp/E1WU0nTXjyftnHM08eRxTU6asgG09a8TA9XSoIM=,iv:u8eNDX2UThz8F4RxbLTIflV17Na14NGzyaHOP9Xk6bU=,tag:Ufv+P7QT4IoKFUpM+TE5Vg==,type:str] sops: kms: [] gcp_kms: [] @@ -43,8 +43,8 @@ sops: emRhdUUyUGdHMWFlSVE3alNuekp3YTQKAN5Dt+v1PnZlFDZgN2NQ09dGiIfMr1S9 pIUfYyxRjC2qM5prt3Lhfcp0qvvvCsrm857wvxteXNb1jAfGHQoQHQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-19T02:31:11Z" - mac: ENC[AES256_GCM,data:Tb1H7Rb4GaMsRFlocUtN/8nmK11hN80Z+P5iwx82H31Z8Hr/tjP/ZltJHXXCoudq2rajR7cjsxdgmaljqJj7mPiGsS8Bwyf3qE4IIo7QuaIfq4nXKpnyynkXiW/tg6tipyWt96DdqyLjLOLjSvhVJlUwDa6HM7vFN+3d4bSqTPw=,iv:+uuSW6r2rnw7SSqvQRg63KdAAbBcIXLJlDANgSd7rog=,tag:iIL/uPrMFnS3QZRRuZBVsQ==,type:str] + lastmodified: "2026-08-19T03:55:44Z" + mac: ENC[AES256_GCM,data:QrVdd5IwcTIZix1uLesau/3xImqBh1QCkXL7ZgfFe4zCFSHPg9+Lm32QUCBqCQ7zLhHeJmah3I5Qei7RXVaYuJwsBMiVKQwfVzJbO1MfFJiGZnrLLfG0suyHOG7NC/8LdS7FGo54la/88UmqQmPoxh6heC3ANo3iPeOybX9C3GY=,iv:TgMNN4hLnmur2fWN/NWVR7Zd3+2htZW6lqKFsyAU03o=,tag:P3jbtd9chHxPo2wBcbHEYg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.4