From 78f2779bc8b3894804aec656a2136f4f23b46109 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 19:28:54 +0000 Subject: [PATCH] Add hourly health check with external dead-man's-switch alerting modules/monitoring.nix checks services and failed units, the TLS certificate (expiry, self-signed placeholder), disk usage, queue size, outbound SMTP and the public IP on Spamhaus ZEN/SpamCop, and reports to settings.monitoring.healthcheckUrl (healthchecks.io, Uptime Kuma push, ...), which also alerts when reports stop. Without a URL, problems are mailed to adminEmail at most once a day. just health runs it on demand. Closes #7 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22o7QQ17GRtkCqpJNFr9B --- README.md | 1 + flake.nix | 1 + justfile | 4 ++ modules/monitoring.nix | 139 +++++++++++++++++++++++++++++++++++++++++ settings.nix | 12 ++++ tests/settings.nix | 5 ++ 6 files changed, 162 insertions(+) create mode 100644 modules/monitoring.nix diff --git a/README.md b/README.md index a6b0acb..329040d 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ can't change secrets or reinstall the server from this repository. | Rotate a DKIM key | `just add-domain ` (new date-based selector), then `just apply && just deploy`. A few days later, delete the old `dkim//.txt` and its entry in `just secrets`, then apply and deploy again | | Send through a relay | Fill in `relay` in `settings.nix` and set `enable = true`, run `just relay-password`, then `just apply && just deploy` | | Security updates | `just update && just deploy`. A weekly GitHub Action also opens a PR that updates `flake.lock`; or turn on `autoUpgrade` | +| Monitoring | Set `monitoring.healthcheckUrl` (healthchecks.io or similar): an hourly self-check of services, certificate, disk, queue, outbound SMTP and blocklists alerts you on failure **and** when the server stops reporting. `just health` runs it now | | Logs and queue | `just logs`, or `just ssh` | | Rspamd web UI | `ssh -L 11334:/run/rspamd/worker-controller.sock root@mail.example.com`, then open http://localhost:11334 | diff --git a/flake.nix b/flake.nix index 4ee6c01..85495b5 100644 --- a/flake.nix +++ b/flake.nix @@ -67,6 +67,7 @@ ./modules/mail.nix ./modules/relay.nix ./modules/web.nix + ./modules/monitoring.nix { networking.hostName = settings.hostname; networking.domain = settings.primaryDomain; diff --git a/justfile b/justfile index 6889fb6..8007d5f 100644 --- a/justfile +++ b/justfile @@ -65,6 +65,10 @@ check: ssh *command: source scripts/lib.sh && ssh $(ssh_opts) "root@$(fqdn)" {{command}} +# Run the health check now and show its result +health: + just ssh "'systemctl start mail-health-check; journalctl -u mail-health-check -n 20 --no-pager -o cat'" + # Show the mail queue and recent mail log on the server logs: just ssh "'postqueue -p; journalctl -u postfix -u dovecot -u rspamd -n 100 --no-pager'" diff --git a/modules/monitoring.nix b/modules/monitoring.nix new file mode 100644 index 0000000..4d80a39 --- /dev/null +++ b/modules/monitoring.nix @@ -0,0 +1,139 @@ +# Hourly self-check of the mail server, reported to an external +# "dead man's switch" (healthchecks.io, a self-hosted Healthchecks or Uptime +# Kuma push monitor, ...). The external service alerts you when a check +# fails AND when reports stop arriving, so it also catches the server being +# down entirely, which no check running on the server itself can. +# +# Without monitoring.healthcheckUrl, failures are mailed to adminEmail +# instead (at most once a day), which only works while the server can still +# send mail. +{ + lib, + pkgs, + settings, + ctx, + ... +}: +let + cfg = settings.monitoring; + stateDir = "/var/lib/mail-health"; +in +{ + systemd.services.mail-health-check = { + description = "Mail server health check"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = with pkgs; [ + bash + coreutils + curl + dig + gawk + gnugrep + openssl + postfix + systemd + util-linux + ]; + serviceConfig = { + Type = "oneshot"; + StateDirectory = "mail-health"; + }; + script = '' + set -u + problems=() + problem() { problems+=("$*"); echo "PROBLEM: $*" >&2; } + + # 1. Services + for unit in postfix dovecot rspamd redis-rspamd nginx kresd@1 fail2ban; do + systemctl is-active --quiet "$unit" || problem "service $unit is not running" + done + failed=$(systemctl list-units --failed --plain --no-legend | awk '{print $1}' | tr '\n' ' ') + [[ -z "$failed" ]] || problem "failed systemd units: $failed" + + # 2. TLS certificate (Postfix/Dovecot); MTA-STS makes an expired one fatal + cert=/var/lib/acme/${ctx.fqdn}/cert.pem + if [[ ! -f $cert ]]; then + problem "no TLS certificate at $cert" + elif ! openssl x509 -checkend $((14 * 86400)) -noout -in "$cert" > /dev/null; then + problem "TLS certificate expires within 14 days: $(openssl x509 -enddate -noout -in "$cert")" + elif openssl x509 -noout -issuer -in "$cert" | grep -qi minica; then + problem "TLS certificate is the self-signed placeholder: Let's Encrypt issuance failed" + fi + + # 3. Disk + used=$(df --output=pcent / | tail -n1 | tr -dc 0-9) + (( used < ${toString cfg.diskWarnPercent} )) || problem "disk / is $used% full" + + # 4. Mail queue + queued=$(postqueue -j 2> /dev/null | wc -l) + (( queued < ${toString cfg.queueWarn} )) || problem "$queued messages in the mail queue" + + # 5. Outbound SMTP (blocked by the provider, or by OVH's anti-spam) + if ! timeout 10 bash -c 'exec 3<>/dev/tcp/gmail-smtp-in.l.google.com/25' 2> /dev/null; then + problem "cannot connect to gmail-smtp-in.l.google.com:25: outbound SMTP blocked?" + fi + + # 6. Blocklists, for the public IPv4 address (queried through the local + # resolver; 127.255.255.x means the list refused the query) + ip=$(curl -4 -fsS -m 10 https://api.ipify.org || true) + if [[ $ip =~ ^[0-9.]+$ ]]; then + rev=$(awk -F. '{print $4"."$3"."$2"."$1}' <<< "$ip") + for bl in zen.spamhaus.org bl.spamcop.net; do + answer=$(dig +short "$rev.$bl" A @127.0.0.1 | head -n1) + if [[ -n $answer && $answer != 127.255.255.* ]]; then + problem "$ip is listed on $bl ($answer)" + fi + done + fi + + # Report + if (( ''${#problems[@]} == 0 )); then + report="OK" + else + report=$(printf '%s\n' "''${problems[@]}") + fi + echo "$report" + ${ + if cfg.healthcheckUrl != "" then + '' + if (( ''${#problems[@]} == 0 )); then + curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}" > /dev/null + else + curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}/fail" > /dev/null + fi + '' + else + '' + if (( ''${#problems[@]} > 0 )) \ + && ! [[ $(find ${stateDir}/last-mail -mmin -1440 2> /dev/null) ]]; then + /run/wrappers/bin/sendmail -t <" + diskWarnPercent = 85; + queueWarn = 100; + }; + timeZone = "UTC"; # Where the server runs. Each provider has a Terraform step in diff --git a/tests/settings.nix b/tests/settings.nix index bfd5b14..013de14 100644 --- a/tests/settings.nix +++ b/tests/settings.nix @@ -41,6 +41,11 @@ import ../settings.nix username = "ci"; spfInclude = "spf.relay.test"; }; + monitoring = { + healthcheckUrl = "https://hc-ping.example.test/ci"; + diskWarnPercent = 85; + queueWarn = 100; + }; autoUpgrade = { enable = true; flake = "github:example/mailserver";