Add hourly health check with external dead-man's-switch alerting

README.md

@@ -101,6 +101,7 @@ can't change secrets or reinstall the server from this repository.
 | Rotate a DKIM key | `just add-domain <domain>` (new date-based selector), then `just apply && just deploy`. A few days later, delete the old `dkim/<domain>/<selector>.txt` and its entry in `just secrets`, then apply and deploy again |
 | Send through a relay | Fill in `relay` in `settings.nix` and set `enable = true`, run `just relay-password`, then `just apply && just deploy` |
 | Security updates | `just update && just deploy`. A weekly GitHub Action also opens a PR that updates `flake.lock`; or turn on `autoUpgrade` |
+| Monitoring | Set `monitoring.healthcheckUrl` (healthchecks.io or similar): an hourly self-check of services, certificate, disk, queue, outbound SMTP and blocklists alerts you on failure **and** when the server stops reporting. `just health` runs it now |
 | Logs and queue | `just logs`, or `just ssh` |
 | Rspamd web UI | `ssh -L 11334:/run/rspamd/worker-controller.sock [email protected]`, then open http://localhost:11334 |
 

flake.nix

@@ -67,6 +67,7 @@
               ./modules/mail.nix
               ./modules/relay.nix
               ./modules/web.nix
+              ./modules/monitoring.nix
               ./modules/release.nix
               {
                 networking.hostName = settings.hostname;

justfile

@@ -70,6 +70,10 @@ check:
 ssh *command:
     source scripts/lib.sh && ssh $(ssh_opts) "root@$(fqdn)" {{command}}
 
+# Run the health check now and show its result
+health:
+    just ssh "'systemctl start mail-health-check; journalctl -u mail-health-check -n 20 --no-pager -o cat'"
+
 # Show the mail queue and recent mail log on the server
 logs:
     just ssh "'postqueue -p; journalctl -u postfix -u dovecot -u rspamd -n 100 --no-pager'"

modules/monitoring.nix

@@ -0,0 +1,139 @@
+# Hourly self-check of the mail server, reported to an external
+# "dead man's switch" (healthchecks.io, a self-hosted Healthchecks or Uptime
+# Kuma push monitor, ...). The external service alerts you when a check
+# fails AND when reports stop arriving, so it also catches the server being
+# down entirely, which no check running on the server itself can.
+#
+# Without monitoring.healthcheckUrl, failures are mailed to adminEmail
+# instead (at most once a day), which only works while the server can still
+# send mail.
+{
+  lib,
+  pkgs,
+  settings,
+  ctx,
+  ...
+}:
+let
+  cfg = settings.monitoring;
+  stateDir = "/var/lib/mail-health";
+in
+{
+  systemd.services.mail-health-check = {
+    description = "Mail server health check";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    path = with pkgs; [
+      bash
+      coreutils
+      curl
+      dig
+      gawk
+      gnugrep
+      openssl
+      postfix
+      systemd
+      util-linux
+    ];
+    serviceConfig = {
+      Type = "oneshot";
+      StateDirectory = "mail-health";
+    };
+    script = ''
+      set -u
+      problems=()
+      problem() { problems+=("$*"); echo "PROBLEM: $*" >&2; }
+
+      # 1. Services
+      for unit in postfix dovecot rspamd redis-rspamd nginx kresd@1 fail2ban; do
+        systemctl is-active --quiet "$unit" || problem "service $unit is not running"
+      done
+      failed=$(systemctl list-units --failed --plain --no-legend | awk '{print $1}' | tr '\n' ' ')
+      [[ -z "$failed" ]] || problem "failed systemd units: $failed"
+
+      # 2. TLS certificate (Postfix/Dovecot); MTA-STS makes an expired one fatal
+      cert=/var/lib/acme/${ctx.fqdn}/cert.pem
+      if [[ ! -f $cert ]]; then
+        problem "no TLS certificate at $cert"
+      elif ! openssl x509 -checkend $((14 * 86400)) -noout -in "$cert" > /dev/null; then
+        problem "TLS certificate expires within 14 days: $(openssl x509 -enddate -noout -in "$cert")"
+      elif openssl x509 -noout -issuer -in "$cert" | grep -qi minica; then
+        problem "TLS certificate is the self-signed placeholder: Let's Encrypt issuance failed"
+      fi
+
+      # 3. Disk
+      used=$(df --output=pcent / | tail -n1 | tr -dc 0-9)
+      (( used < ${toString cfg.diskWarnPercent} )) || problem "disk / is $used% full"
+
+      # 4. Mail queue
+      queued=$(postqueue -j 2> /dev/null | wc -l)
+      (( queued < ${toString cfg.queueWarn} )) || problem "$queued messages in the mail queue"
+
+      # 5. Outbound SMTP (blocked by the provider, or by OVH's anti-spam)
+      if ! timeout 10 bash -c 'exec 3<>/dev/tcp/gmail-smtp-in.l.google.com/25' 2> /dev/null; then
+        problem "cannot connect to gmail-smtp-in.l.google.com:25: outbound SMTP blocked?"
+      fi
+
+      # 6. Blocklists, for the public IPv4 address (queried through the local
+      #    resolver; 127.255.255.x means the list refused the query)
+      ip=$(curl -4 -fsS -m 10 https://api.ipify.org || true)
+      if [[ $ip =~ ^[0-9.]+$ ]]; then
+        rev=$(awk -F. '{print $4"."$3"."$2"."$1}' <<< "$ip")
+        for bl in zen.spamhaus.org bl.spamcop.net; do
+          answer=$(dig +short "$rev.$bl" A @127.0.0.1 | head -n1)
+          if [[ -n $answer && $answer != 127.255.255.* ]]; then
+            problem "$ip is listed on $bl ($answer)"
+          fi
+        done
+      fi
+
+      # Report
+      if (( ''${#problems[@]} == 0 )); then
+        report="OK"
+      else
+        report=$(printf '%s\n' "''${problems[@]}")
+      fi
+      echo "$report"
+      ${
+        if cfg.healthcheckUrl != "" then
+          ''
+            if (( ''${#problems[@]} == 0 )); then
+              curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}" > /dev/null
+            else
+              curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}/fail" > /dev/null
+            fi
+          ''
+        else
+          ''
+            if (( ''${#problems[@]} > 0 )) \
+               && ! [[ $(find ${stateDir}/last-mail -mmin -1440 2> /dev/null) ]]; then
+              /run/wrappers/bin/sendmail -t <<MAIL
+            From: root@${ctx.fqdn}
+            To: ${settings.adminEmail}
+            Subject: Mail server ${ctx.fqdn}: ''${#problems[@]} problem(s)
+
+            $report
+            MAIL
+              touch ${stateDir}/last-mail
+            fi
+          ''
+      }
+    '';
+  };
+
+  systemd.timers.mail-health-check = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "hourly";
+      RandomizedDelaySec = "5min";
+      Persistent = true;
+    };
+  };
+
+  assertions = [
+    {
+      assertion = cfg.healthcheckUrl == "" || lib.hasPrefix "https://" cfg.healthcheckUrl;
+      message = "settings.nix: monitoring.healthcheckUrl must be an https:// URL.";
+    }
+  ];
+}

settings.nix

@@ -126,6 +126,18 @@
     flake = "github:your-user/mailserver";
   };
 
+  # Hourly self-check: services, TLS certificate, disk, mail queue, outbound
+  # SMTP and blocklists. Create a check at https://healthchecks.io (free) or
+  # your own Healthchecks / Uptime Kuma "push" monitor with a 1 hour period,
+  # and paste its ping URL here: you are then alerted when a check fails AND
+  # when the server stops reporting (e.g. because it is down). Without a URL,
+  # problems are mailed to adminEmail, which can't work if mail is broken.
+  monitoring = {
+    healthcheckUrl = ""; # e.g. "https://hc-ping.com/<uuid>"
+    diskWarnPercent = 85;
+    queueWarn = 100;
+  };
+
   timeZone = "UTC";
 
   # Where the server runs. Each provider has a Terraform step in

tests/settings.nix

@@ -41,6 +41,11 @@ import ../settings.nix
     username = "ci";
     spfInclude = "spf.relay.test";
   };
+  monitoring = {
+    healthcheckUrl = "https://hc-ping.example.test/ci";
+    diskWarnPercent = 85;
+    queueWarn = 100;
+  };
   autoUpgrade = {
     enable = true;
     flake = "github:example/mailserver";