diff --git a/README.md b/README.md index 390464b..58adb94 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ can't change secrets or reinstall the server from this repository. | Rotate a DKIM key | `just add-domain ` (new date-based selector), then `just apply && just deploy`. A few days later, delete the old `dkim//.txt` and its entry in `just secrets`, then apply and deploy again | | Send through a relay | Fill in `relay` in `settings.nix` and set `enable = true`, run `just relay-password`, then `just apply && just deploy` | | Security updates | `just update && just deploy`. A weekly GitHub Action also opens a PR that updates `flake.lock`; or turn on `autoUpgrade` | +| Monitoring | Set `monitoring.healthcheckUrl` (healthchecks.io or similar): an hourly self-check of services, certificate, disk, queue, outbound SMTP and blocklists alerts you on failure **and** when the server stops reporting. `just health` runs it now | | Logs and queue | `just logs`, or `just ssh` | | Rspamd web UI | `ssh -L 11334:/run/rspamd/worker-controller.sock root@mail.example.com`, then open http://localhost:11334 | diff --git a/flake.nix b/flake.nix index 2c3b9b5..aff53aa 100644 --- a/flake.nix +++ b/flake.nix @@ -67,6 +67,7 @@ ./modules/mail.nix ./modules/relay.nix ./modules/web.nix + ./modules/monitoring.nix ./modules/release.nix { networking.hostName = settings.hostname; diff --git a/justfile b/justfile index 4bb0032..3f0d771 100644 --- a/justfile +++ b/justfile @@ -70,6 +70,10 @@ check: ssh *command: source scripts/lib.sh && ssh $(ssh_opts) "root@$(fqdn)" {{command}} +# Run the health check now and show its result +health: + just ssh "'systemctl start mail-health-check; journalctl -u mail-health-check -n 20 --no-pager -o cat'" + # Show the mail queue and recent mail log on the server logs: just ssh "'postqueue -p; journalctl -u postfix -u dovecot -u rspamd -n 100 --no-pager'" diff --git a/modules/monitoring.nix b/modules/monitoring.nix new file mode 100644 index 0000000..4d80a39 --- /dev/null +++ b/modules/monitoring.nix @@ -0,0 +1,139 @@ +# Hourly self-check of the mail server, reported to an external +# "dead man's switch" (healthchecks.io, a self-hosted Healthchecks or Uptime +# Kuma push monitor, ...). The external service alerts you when a check +# fails AND when reports stop arriving, so it also catches the server being +# down entirely, which no check running on the server itself can. +# +# Without monitoring.healthcheckUrl, failures are mailed to adminEmail +# instead (at most once a day), which only works while the server can still +# send mail. +{ + lib, + pkgs, + settings, + ctx, + ... +}: +let + cfg = settings.monitoring; + stateDir = "/var/lib/mail-health"; +in +{ + systemd.services.mail-health-check = { + description = "Mail server health check"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = with pkgs; [ + bash + coreutils + curl + dig + gawk + gnugrep + openssl + postfix + systemd + util-linux + ]; + serviceConfig = { + Type = "oneshot"; + StateDirectory = "mail-health"; + }; + script = '' + set -u + problems=() + problem() { problems+=("$*"); echo "PROBLEM: $*" >&2; } + + # 1. Services + for unit in postfix dovecot rspamd redis-rspamd nginx kresd@1 fail2ban; do + systemctl is-active --quiet "$unit" || problem "service $unit is not running" + done + failed=$(systemctl list-units --failed --plain --no-legend | awk '{print $1}' | tr '\n' ' ') + [[ -z "$failed" ]] || problem "failed systemd units: $failed" + + # 2. TLS certificate (Postfix/Dovecot); MTA-STS makes an expired one fatal + cert=/var/lib/acme/${ctx.fqdn}/cert.pem + if [[ ! -f $cert ]]; then + problem "no TLS certificate at $cert" + elif ! openssl x509 -checkend $((14 * 86400)) -noout -in "$cert" > /dev/null; then + problem "TLS certificate expires within 14 days: $(openssl x509 -enddate -noout -in "$cert")" + elif openssl x509 -noout -issuer -in "$cert" | grep -qi minica; then + problem "TLS certificate is the self-signed placeholder: Let's Encrypt issuance failed" + fi + + # 3. Disk + used=$(df --output=pcent / | tail -n1 | tr -dc 0-9) + (( used < ${toString cfg.diskWarnPercent} )) || problem "disk / is $used% full" + + # 4. Mail queue + queued=$(postqueue -j 2> /dev/null | wc -l) + (( queued < ${toString cfg.queueWarn} )) || problem "$queued messages in the mail queue" + + # 5. Outbound SMTP (blocked by the provider, or by OVH's anti-spam) + if ! timeout 10 bash -c 'exec 3<>/dev/tcp/gmail-smtp-in.l.google.com/25' 2> /dev/null; then + problem "cannot connect to gmail-smtp-in.l.google.com:25: outbound SMTP blocked?" + fi + + # 6. Blocklists, for the public IPv4 address (queried through the local + # resolver; 127.255.255.x means the list refused the query) + ip=$(curl -4 -fsS -m 10 https://api.ipify.org || true) + if [[ $ip =~ ^[0-9.]+$ ]]; then + rev=$(awk -F. '{print $4"."$3"."$2"."$1}' <<< "$ip") + for bl in zen.spamhaus.org bl.spamcop.net; do + answer=$(dig +short "$rev.$bl" A @127.0.0.1 | head -n1) + if [[ -n $answer && $answer != 127.255.255.* ]]; then + problem "$ip is listed on $bl ($answer)" + fi + done + fi + + # Report + if (( ''${#problems[@]} == 0 )); then + report="OK" + else + report=$(printf '%s\n' "''${problems[@]}") + fi + echo "$report" + ${ + if cfg.healthcheckUrl != "" then + '' + if (( ''${#problems[@]} == 0 )); then + curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}" > /dev/null + else + curl -fsS -m 10 --retry 3 --data-raw "$report" "${cfg.healthcheckUrl}/fail" > /dev/null + fi + '' + else + '' + if (( ''${#problems[@]} > 0 )) \ + && ! [[ $(find ${stateDir}/last-mail -mmin -1440 2> /dev/null) ]]; then + /run/wrappers/bin/sendmail -t <" + diskWarnPercent = 85; + queueWarn = 100; + }; + timeZone = "UTC"; # Where the server runs. Each provider has a Terraform step in diff --git a/tests/settings.nix b/tests/settings.nix index bfd5b14..013de14 100644 --- a/tests/settings.nix +++ b/tests/settings.nix @@ -41,6 +41,11 @@ import ../settings.nix username = "ci"; spfInclude = "spf.relay.test"; }; + monitoring = { + healthcheckUrl = "https://hc-ping.example.test/ci"; + diskWarnPercent = 85; + queueWarn = 100; + }; autoUpgrade = { enable = true; flake = "github:example/mailserver";