feat(observability): serve Grafana over TLS with the lab CA

Makefile

@@ -36,7 +36,8 @@ up: render ## Render config and start the stack
 	@# format, so a stray % in the value cannot be read as a format spec.
 	@# tail -1 because compose takes the last of duplicate keys.
 	@port="$$(grep -E '^GRAFANA_PORT=' $(STACK_DIR)/.env 2>/dev/null | tail -1 | cut -d= -f2-)"; \
-	printf '\n\033[0;32mup\033[0m — Grafana: http://localhost:%s\n' "$${port:-3000}"
+	printf '\n\033[0;32mup\033[0m — Grafana: https://localhost:%s\n' "$${port:-3000}"
+	@printf '   (self-signed by the lab CA — trust certificates/ca.pem, see docs/runbooks/generate-certificates.md)\n'
 
 .PHONY: down
 down: ## Stop the stack (volumes are preserved)

docs/images/README.md

@@ -12,7 +12,7 @@ Once the stack has a few days of real data:
 
 ```bash
 make up
-# open http://<monitoring-host>:3000, log in, HomeLab folder
+# open https://<monitoring-host>:3000, log in, HomeLab folder
 ```
 
 For each dashboard, set the time range to something with visible activity

docs/runbooks/deploy-stack.md

@@ -36,7 +36,7 @@ make ps                                     # all six services healthy
 curl -s localhost:9090/-/healthy            # Prometheus
 curl -s localhost:3100/ready                # Loki
 curl -s localhost:9093/-/healthy            # Alertmanager
-curl -s localhost:3000/api/health           # Grafana
+curl -sk https://localhost:3000/api/health  # Grafana (-k: lab CA)
 ```
 
 Then in the UI:

scripts/gen-certs.sh

@@ -15,11 +15,15 @@
 #
 # Usage:
 #   scripts/gen-certs.sh --ca                     create the CA (refuses if it exists)
-#   scripts/gen-certs.sh --host <fqdn> [--ip IP]  issue a leaf signed by the CA
+#   scripts/gen-certs.sh --host <fqdn> [--ip IP] [--dns NAME]
 #   scripts/gen-certs.sh --list                   show what exists, and when it expires
 #
 #   --ip <addr>     add an IP SAN; repeatable. Internal DNS is unresolved in
 #                   this lab (docs/roadmap.md), so most leaves want one.
+#   --dns <name>    add an extra DNS SAN; repeatable. A service reached under
+#                   more than one name — its FQDN from a browser and its compose
+#                   service name from inside the network — needs every one of
+#                   them, or verification fails for the names that are missing.
 #   --days <n>      leaf lifetime, default 825
 #   --force         overwrite an existing CA or leaf
 #
@@ -50,6 +54,7 @@ head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }
 MODE=""
 HOST=""
 IPS=()
+DNS=()
 FORCE=0
 
 while (($#)); do
@@ -58,6 +63,7 @@ while (($#)); do
     --list)  MODE="list"; shift ;;
     --host)  MODE="leaf"; HOST="${2:?--host needs an FQDN}"; shift 2 ;;
     --ip)    IPS+=("${2:?--ip needs an address}"); shift 2 ;;
+    --dns)   DNS+=("${2:?--dns needs a name}"); shift 2 ;;
     --days)  LEAF_DAYS="${2:?--days needs a number}"; shift 2 ;;
     --force) FORCE=1; shift ;;
     -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
@@ -143,6 +149,9 @@ fi
 # failure — "x509: certificate relies on legacy Common Name field" — reads as a
 # trust problem rather than a missing field, so it costs an hour to diagnose.
 SAN="DNS:${HOST}"
+for name in "${DNS[@]}"; do
+  SAN+=",DNS:${name}"
+done
 for ip in "${IPS[@]}"; do
   SAN+=",IP:${ip}"
 done
@@ -169,7 +178,12 @@ openssl x509 -req -in "${TMP}/csr.pem" -sha256 \
   -out "${CRT}" -days "${LEAF_DAYS}" \
   -extfile <(printf 'subjectAltName=%s\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n' "${SAN}") 2>/dev/null
 
-chmod 600 "${KEY}"
+# 0640, not 0600. A container serving this key runs as its own uid — Grafana is
+# 472:0 — and cannot read a file owned by the operator at 0600. The compose
+# service is given the operator's gid as a supplementary group instead of the
+# key being made world-readable. This costs nothing: certificates/ is 0700, so
+# no other local user can traverse to the file whatever its own mode says.
+chmod 640 "${KEY}"
 chmod 644 "${CRT}"
 
 # Prove it verifies against the CA now, rather than discovering at deploy time

stacks/observability/compose.yaml

@@ -40,6 +40,9 @@ services:
     volumes:
       - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
       - ./prometheus/rules:/etc/prometheus/rules:ro
+      # Only the CA certificate, never a key. Prometheus needs it to verify
+      # Grafana's TLS now that the grafana job scrapes over https.
+      - ../../certificates/ca.pem:/etc/prometheus/tls/ca.pem:ro
       - ./prometheus/targets:/etc/prometheus/targets:ro
       - prometheus-data:/prometheus
     ports:
@@ -165,7 +168,13 @@ services:
     environment:
       GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/observability.sops.yaml}
-      GF_SECURITY_COOKIE_SECURE: "false"
+      # Now true: the cookie is only ever sent over TLS. It was false because
+      # Grafana spoke plain HTTP, and a secure cookie over HTTP is simply never
+      # sent — you get a login page that accepts your password and loops.
+      GF_SECURITY_COOKIE_SECURE: "true"
+      GF_SERVER_PROTOCOL: https
+      GF_SERVER_CERT_FILE: /etc/grafana/tls/cert.pem
+      GF_SERVER_CERT_KEY: /etc/grafana/tls/key.pem
       GF_SECURITY_DISABLE_GRAVATAR: "true"
       GF_USERS_ALLOW_SIGN_UP: "false"
       GF_AUTH_ANONYMOUS_ENABLED: "false"
@@ -179,6 +188,16 @@ services:
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
       - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
       - grafana-data:/var/lib/grafana
+      # Generated by scripts/gen-certs.sh into a gitignored certificates/ at the
+      # repository root, which is why this reaches outside the stack directory.
+      - ../../certificates/grafana.matrix.elysium.pem:/etc/grafana/tls/cert.pem:ro
+      - ../../certificates/grafana.matrix.elysium-key.pem:/etc/grafana/tls/key.pem:ro
+    # Grafana runs as 472:0 and the key is owned by the operator at 0640, so the
+    # container is given the operator's gid as a supplementary group. The
+    # alternative — a world-readable private key — is worse, and running Grafana
+    # as the operator's uid would strand it from its own 472-owned data volume.
+    group_add:
+      - "${RENDER_GID:?run make render}"
     ports:
       - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000"
     depends_on:
@@ -191,7 +210,10 @@ services:
       loki:
         condition: service_started
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
+      # --no-check-certificate: the probe runs inside the container and talks to
+      # itself. It is testing that Grafana is serving, not that a self-signed
+      # certificate chains — and busybox wget has no way to be handed a CA.
+      test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "https://localhost:3000/api/health"]
       interval: 30s
       timeout: 5s
       retries: 3

stacks/observability/prometheus/prometheus.yaml

@@ -32,7 +32,15 @@ scrape_configs:
     static_configs:
       - targets: ["loki:3100"]
 
+  # Grafana serves TLS with a certificate from the lab's own CA. Verified
+  # properly rather than with insecure_skip_verify: the CA is mounted in, and
+  # the leaf carries `grafana` as a SAN alongside its FQDN precisely so this
+  # scrape can check the name it actually connects to.
   - job_name: grafana
+    scheme: https
+    tls_config:
+      ca_file: /etc/prometheus/tls/ca.pem
+      server_name: grafana
     static_configs:
       - targets: ["grafana:3000"]