From 09df7b83d5e39c50ceea2aa89f22899f3ed4c1a1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Aug 2026 21:53:11 +0000 Subject: [PATCH] feat(observability): serve Grafana over TLS with the lab CA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grafana published plain HTTP on the management VLAN, so every login sent its password in cleartext to anything with a port on VLAN 99. It now serves the grafana.matrix.elysium leaf issued by the lab's own CA. Four things had to move together, and each one silently breaks the stack on its own: The cookie. GF_SECURITY_COOKIE_SECURE was false because a secure cookie over plain HTTP is never sent at all — the login page accepts the password and then loops. It flips to true in the same commit that makes TLS available, never before. The healthcheck. busybox wget cannot be handed a CA, and the probe is testing that Grafana is serving rather than that a self-signed chain builds, so it uses --no-check-certificate against its own loopback. The scrape. Prometheus reaches Grafana as `grafana:3000`, not by FQDN, so the leaf carries `grafana` as an additional SAN and the job verifies properly with ca_file and server_name rather than insecure_skip_verify. Verification that is switched off is not verification. The key permissions. Grafana runs as 472:0 and cannot read a key owned by the operator at 0600. The container is given the operator's gid as a supplementary group and leaf keys are now 0640 — certificates/ is 0700, so nothing else on the host can traverse to them regardless. The alternatives were a world-readable private key, or running Grafana as the operator's uid and stranding it from its own 472-owned data volume. gen-certs.sh gains a repeatable --dns for exactly this: a service reached under more than one name needs every one of them, or verification fails for whichever is missing. Verified end to end: curl against the CA with no -k, all ten scrape targets up, Prometheus scraping https://grafana:3000/metrics with no error, login and both working datasources fine over TLS, 5 dashboards still provisioned. --- Makefile | 3 ++- docs/images/README.md | 2 +- docs/runbooks/deploy-stack.md | 2 +- scripts/gen-certs.sh | 18 +++++++++++-- stacks/observability/compose.yaml | 26 +++++++++++++++++-- .../observability/prometheus/prometheus.yaml | 8 ++++++ 6 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 16d3951..f3834a2 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,8 @@ up: render ## Render config and start the stack @# format, so a stray % in the value cannot be read as a format spec. @# tail -1 because compose takes the last of duplicate keys. @port="$$(grep -E '^GRAFANA_PORT=' $(STACK_DIR)/.env 2>/dev/null | tail -1 | cut -d= -f2-)"; \ - printf '\n\033[0;32mup\033[0m — Grafana: http://localhost:%s\n' "$${port:-3000}" + printf '\n\033[0;32mup\033[0m — Grafana: https://localhost:%s\n' "$${port:-3000}" + @printf ' (self-signed by the lab CA — trust certificates/ca.pem, see docs/runbooks/generate-certificates.md)\n' .PHONY: down down: ## Stop the stack (volumes are preserved) diff --git a/docs/images/README.md b/docs/images/README.md index 2106e90..0ea42c8 100644 --- a/docs/images/README.md +++ b/docs/images/README.md @@ -12,7 +12,7 @@ Once the stack has a few days of real data: ```bash make up -# open http://:3000, log in, HomeLab folder +# open https://:3000, log in, HomeLab folder ``` For each dashboard, set the time range to something with visible activity diff --git a/docs/runbooks/deploy-stack.md b/docs/runbooks/deploy-stack.md index 55ca61b..a8829eb 100644 --- a/docs/runbooks/deploy-stack.md +++ b/docs/runbooks/deploy-stack.md @@ -36,7 +36,7 @@ make ps # all six services healthy curl -s localhost:9090/-/healthy # Prometheus curl -s localhost:3100/ready # Loki curl -s localhost:9093/-/healthy # Alertmanager -curl -s localhost:3000/api/health # Grafana +curl -sk https://localhost:3000/api/health # Grafana (-k: lab CA) ``` Then in the UI: diff --git a/scripts/gen-certs.sh b/scripts/gen-certs.sh index 79d38b8..1f78ba0 100755 --- a/scripts/gen-certs.sh +++ b/scripts/gen-certs.sh @@ -15,11 +15,15 @@ # # Usage: # scripts/gen-certs.sh --ca create the CA (refuses if it exists) -# scripts/gen-certs.sh --host [--ip IP] issue a leaf signed by the CA +# scripts/gen-certs.sh --host [--ip IP] [--dns NAME] # scripts/gen-certs.sh --list show what exists, and when it expires # # --ip add an IP SAN; repeatable. Internal DNS is unresolved in # this lab (docs/roadmap.md), so most leaves want one. +# --dns add an extra DNS SAN; repeatable. A service reached under +# more than one name — its FQDN from a browser and its compose +# service name from inside the network — needs every one of +# them, or verification fails for the names that are missing. # --days leaf lifetime, default 825 # --force overwrite an existing CA or leaf # @@ -50,6 +54,7 @@ head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } MODE="" HOST="" IPS=() +DNS=() FORCE=0 while (($#)); do @@ -58,6 +63,7 @@ while (($#)); do --list) MODE="list"; shift ;; --host) MODE="leaf"; HOST="${2:?--host needs an FQDN}"; shift 2 ;; --ip) IPS+=("${2:?--ip needs an address}"); shift 2 ;; + --dns) DNS+=("${2:?--dns needs a name}"); shift 2 ;; --days) LEAF_DAYS="${2:?--days needs a number}"; shift 2 ;; --force) FORCE=1; shift ;; -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; @@ -143,6 +149,9 @@ fi # failure — "x509: certificate relies on legacy Common Name field" — reads as a # trust problem rather than a missing field, so it costs an hour to diagnose. SAN="DNS:${HOST}" +for name in "${DNS[@]}"; do + SAN+=",DNS:${name}" +done for ip in "${IPS[@]}"; do SAN+=",IP:${ip}" done @@ -169,7 +178,12 @@ openssl x509 -req -in "${TMP}/csr.pem" -sha256 \ -out "${CRT}" -days "${LEAF_DAYS}" \ -extfile <(printf 'subjectAltName=%s\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n' "${SAN}") 2>/dev/null -chmod 600 "${KEY}" +# 0640, not 0600. A container serving this key runs as its own uid — Grafana is +# 472:0 — and cannot read a file owned by the operator at 0600. The compose +# service is given the operator's gid as a supplementary group instead of the +# key being made world-readable. This costs nothing: certificates/ is 0700, so +# no other local user can traverse to the file whatever its own mode says. +chmod 640 "${KEY}" chmod 644 "${CRT}" # Prove it verifies against the CA now, rather than discovering at deploy time diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 485e51e..4dd3b28 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -40,6 +40,9 @@ services: volumes: - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - ./prometheus/rules:/etc/prometheus/rules:ro + # Only the CA certificate, never a key. Prometheus needs it to verify + # Grafana's TLS now that the grafana job scrapes over https. + - ../../certificates/ca.pem:/etc/prometheus/tls/ca.pem:ro - ./prometheus/targets:/etc/prometheus/targets:ro - prometheus-data:/prometheus ports: @@ -165,7 +168,13 @@ services: environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/observability.sops.yaml} - GF_SECURITY_COOKIE_SECURE: "false" + # Now true: the cookie is only ever sent over TLS. It was false because + # Grafana spoke plain HTTP, and a secure cookie over HTTP is simply never + # sent — you get a login page that accepts your password and loops. + GF_SECURITY_COOKIE_SECURE: "true" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /etc/grafana/tls/cert.pem + GF_SERVER_CERT_KEY: /etc/grafana/tls/key.pem GF_SECURITY_DISABLE_GRAVATAR: "true" GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" @@ -179,6 +188,16 @@ services: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana-data:/var/lib/grafana + # Generated by scripts/gen-certs.sh into a gitignored certificates/ at the + # repository root, which is why this reaches outside the stack directory. + - ../../certificates/grafana.matrix.elysium.pem:/etc/grafana/tls/cert.pem:ro + - ../../certificates/grafana.matrix.elysium-key.pem:/etc/grafana/tls/key.pem:ro + # Grafana runs as 472:0 and the key is owned by the operator at 0640, so the + # container is given the operator's gid as a supplementary group. The + # alternative — a world-readable private key — is worse, and running Grafana + # as the operator's uid would strand it from its own 472-owned data volume. + group_add: + - "${RENDER_GID:?run make render}" ports: - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" depends_on: @@ -191,7 +210,10 @@ services: loki: condition: service_started healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] + # --no-check-certificate: the probe runs inside the container and talks to + # itself. It is testing that Grafana is serving, not that a self-signed + # certificate chains — and busybox wget has no way to be handed a CA. + test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "https://localhost:3000/api/health"] interval: 30s timeout: 5s retries: 3 diff --git a/stacks/observability/prometheus/prometheus.yaml b/stacks/observability/prometheus/prometheus.yaml index 860edca..c000de3 100644 --- a/stacks/observability/prometheus/prometheus.yaml +++ b/stacks/observability/prometheus/prometheus.yaml @@ -32,7 +32,15 @@ scrape_configs: static_configs: - targets: ["loki:3100"] + # Grafana serves TLS with a certificate from the lab's own CA. Verified + # properly rather than with insecure_skip_verify: the CA is mounted in, and + # the leaf carries `grafana` as a SAN alongside its FQDN precisely so this + # scrape can check the name it actually connects to. - job_name: grafana + scheme: https + tls_config: + ca_file: /etc/prometheus/tls/ca.pem + server_name: grafana static_configs: - targets: ["grafana:3000"]