diff --git a/Makefile b/Makefile index 16d3951..f3834a2 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,8 @@ up: render ## Render config and start the stack @# format, so a stray % in the value cannot be read as a format spec. @# tail -1 because compose takes the last of duplicate keys. @port="$$(grep -E '^GRAFANA_PORT=' $(STACK_DIR)/.env 2>/dev/null | tail -1 | cut -d= -f2-)"; \ - printf '\n\033[0;32mup\033[0m — Grafana: http://localhost:%s\n' "$${port:-3000}" + printf '\n\033[0;32mup\033[0m — Grafana: https://localhost:%s\n' "$${port:-3000}" + @printf ' (self-signed by the lab CA — trust certificates/ca.pem, see docs/runbooks/generate-certificates.md)\n' .PHONY: down down: ## Stop the stack (volumes are preserved) diff --git a/docs/images/README.md b/docs/images/README.md index 2106e90..0ea42c8 100644 --- a/docs/images/README.md +++ b/docs/images/README.md @@ -12,7 +12,7 @@ Once the stack has a few days of real data: ```bash make up -# open http://:3000, log in, HomeLab folder +# open https://:3000, log in, HomeLab folder ``` For each dashboard, set the time range to something with visible activity diff --git a/docs/runbooks/deploy-stack.md b/docs/runbooks/deploy-stack.md index 55ca61b..a8829eb 100644 --- a/docs/runbooks/deploy-stack.md +++ b/docs/runbooks/deploy-stack.md @@ -36,7 +36,7 @@ make ps # all six services healthy curl -s localhost:9090/-/healthy # Prometheus curl -s localhost:3100/ready # Loki curl -s localhost:9093/-/healthy # Alertmanager -curl -s localhost:3000/api/health # Grafana +curl -sk https://localhost:3000/api/health # Grafana (-k: lab CA) ``` Then in the UI: diff --git a/scripts/gen-certs.sh b/scripts/gen-certs.sh index 79d38b8..1f78ba0 100755 --- a/scripts/gen-certs.sh +++ b/scripts/gen-certs.sh @@ -15,11 +15,15 @@ # # Usage: # scripts/gen-certs.sh --ca create the CA (refuses if it exists) -# scripts/gen-certs.sh --host [--ip IP] issue a leaf signed by the CA +# scripts/gen-certs.sh --host [--ip IP] [--dns NAME] # scripts/gen-certs.sh --list show what exists, and when it expires # # --ip add an IP SAN; repeatable. Internal DNS is unresolved in # this lab (docs/roadmap.md), so most leaves want one. +# --dns add an extra DNS SAN; repeatable. A service reached under +# more than one name — its FQDN from a browser and its compose +# service name from inside the network — needs every one of +# them, or verification fails for the names that are missing. # --days leaf lifetime, default 825 # --force overwrite an existing CA or leaf # @@ -50,6 +54,7 @@ head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } MODE="" HOST="" IPS=() +DNS=() FORCE=0 while (($#)); do @@ -58,6 +63,7 @@ while (($#)); do --list) MODE="list"; shift ;; --host) MODE="leaf"; HOST="${2:?--host needs an FQDN}"; shift 2 ;; --ip) IPS+=("${2:?--ip needs an address}"); shift 2 ;; + --dns) DNS+=("${2:?--dns needs a name}"); shift 2 ;; --days) LEAF_DAYS="${2:?--days needs a number}"; shift 2 ;; --force) FORCE=1; shift ;; -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; @@ -143,6 +149,9 @@ fi # failure — "x509: certificate relies on legacy Common Name field" — reads as a # trust problem rather than a missing field, so it costs an hour to diagnose. SAN="DNS:${HOST}" +for name in "${DNS[@]}"; do + SAN+=",DNS:${name}" +done for ip in "${IPS[@]}"; do SAN+=",IP:${ip}" done @@ -169,7 +178,12 @@ openssl x509 -req -in "${TMP}/csr.pem" -sha256 \ -out "${CRT}" -days "${LEAF_DAYS}" \ -extfile <(printf 'subjectAltName=%s\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n' "${SAN}") 2>/dev/null -chmod 600 "${KEY}" +# 0640, not 0600. A container serving this key runs as its own uid — Grafana is +# 472:0 — and cannot read a file owned by the operator at 0600. The compose +# service is given the operator's gid as a supplementary group instead of the +# key being made world-readable. This costs nothing: certificates/ is 0700, so +# no other local user can traverse to the file whatever its own mode says. +chmod 640 "${KEY}" chmod 644 "${CRT}" # Prove it verifies against the CA now, rather than discovering at deploy time diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 485e51e..4dd3b28 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -40,6 +40,9 @@ services: volumes: - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - ./prometheus/rules:/etc/prometheus/rules:ro + # Only the CA certificate, never a key. Prometheus needs it to verify + # Grafana's TLS now that the grafana job scrapes over https. + - ../../certificates/ca.pem:/etc/prometheus/tls/ca.pem:ro - ./prometheus/targets:/etc/prometheus/targets:ro - prometheus-data:/prometheus ports: @@ -165,7 +168,13 @@ services: environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/observability.sops.yaml} - GF_SECURITY_COOKIE_SECURE: "false" + # Now true: the cookie is only ever sent over TLS. It was false because + # Grafana spoke plain HTTP, and a secure cookie over HTTP is simply never + # sent — you get a login page that accepts your password and loops. + GF_SECURITY_COOKIE_SECURE: "true" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /etc/grafana/tls/cert.pem + GF_SERVER_CERT_KEY: /etc/grafana/tls/key.pem GF_SECURITY_DISABLE_GRAVATAR: "true" GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" @@ -179,6 +188,16 @@ services: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana-data:/var/lib/grafana + # Generated by scripts/gen-certs.sh into a gitignored certificates/ at the + # repository root, which is why this reaches outside the stack directory. + - ../../certificates/grafana.matrix.elysium.pem:/etc/grafana/tls/cert.pem:ro + - ../../certificates/grafana.matrix.elysium-key.pem:/etc/grafana/tls/key.pem:ro + # Grafana runs as 472:0 and the key is owned by the operator at 0640, so the + # container is given the operator's gid as a supplementary group. The + # alternative — a world-readable private key — is worse, and running Grafana + # as the operator's uid would strand it from its own 472-owned data volume. + group_add: + - "${RENDER_GID:?run make render}" ports: - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" depends_on: @@ -191,7 +210,10 @@ services: loki: condition: service_started healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] + # --no-check-certificate: the probe runs inside the container and talks to + # itself. It is testing that Grafana is serving, not that a self-signed + # certificate chains — and busybox wget has no way to be handed a CA. + test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "https://localhost:3000/api/health"] interval: 30s timeout: 5s retries: 3 diff --git a/stacks/observability/prometheus/prometheus.yaml b/stacks/observability/prometheus/prometheus.yaml index 860edca..c000de3 100644 --- a/stacks/observability/prometheus/prometheus.yaml +++ b/stacks/observability/prometheus/prometheus.yaml @@ -32,7 +32,15 @@ scrape_configs: static_configs: - targets: ["loki:3100"] + # Grafana serves TLS with a certificate from the lab's own CA. Verified + # properly rather than with insecure_skip_verify: the CA is mounted in, and + # the leaf carries `grafana` as a SAN alongside its FQDN precisely so this + # scrape can check the name it actually connects to. - job_name: grafana + scheme: https + tls_config: + ca_file: /etc/prometheus/tls/ca.pem + server_name: grafana static_configs: - targets: ["grafana:3000"]