diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd3f6aa..25ffdef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,7 +206,7 @@ jobs: - name: Assert no decrypted artefact is tracked run: | fail=0 - for pattern in '.env' '.rendered/' '.purge-secrets.txt'; do + for pattern in '.env' '.rendered/' '.purge-secrets.txt' 'certificates/'; do if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then echo "::error::tracked file matching '${pattern}' — it must be gitignored" fail=1 diff --git a/.gitleaks.toml b/.gitleaks.toml index 52bc4be..06b180b 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -48,6 +48,20 @@ paths = [ '''secrets/.*\.sops\.yaml$''', # Documents required key names and deliberately carries change-me values. '''secrets/.*\.example\.yaml$''', + # The next two hold real secrets on purpose, and are gitignored. They exist + # only on a host that has actually deployed, so CI — which scans a fresh + # checkout — never sees them; the filesystem scan on the monitoring host does, + # and reported eight findings that are all working as designed. + # + # Allowlisting them is not a weakening. gitleaks was never the control here: + # the control is that neither may ever become a tracked file, which CI and + # scripts/validate.sh both assert directly. A `make validate` that is + # permanently red for a known reason is a `make validate` nobody reads. + # + # Decrypted at deploy time by scripts/render-config.sh. + '''\.rendered/''', + # The lab's own CA and leaf keys, created by scripts/gen-certs.sh. + '''certificates/''', ] regexes = [ diff --git a/Makefile b/Makefile index 6596270..16d3951 100644 --- a/Makefile +++ b/Makefile @@ -218,6 +218,13 @@ snmp-verify: ## Check each SNMP device answers to its community (ARGS=--old) @# it into `make validate`. ./scripts/snmp-verify.sh $(ARGS) +.PHONY: certs +certs: ## Create the internal CA / issue a leaf (ARGS="--host x.matrix.elysium --ip 10.0.0.1") + @# certificates/ is gitignored. Nothing in the stack terminates TLS yet — + @# see docs/roadmap.md — so this exists so that the CA is created + @# deliberately rather than improvised the day something needs it. + ./scripts/gen-certs.sh $(ARGS) + .PHONY: gen-secret gen-secret: ## Generate a random secret (ARGS=--snmp for one per SNMP device) ./scripts/gen-secret.sh $(ARGS) diff --git a/README.md b/README.md index ab8212d..bec58d9 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ incident. `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it; `make pin-digests` re-resolves them from the registry. - **Documented decisions and runbooks.** Five ADRs covering what was chosen and - what was rejected; four runbooks for the operations that are easy to get wrong + what was rejected; five runbooks for the operations that are easy to get wrong at 1am. ## Architecture @@ -137,7 +137,7 @@ the internet and nothing more. Full topology and data flow in │ ├── architecture.md network.md hardware.md │ ├── observability.md security.md roadmap.md │ ├── adr/ # 5 architecture decision records -│ └── runbooks/ # deploy, add device, rotate creds, purge history +│ └── runbooks/ # deploy, add device, rotate creds, certs, purge history └── Makefile # make help ``` diff --git a/SECURITY.md b/SECURITY.md index 4a84969..0c87360 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,7 +40,7 @@ is a very different thing from an overlooked one. Full detail in | --- | --- | | SNMP community committed in plaintext, shared across firewall, switch, UPS and BMC | Removed from `HEAD` and replaced with four distinct per-device SOPS-encrypted values. Rotated on all four devices; each answers to its own new community. The firewall, the UPS and the BMC additionally refuse the old one. **The switch still accepts its previous community alongside the new one** — see below. The original shared string has been purged from git history, though it must still be treated as public — it was reachable in a public repository and cannot be un-seen. | | Grafana `admin`/`admin` with anonymous Admin access enabled | Fixed — anonymous auth off, password from SOPS | -| Passphrase-encrypted TLS private keys under `certificates/` | Removed from `HEAD` and purged from history. The CA and leaf certificates still need regenerating — assume the old keys are compromised. | +| Passphrase-encrypted TLS private keys under `certificates/` | Removed from `HEAD` and purged from history. A new CA and leaf have been generated with [`scripts/gen-certs.sh`](scripts/gen-certs.sh); the old keys are superseded and should be treated as compromised wherever they were ever trusted. | The switch is the honest gap, and it is a deliberate one. `neo` (10.7.7.2) is rotated and polling, but it also still accepts the community it held before the diff --git a/docs/roadmap.md b/docs/roadmap.md index f6c9d98..1f361be 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -13,14 +13,12 @@ inventory. Ordered roughly by how much it matters. and reachable only from the management VLAN. Overwrite the row with a fresh value rather than deleting it, next time the switch is down anyway. → [runbook](runbooks/rotate-snmp-community.md) -- [ ] **Regenerate the CA and leaf certificates.** The old keys were purged - from git history, but they were public before that and must be treated as - compromised. - → [runbook](runbooks/purge-git-history.md) - [ ] Move to SNMPv3 authPriv where the hardware supports it. pfSense, the APC and iLO all do; the MokerLink switch does not, which is the blocker for doing it uniformly. - [ ] Put Grafana behind TLS rather than plain HTTP on the management VLAN. + The CA and a `grafana.matrix.elysium` leaf already exist — see + [runbook](runbooks/generate-certificates.md) — so this is wiring, not PKI. - [ ] Decide whether the lab VLAN needs egress filtering before the deliberately-vulnerable playground exists. @@ -77,6 +75,8 @@ inventory. Ordered roughly by how much it matters. - [x] Add SECURITY.md with a disclosure policy and known-exposure summary - [x] Loki alerting rules (8) for auth, SSH brute force and disk/OOM events, validated in CI by booting the pinned Loki image against them +- [x] Replace the CA and leaf certificates that leaked, and add tooling so + issuing one is a command rather than a research project - [x] Purge the shared SNMP community, the inline Grafana password and the TLS private keys under `certificates/` from git history, and delete the `.gitleaksignore` that acknowledged them diff --git a/docs/runbooks/generate-certificates.md b/docs/runbooks/generate-certificates.md new file mode 100644 index 0000000..6972c43 --- /dev/null +++ b/docs/runbooks/generate-certificates.md @@ -0,0 +1,128 @@ +# Runbook: Generate the internal CA and leaf certificates + +**Target:** the lab's own PKI, under `certificates/` +**Time:** a couple of minutes +**You will need:** nothing but `openssl` and this repository + +The previous CA and leaf keys were committed to this public repository and had +to be removed by rewriting every commit — see +[`purge-git-history.md`](purge-git-history.md). Treat anything issued before +that rewrite as compromised, including the CA. What follows replaces it. + +> **Nothing in the stack terminates TLS yet.** Prometheus, Alertmanager, Loki +> and Grafana all publish plain HTTP on the management VLAN. Putting Grafana +> behind TLS is tracked separately in [`roadmap.md`](../roadmap.md), and it is +> the first thing that will consume a certificate from here. The CA exists now +> so that it is created deliberately, once, rather than improvised on the day +> something needs it. + +## Where things live + +Everything lands in `certificates/`, which is gitignored — as are `*.pem` and +`*.key`. That is the control that matters. File modes are useful; not being a +tracked file is what stops a repeat of the last leak, and both `make validate` +and CI assert it. + +| File | Mode | Secret? | +| --- | --- | --- | +| `certificates/ca-key.pem` | 0600 | **yes** — never copy it off this host | +| `certificates/ca.pem` | 0644 | no — this is what clients trust, distribute freely | +| `certificates/-key.pem` | 0600 | **yes** — belongs only to that service | +| `certificates/.pem` | 0644 | no | + +## 1. Create the CA + +Once, ever. Skip if `certificates/ca.pem` already exists. + +```bash +make certs ARGS=--ca +``` + +Valid for ten years, `CN=Matrix Elysium Internal CA`. It refuses to overwrite an +existing CA: regenerating it invalidates every leaf it has signed and every +trust store holding it, so that has to be deliberate (`--force`). + +## 2. Issue a leaf + +```bash +make certs ARGS="--host grafana.matrix.elysium --ip 10.0.99.20" +``` + +Include `--ip` for anything reached by address. `docs/roadmap.md` still lists +internal DNS as unresolved, so in practice most services here are reached by IP, +and a certificate without a matching IP SAN will be rejected at the point you +most want it to work. + +Two limits the script enforces rather than lets you discover later: + +- **A certificate with no `subjectAltName` is rejected outright** by every + current browser and by Go's `crypto/tls` — which is what Prometheus, + Alertmanager and Grafana are built on. The error, `x509: certificate relies on + legacy Common Name field`, reads like a trust problem rather than a missing + field. +- **Leaf lifetimes above 825 days are rejected by browsers.** A ten-year leaf + looks like less future work and produces something nothing will trust. + +Every issued certificate is verified against the CA before the script reports +success, so a chain that does not build fails here rather than at deploy time. + +## 3. Check what exists + +```bash +make certs ARGS=--list +``` + +Shows each certificate, its subject and its expiry, and marks expired ones. Worth +running before you debug a TLS error — an expired leaf and a misconfigured one +look identical from the client side. + +## 4. Trust the CA where you need it + +Distribute `certificates/ca.pem` — never the key. + +```bash +# Debian/Ubuntu +sudo cp certificates/ca.pem /usr/local/share/ca-certificates/matrix-elysium.crt +sudo update-ca-certificates +``` + +Firefox keeps its own store and will not read the system one; import it under +**Settings → Privacy & Security → Certificates → View Certificates → Authorities**. + +## Renewal + +Leaves expire in 825 days. There is no automation and deliberately no cron: a +lab with one certificate is better served by a calendar reminder than by a +renewal daemon nobody maintains. + +```bash +make certs ARGS="--host grafana.matrix.elysium --ip 10.0.99.20 --force" +``` + +Then restart whatever serves it. The CA does not change, so nothing needs +re-trusting. + +Adding blackbox-exporter for TLS-expiry checks is on +[`roadmap.md`](../roadmap.md); until that exists, expiry is something you find +out about from a browser warning. + +## If something goes wrong + +| Symptom | Cause | Fix | +| --- | --- | --- | +| `x509: certificate relies on legacy Common Name field` | No SAN | Reissue; the script always sets one, so this is an older certificate | +| `x509: certificate is valid for , not ` | Reached by a name or IP not in the SANs | Reissue with the right `--host`/`--ip` | +| `x509: certificate signed by unknown authority` | The client does not trust the CA | Install `ca.pem` — step 4 | +| Browser rejects a certificate that `openssl verify` accepts | Leaf lifetime over 825 days, or an old cert | Reissue with the default lifetime | +| `no CA yet` | No `certificates/ca.pem` | Step 1 | +| `already exists` | Guard against clobbering a CA or leaf | `--force`, once you are sure | + +## Also worth knowing + +The CA key is **not** passphrase-protected, which is a deliberate trade. A +passphrase is what made the previous leak survivable, but it also makes every +issuance interactive — which is how a lab ends up with one ancient certificate +nobody dares reissue. The exposure is bounded instead by the key never leaving +this host and never becoming a tracked file. To change that, add `-aes256` to +the CA key generation in [`gen-certs.sh`](../../scripts/gen-certs.sh) and accept +the prompt on every issue. diff --git a/docs/security.md b/docs/security.md index 462bf91..c3e22d4 100644 --- a/docs/security.md +++ b/docs/security.md @@ -61,7 +61,7 @@ repository must be treated as compromised: | --- | --- | --- | | SNMP community shared across all four devices | `snmp.yaml`, from commit `ee3d443` (now rewritten) | Purged from history. Replaced with four distinct per-device values, SOPS-encrypted. Rotated on all four. `morpheus`, `mjolnir` and `shiva` verified answering the new community and refusing the old; `neo` answers the new one but still accepts its previous community — accepted risk, see [`SECURITY.md`](../SECURITY.md) and the [runbook](runbooks/rotate-snmp-community.md) | | Grafana `admin` / `admin` with anonymous Admin access | compose file | Fixed: password from SOPS, anonymous auth disabled | -| Passphrase-encrypted TLS private keys | `certificates/`, added in `efb2632`, deleted in `647d90a` | Purged from history. **Still to regenerate** the CA and leaf certificates — see [runbook](runbooks/purge-git-history.md) | +| Passphrase-encrypted TLS private keys | `certificates/`, added in `efb2632`, deleted in `647d90a` | Purged from history, and the CA replaced — see [runbook](runbooks/generate-certificates.md). Anything that trusted the old CA must be re-pointed at the new one | CI scans both the working tree and the full history, with no ignore file. Both must be clean unconditionally. diff --git a/scripts/gen-certs.sh b/scripts/gen-certs.sh new file mode 100755 index 0000000..79d38b8 --- /dev/null +++ b/scripts/gen-certs.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# +# Generate the lab's internal CA and the leaf certificates it signs. +# +# Everything lands in certificates/, which is gitignored — along with *.pem and +# *.key — because the previous set of these was committed to a public repository +# and had to be purged from history. See docs/runbooks/purge-git-history.md. +# The control that matters is not the file mode, it is that these never become +# tracked files. `make validate` asserts that; so does CI. +# +# Nothing in this repository terminates TLS yet. This exists so that when +# something does — Grafana is the first candidate, see docs/roadmap.md — issuing +# a certificate is one command rather than a research project, and so the CA is +# created deliberately rather than improvised at the point of need. +# +# Usage: +# scripts/gen-certs.sh --ca create the CA (refuses if it exists) +# scripts/gen-certs.sh --host [--ip IP] issue a leaf signed by the CA +# scripts/gen-certs.sh --list show what exists, and when it expires +# +# --ip add an IP SAN; repeatable. Internal DNS is unresolved in +# this lab (docs/roadmap.md), so most leaves want one. +# --days leaf lifetime, default 825 +# --force overwrite an existing CA or leaf +# +# The CA key is not passphrase-protected. That is a deliberate trade and worth +# stating: a passphrase is what made the previous leak survivable, but it also +# means every issuance is interactive, which is how a lab ends up with one +# long-lived certificate nobody dares reissue. The exposure this time is bounded +# by the key never leaving this host and never being committed. If you would +# rather have the passphrase, add -aes256 to the CA key generation below and +# accept the prompt on every issue. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CERT_DIR="${REPO_ROOT}/certificates" +CA_KEY="${CERT_DIR}/ca-key.pem" +CA_CRT="${CERT_DIR}/ca.pem" + +CA_SUBJECT='/CN=Matrix Elysium Internal CA/O=matrix.elysium' +CA_DAYS=3650 +LEAF_DAYS=825 + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } +ok() { printf '\033[0;32mok\033[0m — %s\n' "$*"; } +head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +MODE="" +HOST="" +IPS=() +FORCE=0 + +while (($#)); do + case "$1" in + --ca) MODE="ca"; shift ;; + --list) MODE="list"; shift ;; + --host) MODE="leaf"; HOST="${2:?--host needs an FQDN}"; shift 2 ;; + --ip) IPS+=("${2:?--ip needs an address}"); shift 2 ;; + --days) LEAF_DAYS="${2:?--days needs a number}"; shift 2 ;; + --force) FORCE=1; shift ;; + -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[[ -n "${MODE}" ]] || die "nothing to do — try --ca, --host , or --list (see --help)" +command -v openssl >/dev/null 2>&1 || die "openssl not found" + +# 0700: the directory itself, not just the keys. A 0644 directory with 0600 keys +# still tells any local user exactly which hosts have certificates. +mkdir -p "${CERT_DIR}" +chmod 700 "${CERT_DIR}" +umask 077 + +# --------------------------------------------------------------------------- +# --list +# --------------------------------------------------------------------------- +if [[ "${MODE}" == "list" ]]; then + head_ "certificates/" + shopt -s nullglob + found=0 + for crt in "${CERT_DIR}"/*.pem; do + [[ "${crt}" == *-key.pem ]] && continue + found=1 + subject="$(openssl x509 -in "${crt}" -noout -subject 2>/dev/null | sed 's/^subject=//')" + enddate="$(openssl x509 -in "${crt}" -noout -enddate 2>/dev/null | cut -d= -f2)" + if openssl x509 -in "${crt}" -noout -checkend 0 >/dev/null 2>&1; then + state="valid" + else + state="\033[0;31mEXPIRED\033[0m" + fi + printf ' %-28s %b until %s\n' "$(basename "${crt}")" "${state}" "${enddate}" + printf ' %s\n' "${subject}" + done + ((found)) || printf ' (nothing yet — start with %s --ca)\n' "$(basename "$0")" + printf '\n' + exit 0 +fi + +# --------------------------------------------------------------------------- +# --ca +# --------------------------------------------------------------------------- +if [[ "${MODE}" == "ca" ]]; then + if [[ -s "${CA_KEY}" ]] && ((! FORCE)); then + die "${CA_KEY#"${REPO_ROOT}"/} already exists. +Regenerating the CA invalidates every leaf it has signed and every trust store +that holds it. Pass --force only if that is what you mean." + fi + + info "creating the CA (${CA_DAYS} days)" + openssl req -x509 -newkey rsa:4096 -sha256 -nodes \ + -keyout "${CA_KEY}" -out "${CA_CRT}" -days "${CA_DAYS}" \ + -subj "${CA_SUBJECT}" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" 2>/dev/null + chmod 600 "${CA_KEY}" + chmod 644 "${CA_CRT}" + + ok "CA created" + printf ' key %s (never commit, never copy off this host)\n' "${CA_KEY#"${REPO_ROOT}"/}" + printf ' cert %s (safe to distribute — this is what clients trust)\n' "${CA_CRT#"${REPO_ROOT}"/}" + printf '\nIssue a leaf with:\n scripts/gen-certs.sh --host grafana.matrix.elysium --ip 10.0.99.20\n\n' + exit 0 +fi + +# --------------------------------------------------------------------------- +# --host +# --------------------------------------------------------------------------- +[[ -s "${CA_KEY}" && -s "${CA_CRT}" ]] || die "no CA yet — run: scripts/gen-certs.sh --ca" + +KEY="${CERT_DIR}/${HOST}-key.pem" +CRT="${CERT_DIR}/${HOST}.pem" + +if [[ -s "${KEY}" ]] && ((! FORCE)); then + die "${KEY#"${REPO_ROOT}"/} already exists — pass --force to replace it" +fi + +# A certificate with no subjectAltName is rejected outright by every current +# browser and by Go's crypto/tls, which is what Prometheus, Alertmanager and +# Grafana are built on. CN alone has not been sufficient for years, and the +# failure — "x509: certificate relies on legacy Common Name field" — reads as a +# trust problem rather than a missing field, so it costs an hour to diagnose. +SAN="DNS:${HOST}" +for ip in "${IPS[@]}"; do + SAN+=",IP:${ip}" +done + +# 825 days is not arbitrary: browsers reject leaf certificates valid for longer. +# A 10-year leaf looks like less work and produces something nothing will trust. +if ((LEAF_DAYS > 825)); then + die "leaf lifetime ${LEAF_DAYS} exceeds 825 days, which browsers reject outright" +fi + +info "issuing ${HOST} (${LEAF_DAYS} days)" +info "SANs: ${SAN}" + +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT INT TERM + +openssl req -newkey rsa:2048 -sha256 -nodes \ + -keyout "${KEY}" -out "${TMP}/csr.pem" \ + -subj "/CN=${HOST}/O=matrix.elysium" 2>/dev/null + +openssl x509 -req -in "${TMP}/csr.pem" -sha256 \ + -CA "${CA_CRT}" -CAkey "${CA_KEY}" -CAcreateserial \ + -CAserial "${CERT_DIR}/ca.srl" \ + -out "${CRT}" -days "${LEAF_DAYS}" \ + -extfile <(printf 'subjectAltName=%s\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n' "${SAN}") 2>/dev/null + +chmod 600 "${KEY}" +chmod 644 "${CRT}" + +# Prove it verifies against the CA now, rather than discovering at deploy time +# that the chain does not build. +openssl verify -CAfile "${CA_CRT}" "${CRT}" >/dev/null 2>&1 \ + || die "the issued certificate does not verify against the CA — refusing to report success" + +ok "issued ${CRT#"${REPO_ROOT}"/}" +printf ' key %s\n' "${KEY#"${REPO_ROOT}"/}" +printf ' SANs %s\n' "${SAN}" +printf ' verified against %s\n\n' "${CA_CRT#"${REPO_ROOT}"/}" diff --git a/scripts/validate.sh b/scripts/validate.sh index fe040be..96995c8 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -229,12 +229,16 @@ fi # gitleaks cannot cover .purge-secrets.txt — it is gitignored (so the filesystem # scan skips it) and holds bare literals with no keyword context to match. Being # untracked is the control. +# certificates/ is in that list because its contents were committed once and +# had to be removed by rewriting every commit in the repository. The cheapest +# possible check is that it never becomes tracked again. if git ls-files --error-unmatch "${STACK}/.env" >/dev/null 2>&1 \ || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q . \ - || git ls-files | grep -q '\.purge-secrets\.txt'; then - fail "a rendered, decrypted or purge-secrets file is tracked by git" + || git ls-files | grep -q '\.purge-secrets\.txt' \ + || git ls-files | grep -q '^certificates/'; then + fail "a rendered, decrypted, purge-secrets or certificate file is tracked by git" else - pass "no rendered, decrypted or purge-secrets files tracked" + pass "no rendered, decrypted, purge-secrets or certificate files tracked" fi printf '\n'