From 2b1d5c2bd4d0d02db96a0417b9af3a166ce796c0 Mon Sep 17 00:00:00 2001 From: Zeke Date: Fri, 24 Jul 2026 11:23:00 -0700 Subject: [PATCH] test(k8s): finish P2-3 -- add the upgrade-path e2e + a live cluster-secret regression gate The kind gate only INSTALLED. The plan's P2-3 asks for install AND upgrade, because a chart can lint clean and install fine yet fail to UPGRADE. Adds two steps to the existing kind cluster: 1. UPGRADE + re-test. `helm upgrade` with values that hit BOTH roll mechanisms -- `extraConfig` (rewrites the ConfigMap, flipping the `checksum/config` annotation, and proves the node still BOOTS on the re-rendered TOML since the parse is strict) and `logLevel` (a container-arg change in the pod template) -- then `--wait` + `rollout status` + `helm test` again. So the real rolling-update path (readiness gate, minReadySeconds, preStop lame-duck) is exercised, not a no-op re-apply. Verified `timeout` is in KNOWN_TOML_KEYS (Option), so the strict parse accepts it. 2. CLUSTER-SECRET PRESERVED ACROSS A BARE UPGRADE (#747). This is the only place that guard CAN be tested: it is a `lookup` of the live Secret, which is always empty under `helm template` / --dry-run, so deploy-lint can never reach that branch -- a gap a reviewer flagged when #747 landed. A kind cluster closes it. Installs with cluster.enabled=true, records cluster_secret, bare-upgrades, and fails if it ROTATED (pre-#747 behaviour, which splits a live cluster). No --wait: it asserts a template/lookup invariant, not pod readiness, so it stays fast. Diagnostics-on-failure now also dumps the second namespace. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01FfFZ8gkkNhDBASuntB72HR --- .github/workflows/chart-e2e.yml | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/chart-e2e.yml b/.github/workflows/chart-e2e.yml index 4a72d84..2f341eb 100644 --- a/.github/workflows/chart-e2e.yml +++ b/.github/workflows/chart-e2e.yml @@ -54,6 +54,46 @@ jobs: # even though the PING passed. The Diagnostics-on-failure step below dumps logs when a # real failure leaves the Pod in place (hook-succeeded does not delete a FAILED Pod). helm test ic -n ic-test + - name: Upgrade + re-test (the rolling-update path) + run: | + set -euo pipefail + # P2-3's other half: a chart can lint clean and INSTALL fine yet fail to UPGRADE. Change + # values that hit BOTH roll mechanisms, so this is a real rolling update (readiness gate + # + minReadySeconds + the preStop lame-duck), not a no-op re-apply: + # - extraConfig -> rewrites the ConfigMap, flipping the `checksum/config` annotation + # (the chart's own config-change-rolls-pods mechanism), AND proves + # the node still BOOTS on the re-rendered TOML (the parse is strict); + # - logLevel -> changes a container arg directly in the pod template. + helm upgrade ic deploy/helm/ironcache -n ic-test \ + --set replicas=1 --set cluster.enabled=false \ + --set logLevel=debug --set extraConfig='timeout = 30' \ + --wait --timeout 5m + kubectl -n ic-test rollout status statefulset/ic-ironcache --timeout=5m + # The node must still serve RESP after the roll. + helm test ic -n ic-test + echo "OK: upgrade rolled the pod and the cache still answers PING" + - name: Cluster-secret preserved across a bare `helm upgrade` (#747) + run: | + set -euo pipefail + # Regression gate for #747, and the ONLY place it can run: the guard is a `lookup` of the + # live Secret, which is always empty under `helm template`/--dry-run, so deploy-lint can + # never exercise this branch. A kind cluster can. Before #747 a bare upgrade regenerated + # the random cluster_secret -> peers reject each other -> split-brain. + kubectl create namespace ic-sec + # No --wait: this asserts a TEMPLATE/lookup invariant, not pod readiness (a 1-node raft + # node need not become Ready for the Secret to exist), which also keeps the job fast. + helm install ics deploy/helm/ironcache -n ic-sec \ + --set replicas=1 --set cluster.enabled=true + before="$(kubectl -n ic-sec get secret ics-ironcache-secret -o jsonpath='{.data.cluster_secret}')" + [ -n "$before" ] || { echo "ERROR: no cluster_secret was generated on install"; exit 1; } + helm upgrade ics deploy/helm/ironcache -n ic-sec \ + --set replicas=1 --set cluster.enabled=true + after="$(kubectl -n ic-sec get secret ics-ironcache-secret -o jsonpath='{.data.cluster_secret}')" + if [ "$before" != "$after" ]; then + echo "ERROR: cluster_secret ROTATED across a bare helm upgrade -- this splits a live cluster (#747 regressed)" + exit 1 + fi + echo "OK: cluster_secret preserved across helm upgrade (lookup branch verified live)" - name: Diagnostics on failure if: failure() run: | @@ -61,3 +101,6 @@ jobs: kubectl -n ic-test describe pod -l app.kubernetes.io/instance=ic || true kubectl -n ic-test logs -l app.kubernetes.io/instance=ic --tail=200 || true kubectl -n ic-test get events --sort-by=.lastTimestamp || true + # The cluster-secret namespace (only exists once that step has started). + kubectl -n ic-sec get all,secret 2>/dev/null || true + kubectl -n ic-sec get events --sort-by=.lastTimestamp 2>/dev/null || true