From df7b21f7151151797eb8baace88353e926ed77d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Sat, 10 Oct 2026 19:14:14 +0000 Subject: [PATCH] test: the boot test asserts the database hooks and the cluster after first boot On the step 12 image of 2026-10-09, postgresql@17-main started before keel-host-keys made the snakeoil certificate and stopped. 35pgsqlpass and 36pgsqlverify failed, then 36pgsqlverify started the cluster again, so the cluster was active at the end. The boot test failed only later, on the declared password. Reproduced on the published layer with keel-host-keys made 10 s slower. tests/boot-test.sh now checks, after the first boot: - bt_db_hooks_verdict: 35pgsqlpass and 36pgsqlverify completed, from /var/log/inithooks.log; - bt_cluster_verdict: there is a cluster unit (systemctl list-units postgresql@*.service) and every cluster is active (running). Seven new tests in tests/boot-test.bats; tests/lib/boot-test-lib.sh stays at 100 percent. The fix is Keel-Linux/inithooks#45. --- COVERAGE.md | 4 +-- tests/boot-test.bats | 57 ++++++++++++++++++++++++++++++++++++++ tests/boot-test.sh | 10 +++++++ tests/lib/boot-test-lib.sh | 42 ++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/COVERAGE.md b/COVERAGE.md index 2a82de3..63169f2 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -10,7 +10,7 @@ LXC as the acceptance test of a recipe, docs/org-plan.md section 1). | --- | --- | --- | --- | | overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch | | overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter | -| tests/lib/boot-test-lib.sh | tests/boot-test.bats (44 tests) | 100 percent (154/154) under kcov | argument parsing, address discovery, deadlines, the container marks, the database, module, Webmin and diff verdicts | +| tests/lib/boot-test-lib.sh | tests/boot-test.bats (51 tests) | 100 percent (173/173) under kcov | argument parsing, address discovery, deadlines, the container marks, the database hooks, cluster, database, module, Webmin and diff verdicts | | bin/keel-project-packages | tests/project-packages.bats (6 tests) | 100 percent (32/32) under kcov | inithooks, confconsole and keel are installed at apt's candidate, and the candidate is the Keel archive's in the suite of the track (KEEL_APT_TRACK); a version below the candidate, another source or an unknown track fails | | conf.d/main | the build | integration only | build time script, 0004 pragmatic limits | | tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root | @@ -25,7 +25,7 @@ the workflow sets to 100, the measured number. It is only ever raised kcov line coverage (threshold 100 percent): 100.00 41/41 postgresql.sh 100.00 15/15 36pgsqlverify - 100.00 154/154 boot-test-lib.sh + 100.00 173/173 boot-test-lib.sh 100.00 32/32 keel-project-packages ## What the hook tests cover diff --git a/tests/boot-test.bats b/tests/boot-test.bats index 13e5a36..8af0532 100644 --- a/tests/boot-test.bats +++ b/tests/boot-test.bats @@ -429,3 +429,60 @@ DEF [ "$status" -eq 1 ] [[ $output == *"failed with exit 127"* ]] } + +# What 'systemctl list-units --all --plain --no-legend postgresql@*.service' +# prints on the booted machine: UNIT LOAD ACTIVE SUB DESCRIPTION. +@test "cluster_verdict: every cluster active and running passes" { + run bt_cluster_verdict "postgresql@17-main.service loaded active running PostgreSQL Cluster 17-main" + [ "$status" -eq 0 ] + [[ $output == *"postgresql@17-main.service is active (running)"* ]] +} + +@test "cluster_verdict: a cluster that failed at boot fails the test" { + # the step 12 image of 2026-10-09: the cluster started before the + # snakeoil certificate existed and stopped + run bt_cluster_verdict "postgresql@17-main.service loaded failed failed PostgreSQL Cluster 17-main" + [ "$status" -eq 1 ] + [[ $output == *"postgresql@17-main.service is failed (failed), not active (running)"* ]] +} + +@test "cluster_verdict: one cluster down among several fails the test" { + run bt_cluster_verdict "$(printf '%s\n' \ + "postgresql@17-main.service loaded active running PostgreSQL Cluster 17-main" \ + "postgresql@17-other.service loaded inactive dead PostgreSQL Cluster 17-other")" + [ "$status" -eq 1 ] + [[ $output == *"postgresql@17-other.service is inactive (dead)"* ]] +} + +@test "cluster_verdict: no cluster at all fails the test" { + run bt_cluster_verdict "" + [ "$status" -eq 1 ] + [[ $output == *"no PostgreSQL cluster unit"* ]] +} + +# The database lines of /var/log/inithooks.log after the first boot. +@test "db_hooks_verdict: both database hooks completed passes" { + run bt_db_hooks_verdict "$(printf '%s\n' \ + "INFO: [35pgsqlpass] running" "INFO: [35pgsqlpass] successfully completed" \ + "INFO: [36pgsqlverify] running" "INFO: [36pgsqlverify] successfully completed")" + [ "$status" -eq 0 ] + [[ $output == *"35pgsqlpass completed"* ]] + [[ $output == *"36pgsqlverify completed"* ]] +} + +@test "db_hooks_verdict: a failed hook fails the test even when the cluster runs later" { + # the step 12 image: the hooks failed on a stopped cluster, and + # 36pgsqlverify started the cluster afterwards, so it was active at the end + run bt_db_hooks_verdict "$(printf '%s\n' \ + "INFO: [35pgsqlpass] running" "ERR: [35pgsqlpass] failed - exit code 1" \ + "INFO: [36pgsqlverify] running" "ERR: [36pgsqlverify] failed - exit code 1")" + [ "$status" -eq 1 ] + [[ $output == *"35pgsqlpass did not complete"* ]] + [[ $output == *"36pgsqlverify did not complete"* ]] +} + +@test "db_hooks_verdict: a hook that never ran fails the test" { + run bt_db_hooks_verdict "INFO: [35pgsqlpass] successfully completed" + [ "$status" -eq 1 ] + [[ $output == *"36pgsqlverify did not complete"* ]] +} diff --git a/tests/boot-test.sh b/tests/boot-test.sh index fa70f81..f0645ce 100755 --- a/tests/boot-test.sh +++ b/tests/boot-test.sh @@ -124,6 +124,16 @@ log "first boot finished; ssh root@$addr" log "the database lines of the container's inithooks log:" grep -E '35pgsqlpass|36pgsqlverify|PostgreSQL' "$BT_ROOTFS/var/log/inithooks.log" || true +# 6b. Both database hooks completed, and every PostgreSQL cluster is active +# after the first boot. On the step 12 image of 2026-10-09 the cluster +# started before its certificate existed and stopped, the hooks failed +# on it, and 36pgsqlverify started it again: so the hooks are checked +# as well as the cluster. +bt_db_hooks_verdict "$(cat "$BT_ROOTFS/var/log/inithooks.log")" +clusters=$(lxc attach -- systemctl list-units --all --plain --no-legend \ + 'postgresql@*.service' 2>/dev/null || true) +bt_cluster_verdict "$clusters" + # 7. The declarative path, end to end: a client connection to the database # with the password the description declared. The client runs inside the # container over TCP on [::1], because the cluster listens on the diff --git a/tests/lib/boot-test-lib.sh b/tests/lib/boot-test-lib.sh index 0da5517..935808a 100644 --- a/tests/lib/boot-test-lib.sh +++ b/tests/lib/boot-test-lib.sh @@ -378,6 +378,48 @@ bt_module_verdict() { return 1 } +bt_cluster_verdict() { + # bt_cluster_verdict UNITS: what 'systemctl list-units --all --plain + # --no-legend postgresql@*.service' printed on the booted machine, one + # unit per line (UNIT LOAD ACTIVE SUB DESCRIPTION). After the first + # boot every cluster must be active and running. The step 12 image of + # 2026-10-09 started postgresql@17-main before the snakeoil certificate + # existed; the cluster stopped and 35pgsqlpass and 36pgsqlverify failed. + local unit load active sub rest count=0 bad=0 + while read -r unit load active sub rest; do + [ -n "$unit" ] || continue + count=$((count + 1)) + if [ "$active" = active ] && [ "$sub" = running ]; then + echo "boot-test: $unit is active (running)" + else + echo "boot-test: $unit is $active ($sub), not active (running)" >&2 + bad=1 + fi + done <<< "${1-}" + if [ "$count" -eq 0 ]; then + echo "boot-test: no PostgreSQL cluster unit on the booted machine" >&2 + return 1 + fi + return "$bad" +} + +bt_db_hooks_verdict() { + # bt_db_hooks_verdict LOG: the lines of /var/log/inithooks.log after the + # first boot. Both database hooks must have completed. The cluster being + # active at the end is not enough: on the step 12 image the hooks failed + # on a stopped cluster, and 36pgsqlverify then started it. + local hook bad=0 + for hook in 35pgsqlpass 36pgsqlverify; do + if grep -qxF "INFO: [$hook] successfully completed" <<< "${1-}"; then + echo "boot-test: $hook completed" + else + echo "boot-test: $hook did not complete in the first boot" >&2 + bad=1 + fi + done + return "$bad" +} + bt_db_verdict() { # bt_db_verdict OUTPUT: what the database client printed when it # connected with the declared password and ran the probe query. The