diff --git a/COVERAGE.md b/COVERAGE.md index 2a82de3..63169f2 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -10,7 +10,7 @@ LXC as the acceptance test of a recipe, docs/org-plan.md section 1). | --- | --- | --- | --- | | overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch | | overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter | -| tests/lib/boot-test-lib.sh | tests/boot-test.bats (44 tests) | 100 percent (154/154) under kcov | argument parsing, address discovery, deadlines, the container marks, the database, module, Webmin and diff verdicts | +| tests/lib/boot-test-lib.sh | tests/boot-test.bats (51 tests) | 100 percent (173/173) under kcov | argument parsing, address discovery, deadlines, the container marks, the database hooks, cluster, database, module, Webmin and diff verdicts | | bin/keel-project-packages | tests/project-packages.bats (6 tests) | 100 percent (32/32) under kcov | inithooks, confconsole and keel are installed at apt's candidate, and the candidate is the Keel archive's in the suite of the track (KEEL_APT_TRACK); a version below the candidate, another source or an unknown track fails | | conf.d/main | the build | integration only | build time script, 0004 pragmatic limits | | tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root | @@ -25,7 +25,7 @@ the workflow sets to 100, the measured number. It is only ever raised kcov line coverage (threshold 100 percent): 100.00 41/41 postgresql.sh 100.00 15/15 36pgsqlverify - 100.00 154/154 boot-test-lib.sh + 100.00 173/173 boot-test-lib.sh 100.00 32/32 keel-project-packages ## What the hook tests cover diff --git a/tests/boot-test.bats b/tests/boot-test.bats index 13e5a36..8af0532 100644 --- a/tests/boot-test.bats +++ b/tests/boot-test.bats @@ -429,3 +429,60 @@ DEF [ "$status" -eq 1 ] [[ $output == *"failed with exit 127"* ]] } + +# What 'systemctl list-units --all --plain --no-legend postgresql@*.service' +# prints on the booted machine: UNIT LOAD ACTIVE SUB DESCRIPTION. +@test "cluster_verdict: every cluster active and running passes" { + run bt_cluster_verdict "postgresql@17-main.service loaded active running PostgreSQL Cluster 17-main" + [ "$status" -eq 0 ] + [[ $output == *"postgresql@17-main.service is active (running)"* ]] +} + +@test "cluster_verdict: a cluster that failed at boot fails the test" { + # the step 12 image of 2026-10-09: the cluster started before the + # snakeoil certificate existed and stopped + run bt_cluster_verdict "postgresql@17-main.service loaded failed failed PostgreSQL Cluster 17-main" + [ "$status" -eq 1 ] + [[ $output == *"postgresql@17-main.service is failed (failed), not active (running)"* ]] +} + +@test "cluster_verdict: one cluster down among several fails the test" { + run bt_cluster_verdict "$(printf '%s\n' \ + "postgresql@17-main.service loaded active running PostgreSQL Cluster 17-main" \ + "postgresql@17-other.service loaded inactive dead PostgreSQL Cluster 17-other")" + [ "$status" -eq 1 ] + [[ $output == *"postgresql@17-other.service is inactive (dead)"* ]] +} + +@test "cluster_verdict: no cluster at all fails the test" { + run bt_cluster_verdict "" + [ "$status" -eq 1 ] + [[ $output == *"no PostgreSQL cluster unit"* ]] +} + +# The database lines of /var/log/inithooks.log after the first boot. +@test "db_hooks_verdict: both database hooks completed passes" { + run bt_db_hooks_verdict "$(printf '%s\n' \ + "INFO: [35pgsqlpass] running" "INFO: [35pgsqlpass] successfully completed" \ + "INFO: [36pgsqlverify] running" "INFO: [36pgsqlverify] successfully completed")" + [ "$status" -eq 0 ] + [[ $output == *"35pgsqlpass completed"* ]] + [[ $output == *"36pgsqlverify completed"* ]] +} + +@test "db_hooks_verdict: a failed hook fails the test even when the cluster runs later" { + # the step 12 image: the hooks failed on a stopped cluster, and + # 36pgsqlverify started the cluster afterwards, so it was active at the end + run bt_db_hooks_verdict "$(printf '%s\n' \ + "INFO: [35pgsqlpass] running" "ERR: [35pgsqlpass] failed - exit code 1" \ + "INFO: [36pgsqlverify] running" "ERR: [36pgsqlverify] failed - exit code 1")" + [ "$status" -eq 1 ] + [[ $output == *"35pgsqlpass did not complete"* ]] + [[ $output == *"36pgsqlverify did not complete"* ]] +} + +@test "db_hooks_verdict: a hook that never ran fails the test" { + run bt_db_hooks_verdict "INFO: [35pgsqlpass] successfully completed" + [ "$status" -eq 1 ] + [[ $output == *"36pgsqlverify did not complete"* ]] +} diff --git a/tests/boot-test.sh b/tests/boot-test.sh index fa70f81..f0645ce 100755 --- a/tests/boot-test.sh +++ b/tests/boot-test.sh @@ -124,6 +124,16 @@ log "first boot finished; ssh root@$addr" log "the database lines of the container's inithooks log:" grep -E '35pgsqlpass|36pgsqlverify|PostgreSQL' "$BT_ROOTFS/var/log/inithooks.log" || true +# 6b. Both database hooks completed, and every PostgreSQL cluster is active +# after the first boot. On the step 12 image of 2026-10-09 the cluster +# started before its certificate existed and stopped, the hooks failed +# on it, and 36pgsqlverify started it again: so the hooks are checked +# as well as the cluster. +bt_db_hooks_verdict "$(cat "$BT_ROOTFS/var/log/inithooks.log")" +clusters=$(lxc attach -- systemctl list-units --all --plain --no-legend \ + 'postgresql@*.service' 2>/dev/null || true) +bt_cluster_verdict "$clusters" + # 7. The declarative path, end to end: a client connection to the database # with the password the description declared. The client runs inside the # container over TCP on [::1], because the cluster listens on the diff --git a/tests/lib/boot-test-lib.sh b/tests/lib/boot-test-lib.sh index 0da5517..935808a 100644 --- a/tests/lib/boot-test-lib.sh +++ b/tests/lib/boot-test-lib.sh @@ -378,6 +378,48 @@ bt_module_verdict() { return 1 } +bt_cluster_verdict() { + # bt_cluster_verdict UNITS: what 'systemctl list-units --all --plain + # --no-legend postgresql@*.service' printed on the booted machine, one + # unit per line (UNIT LOAD ACTIVE SUB DESCRIPTION). After the first + # boot every cluster must be active and running. The step 12 image of + # 2026-10-09 started postgresql@17-main before the snakeoil certificate + # existed; the cluster stopped and 35pgsqlpass and 36pgsqlverify failed. + local unit load active sub rest count=0 bad=0 + while read -r unit load active sub rest; do + [ -n "$unit" ] || continue + count=$((count + 1)) + if [ "$active" = active ] && [ "$sub" = running ]; then + echo "boot-test: $unit is active (running)" + else + echo "boot-test: $unit is $active ($sub), not active (running)" >&2 + bad=1 + fi + done <<< "${1-}" + if [ "$count" -eq 0 ]; then + echo "boot-test: no PostgreSQL cluster unit on the booted machine" >&2 + return 1 + fi + return "$bad" +} + +bt_db_hooks_verdict() { + # bt_db_hooks_verdict LOG: the lines of /var/log/inithooks.log after the + # first boot. Both database hooks must have completed. The cluster being + # active at the end is not enough: on the step 12 image the hooks failed + # on a stopped cluster, and 36pgsqlverify then started it. + local hook bad=0 + for hook in 35pgsqlpass 36pgsqlverify; do + if grep -qxF "INFO: [$hook] successfully completed" <<< "${1-}"; then + echo "boot-test: $hook completed" + else + echo "boot-test: $hook did not complete in the first boot" >&2 + bad=1 + fi + done + return "$bad" +} + bt_db_verdict() { # bt_db_verdict OUTPUT: what the database client printed when it # connected with the declared password and ran the probe query. The