test: the boot test asserts the database hooks and the cluster after first boot

COVERAGE.md

@@ -10,7 +10,7 @@ LXC as the acceptance test of a recipe, docs/org-plan.md section 1).
 | --- | --- | --- | --- |
 | overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch |
 | overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter |
-| tests/lib/boot-test-lib.sh | tests/boot-test.bats (44 tests) | 100 percent (154/154) under kcov | argument parsing, address discovery, deadlines, the container marks, the database, module, Webmin and diff verdicts |
+| tests/lib/boot-test-lib.sh | tests/boot-test.bats (51 tests) | 100 percent (173/173) under kcov | argument parsing, address discovery, deadlines, the container marks, the database hooks, cluster, database, module, Webmin and diff verdicts |
 | bin/keel-project-packages | tests/project-packages.bats (6 tests) | 100 percent (32/32) under kcov | inithooks, confconsole and keel are installed at apt's candidate, and the candidate is the Keel archive's in the suite of the track (KEEL_APT_TRACK); a version below the candidate, another source or an unknown track fails |
 | conf.d/main | the build | integration only | build time script, 0004 pragmatic limits |
 | tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root |
@@ -25,7 +25,7 @@ the workflow sets to 100, the measured number. It is only ever raised
     kcov line coverage (threshold 100 percent):
      100.00  41/41  postgresql.sh
      100.00  15/15  36pgsqlverify
-     100.00  154/154  boot-test-lib.sh
+     100.00  173/173  boot-test-lib.sh
      100.00  32/32  keel-project-packages
 
 ## What the hook tests cover

tests/boot-test.bats

@@ -429,3 +429,60 @@ DEF
     [ "$status" -eq 1 ]
     [[ $output == *"failed with exit 127"* ]]
 }
+
+# What 'systemctl list-units --all --plain --no-legend postgresql@*.service'
+# prints on the booted machine: UNIT LOAD ACTIVE SUB DESCRIPTION.
+@test "cluster_verdict: every cluster active and running passes" {
+    run bt_cluster_verdict "[email protected] loaded active running PostgreSQL Cluster 17-main"
+    [ "$status" -eq 0 ]
+    [[ $output == *"[email protected] is active (running)"* ]]
+}
+
+@test "cluster_verdict: a cluster that failed at boot fails the test" {
+    # the step 12 image of 2026-10-09: the cluster started before the
+    # snakeoil certificate existed and stopped
+    run bt_cluster_verdict "[email protected] loaded failed failed PostgreSQL Cluster 17-main"
+    [ "$status" -eq 1 ]
+    [[ $output == *"[email protected] is failed (failed), not active (running)"* ]]
+}
+
+@test "cluster_verdict: one cluster down among several fails the test" {
+    run bt_cluster_verdict "$(printf '%s\n' \
+        "[email protected] loaded active running PostgreSQL Cluster 17-main" \
+        "[email protected] loaded inactive dead PostgreSQL Cluster 17-other")"
+    [ "$status" -eq 1 ]
+    [[ $output == *"[email protected] is inactive (dead)"* ]]
+}
+
+@test "cluster_verdict: no cluster at all fails the test" {
+    run bt_cluster_verdict ""
+    [ "$status" -eq 1 ]
+    [[ $output == *"no PostgreSQL cluster unit"* ]]
+}
+
+# The database lines of /var/log/inithooks.log after the first boot.
+@test "db_hooks_verdict: both database hooks completed passes" {
+    run bt_db_hooks_verdict "$(printf '%s\n' \
+        "INFO: [35pgsqlpass] running" "INFO: [35pgsqlpass] successfully completed" \
+        "INFO: [36pgsqlverify] running" "INFO: [36pgsqlverify] successfully completed")"
+    [ "$status" -eq 0 ]
+    [[ $output == *"35pgsqlpass completed"* ]]
+    [[ $output == *"36pgsqlverify completed"* ]]
+}
+
+@test "db_hooks_verdict: a failed hook fails the test even when the cluster runs later" {
+    # the step 12 image: the hooks failed on a stopped cluster, and
+    # 36pgsqlverify started the cluster afterwards, so it was active at the end
+    run bt_db_hooks_verdict "$(printf '%s\n' \
+        "INFO: [35pgsqlpass] running" "ERR: [35pgsqlpass] failed - exit code 1" \
+        "INFO: [36pgsqlverify] running" "ERR: [36pgsqlverify] failed - exit code 1")"
+    [ "$status" -eq 1 ]
+    [[ $output == *"35pgsqlpass did not complete"* ]]
+    [[ $output == *"36pgsqlverify did not complete"* ]]
+}
+
+@test "db_hooks_verdict: a hook that never ran fails the test" {
+    run bt_db_hooks_verdict "INFO: [35pgsqlpass] successfully completed"
+    [ "$status" -eq 1 ]
+    [[ $output == *"36pgsqlverify did not complete"* ]]
+}

tests/boot-test.sh

@@ -124,6 +124,16 @@ log "first boot finished; ssh root@$addr"
 log "the database lines of the container's inithooks log:"
 grep -E '35pgsqlpass|36pgsqlverify|PostgreSQL' "$BT_ROOTFS/var/log/inithooks.log" || true
 
+# 6b. Both database hooks completed, and every PostgreSQL cluster is active
+#     after the first boot. On the step 12 image of 2026-10-09 the cluster
+#     started before its certificate existed and stopped, the hooks failed
+#     on it, and 36pgsqlverify started it again: so the hooks are checked
+#     as well as the cluster.
+bt_db_hooks_verdict "$(cat "$BT_ROOTFS/var/log/inithooks.log")"
+clusters=$(lxc attach -- systemctl list-units --all --plain --no-legend \
+    'postgresql@*.service' 2>/dev/null || true)
+bt_cluster_verdict "$clusters"
+
 # 7. The declarative path, end to end: a client connection to the database
 #    with the password the description declared. The client runs inside the
 #    container over TCP on [::1], because the cluster listens on the

tests/lib/boot-test-lib.sh

@@ -378,6 +378,48 @@ bt_module_verdict() {
     return 1
 }
 
+bt_cluster_verdict() {
+    # bt_cluster_verdict UNITS: what 'systemctl list-units --all --plain
+    # --no-legend postgresql@*.service' printed on the booted machine, one
+    # unit per line (UNIT LOAD ACTIVE SUB DESCRIPTION). After the first
+    # boot every cluster must be active and running. The step 12 image of
+    # 2026-10-09 started postgresql@17-main before the snakeoil certificate
+    # existed; the cluster stopped and 35pgsqlpass and 36pgsqlverify failed.
+    local unit load active sub rest count=0 bad=0
+    while read -r unit load active sub rest; do
+        [ -n "$unit" ] || continue
+        count=$((count + 1))
+        if [ "$active" = active ] && [ "$sub" = running ]; then
+            echo "boot-test: $unit is active (running)"
+        else
+            echo "boot-test: $unit is $active ($sub), not active (running)" >&2
+            bad=1
+        fi
+    done <<< "${1-}"
+    if [ "$count" -eq 0 ]; then
+        echo "boot-test: no PostgreSQL cluster unit on the booted machine" >&2
+        return 1
+    fi
+    return "$bad"
+}
+
+bt_db_hooks_verdict() {
+    # bt_db_hooks_verdict LOG: the lines of /var/log/inithooks.log after the
+    # first boot. Both database hooks must have completed. The cluster being
+    # active at the end is not enough: on the step 12 image the hooks failed
+    # on a stopped cluster, and 36pgsqlverify then started it.
+    local hook bad=0
+    for hook in 35pgsqlpass 36pgsqlverify; do
+        if grep -qxF "INFO: [$hook] successfully completed" <<< "${1-}"; then
+            echo "boot-test: $hook completed"
+        else
+            echo "boot-test: $hook did not complete in the first boot" >&2
+            bad=1
+        fi
+    done
+    return "$bad"
+}
+
 bt_db_verdict() {
     # bt_db_verdict OUTPUT: what the database client printed when it
     # connected with the declared password and ran the probe query. The