fix(ops): the backup recorder reaches each scope's own folder

README.md

@@ -19,7 +19,7 @@
 ![Rust](https://img.shields.io/badge/Rust-gRPC_Engine-000000?logo=rust&logoColor=white)
 ![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white)
 <!-- BEGIN GENERATED REPOSITORY METRICS: badge-tests -->
-![Tests](https://img.shields.io/badge/Tests-4404_collected-28A745?logo=pytest&logoColor=white)
+![Tests](https://img.shields.io/badge/Tests-4408_collected-28A745?logo=pytest&logoColor=white)
 <!-- END GENERATED REPOSITORY METRICS: badge-tests -->
 <!-- BEGIN GENERATED REPOSITORY METRICS: badge-api-operations -->
 ![API Operations](https://img.shields.io/badge/API_Operations-427-blue)
@@ -79,7 +79,7 @@ backend/                 FastAPI API server
   domains/               3 configurable schemas: default, science, healthcare
   scripts/               maintenance + backfills (nif_bayes, work_type, retrospective) + job worker/scheduler entrypoints
 <!-- BEGIN GENERATED REPOSITORY METRICS: tree-backend-tests -->
-  tests/                 333 test files, 4404 tests
+  tests/                 333 test files, 4408 tests
 <!-- END GENERATED REPOSITORY METRICS: tree-backend-tests -->
 frontend/                Next.js 16 App Router
   app/                   65 pages, 90+ components, 8 context providers
@@ -147,7 +147,7 @@ UKIP manages major product and implementation decisions as architecture decision
 | Analytics | pandas, DuckDB, PyArrow, NumPy, SciPy |
 | Background Jobs | Durable PostgreSQL lease queue (broker-free, at-least-once) with worker/scheduler processes |
 <!-- BEGIN GENERATED REPOSITORY METRICS: techstack-testing -->
-| Testing | pytest (4404 tests collected), Vitest (312 tests), Playwright |
+| Testing | pytest (4408 tests collected), Vitest (312 tests), Playwright |
 <!-- END GENERATED REPOSITORY METRICS: techstack-testing -->
 | Deployment | Docker Compose, GHCR images, Dokploy-oriented production compose |
 | Monitoring | Sentry (opt-in), structured logging |
@@ -284,7 +284,7 @@ cd frontend && npx tsc --noEmit
 ```
 
 <!-- BEGIN GENERATED REPOSITORY METRICS: summary-test-stats -->
-**Current test stats:** 4404 backend tests collected across 333 test files. Frontend: 312 Vitest tests.
+**Current test stats:** 4408 backend tests collected across 333 test files. Frontend: 312 Vitest tests.
 <!-- END GENERATED REPOSITORY METRICS: summary-test-stats -->
 
 ---

backend/tests/test_backup_evidence_recorder.py

@@ -199,3 +199,80 @@ def test_no_credential_or_key_path_reaches_the_output(tmp_path):
     assert "AKIAEXAMPLE" not in everything
     assert "bucket" not in json.dumps(_documents(env)), "documents carry no bucket name"
     assert PREFIX not in json.dumps(_documents(env)), "documents carry no prefix"
+
+
+# Production keeps each scope in its own top-level folder (checked 2026-09-29):
+# dumps under the database service's folder, volume archives under the compose
+# app's. One shared prefix cannot reach both, so each scope may have its own.
+DB_PREFIX = "ukip-dbukip-eqmmhw/pg/"
+VOL_PREFIX = "ukip-app-danmq1_ukip-backend/static/"
+SPLIT_DB_KEY = f"{DB_PREFIX}2026-09-25T03-00-00-076Z.sql.gz"
+SPLIT_VOL_KEY = f"{VOL_PREFIX}ukip_static_data-2026-09-25T03-05-00-066Z.tar"
+
+
+def _split_setup(tmp_path):
+    env = _setup(tmp_path, db_bytes=None)
+    objects = Path(env["STUB_OBJECTS"])
+    (objects / Path(SPLIT_DB_KEY).name).write_bytes(DUMP)
+    (objects / Path(SPLIT_VOL_KEY).name).write_bytes(ARCHIVE)
+    del env["S3_BACKUP_PREFIX"]
+    env.update({
+        "S3_BACKUP_PREFIX_DATABASE": DB_PREFIX,
+        "S3_BACKUP_PREFIX_VOLUME": VOL_PREFIX,
+        "STUB_DB_LISTING": f"{SPLIT_DB_KEY}\\t{len(DUMP)}\\t{_stamp(60)}",
+        "STUB_VOL_LISTING": f"{SPLIT_VOL_KEY}\\t{len(ARCHIVE)}\\t{_stamp(55)}",
+    })
+    return env
+
+
+def _listings(env) -> list[str]:
+    log = Path(env["STUB_LOG"]).read_text(encoding="utf-8").splitlines()
+    return [line for line in log if line.startswith("s3api list-objects-v2")]
+
+
+def test_each_scope_is_listed_under_its_own_prefix(tmp_path):
+    env = _split_setup(tmp_path)
+
+    completed = _run(env)
+
+    assert completed.returncode == 0, completed.stderr
+    listings = _listings(env)
+    db_listing = next(line for line in listings if ".sql.gz" in line)
+    vol_listing = next(line for line in listings if ".tar" in line)
+    assert f"--prefix {DB_PREFIX} " in db_listing
+    assert f"--prefix {VOL_PREFIX} " in vol_listing
+
+
+def test_split_prefixes_record_both_scopes_relative_to_their_own_prefix(tmp_path):
+    env = _split_setup(tmp_path)
+
+    assert _run(env).returncode == 0
+    docs = _documents(env)
+    ids = {doc["scope"]: doc["backup_id"] for doc in docs.values()}
+    assert ids == {
+        "database": "2026-09-25T03-00-00-076Z.sql.gz",
+        "volume": "ukip_static_data-2026-09-25T03-05-00-066Z.tar",
+    }
+    for document in docs.values():
+        assert isinstance(_parse(document), EvidenceDocument)
+        assert DB_PREFIX not in json.dumps(document) and VOL_PREFIX not in json.dumps(document)
+
+
+def test_a_scope_without_any_prefix_is_refused(tmp_path):
+    env = _split_setup(tmp_path)
+    del env["S3_BACKUP_PREFIX_VOLUME"]
+
+    completed = _run(env)
+
+    assert completed.returncode != 0
+    assert "S3_BACKUP_PREFIX_VOLUME" in completed.stderr
+    assert _documents(env) == {}
+
+
+def test_the_shared_prefix_still_covers_both_scopes(tmp_path):
+    # Installations that keep both scopes under one folder need no change.
+    env = _setup(tmp_path, db_bytes=DUMP, vol_bytes=ARCHIVE)
+
+    assert _run(env).returncode == 0
+    for listing in _listings(env):
+        assert f"--prefix {PREFIX} " in listing

docs/generated/repo_metrics.json

@@ -1,7 +1,7 @@
 {
   "schema_version": 1,
   "metrics": {
-    "backend_tests_collected": 4404,
+    "backend_tests_collected": 4408,
     "backend_test_files": 333,
     "frontend_vitest_tests": 312,
     "api_operations": 427,

docs/operating/BACKUP_RESTORE_RUNBOOK.md

@@ -316,15 +316,52 @@ involved** (owner decision on #370). The only credential is a storage one on
 the host, and the trust boundary is root on that host, which already holds the
 database.
 
-1. Create a **read-only** storage credential with `ListBucket` and `GetObject`
-   on the backup prefix only: no write, no delete, nothing outside the prefix.
-   It is separate from the probe's list-only credential.
-2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600), with the
-   same variables as the probe's file: `AWS_ACCESS_KEY_ID`,
-   `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_BACKUP_ENDPOINT`,
-   `S3_BACKUP_BUCKET`, `S3_BACKUP_PREFIX`. Set `UKIP_BACKUP_ENVIRONMENT` if it
-   is not `production`; it must match the backend's, or every document is
-   rejected as `other_environment`.
+**Where the backups are.** Dokploy writes each scope under its own top-level
+folder (production, checked 2026-09-29): database dumps under the database
+service's folder (`ukip-dbukip-eqmmhw/pg/`), volume archives under the compose
+app's (`ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/`).
+No single prefix reaches both, so the recorder takes one per scope. The probe's
+`S3_BACKUP_PREFIX` covers the database folder only.
+
+1. Create a **read-only** storage credential: `ListBucket` limited to the two
+   folders and `GetObject` on them, nothing else. It is separate from the
+   probe's list-only credential. As an IAM policy (bucket name from
+   `/etc/ukip/backup-probe.env`):
+
+   ```json
+   {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Sid": "ListTheTwoBackupFolders",
+         "Effect": "Allow",
+         "Action": "s3:ListBucket",
+         "Resource": "arn:aws:s3:::<bucket>",
+         "Condition": {"StringLike": {"s3:prefix": [
+           "ukip-dbukip-eqmmhw/pg/*",
+           "ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*"
+         ]}}
+       },
+       {
+         "Sid": "ReadTheBackups",
+         "Effect": "Allow",
+         "Action": "s3:GetObject",
+         "Resource": [
+           "arn:aws:s3:::<bucket>/ukip-dbukip-eqmmhw/pg/*",
+           "arn:aws:s3:::<bucket>/ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*"
+         ]
+       }
+     ]
+   }
+   ```
+
+2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600):
+   `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`,
+   `S3_BACKUP_ENDPOINT`, `S3_BACKUP_BUCKET`, and either `S3_BACKUP_PREFIX`
+   (both scopes under one folder) or, as in production,
+   `S3_BACKUP_PREFIX_DATABASE` and `S3_BACKUP_PREFIX_VOLUME`. Set
+   `UKIP_BACKUP_ENVIRONMENT` if it is not `production`; it must match the
+   backend's, or every document is rejected as `other_environment`.
 3. Install the script and the units:
 
 ```bash

openspec/changes/automate-backup-evidence/tasks.md

@@ -61,11 +61,18 @@ Section 4 is operator work and is what actually turns the automation on.
 
 ## 4. Operator actions (not code)
 
-- [ ] 4.0 Check how the manual cycles recorded `backup_id` in production
+- [x] 4.0 Check how the manual cycles recorded `backup_id` in production
       (`GET /ops/backups?environment=production`): full key or relative to
       the prefix. Suffix matching covers both; this confirms it.
+      Checked 2026-09-29: full keys from the bucket root, e.g.
+      `ukip-dbukip-eqmmhw/pg/2026-09-22T03-00-00-217Z.sql.gz`. It also showed
+      the two scopes in different top-level folders (volumes under
+      `…-danmq1_ukip-backend/static/`), which one `S3_BACKUP_PREFIX` could
+      not reach: the recorder now takes `S3_BACKUP_PREFIX_DATABASE` and
+      `S3_BACKUP_PREFIX_VOLUME`, with the shared prefix as the fallback.
+      The last manually recorded cycle was 2026-09-22.
 - [ ] 4.1 Create the read-only credential: `ListBucket` + `GetObject` on the
-      backup prefix only.
+      two backup folders only (policy in the runbook, §5c).
 - [ ] 4.2 Install the recorder and its timer on the production host.
 - [ ] 4.3 Confirm the first automatic cycle appears in `GET /ops/backups` with
       operator `system:backup-recorder`, and that `backup_freshness` stays `ok`

scripts/ukip-backup-evidence-recorder.sh

@@ -13,9 +13,16 @@
 # Credentials come from the systemd EnvironmentFile (root-owned, mode 600),
 # never from this file. Required environment:
 #   AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY  read-only: ListBucket + GetObject
-#                                              on the backup prefix only
+#                                              on the backup prefixes only
 #   AWS_DEFAULT_REGION                         bucket region
-#   S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET, S3_BACKUP_PREFIX
+#   S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET
+#   S3_BACKUP_PREFIX       the folder both scopes live under, or instead:
+#   S3_BACKUP_PREFIX_DATABASE, S3_BACKUP_PREFIX_VOLUME
+#                          one folder per scope, each overriding the shared
+#                          one. Dokploy writes database dumps under the
+#                          database service's folder and volume archives
+#                          under the compose app's, which no single prefix
+#                          reaches.
 # Optional:
 #   UKIP_EVIDENCE_OUT_DIR   default /var/lib/ukip/signals/backup-evidence
 #   UKIP_BACKUP_ENVIRONMENT default production (must match the backend's)
@@ -39,7 +46,11 @@ RECORDER="ukip-backup-evidence-recorder/1"
 KEEP_DAYS=14
 : "${S3_BACKUP_ENDPOINT:?S3_BACKUP_ENDPOINT is required}"
 : "${S3_BACKUP_BUCKET:?S3_BACKUP_BUCKET is required}"
-: "${S3_BACKUP_PREFIX:?S3_BACKUP_PREFIX is required}"
+: "${S3_BACKUP_PREFIX:=}"
+DATABASE_PREFIX=${S3_BACKUP_PREFIX_DATABASE:-$S3_BACKUP_PREFIX}
+VOLUME_PREFIX=${S3_BACKUP_PREFIX_VOLUME:-$S3_BACKUP_PREFIX}
+: "${DATABASE_PREFIX:?S3_BACKUP_PREFIX_DATABASE (or S3_BACKUP_PREFIX) is required}"
+: "${VOLUME_PREFIX:?S3_BACKUP_PREFIX_VOLUME (or S3_BACKUP_PREFIX) is required}"
 
 RUNNER=${UKIP_AWS_RUNNER:-auto}
 if [ "$RUNNER" = "auto" ]; then
@@ -65,14 +76,14 @@ aws_cli() {
   fi
 }
 
-# newest <suffix> → "key<TAB>size<TAB>last_modified", or nothing when there is none.
-# The CLI paginates the whole listing before applying --query.
+# newest <prefix> <suffix> → "key<TAB>size<TAB>last_modified", or nothing when
+# there is none. The CLI paginates the whole listing before applying --query.
 newest() {
   aws_cli s3api list-objects-v2 \
     --endpoint-url "$S3_BACKUP_ENDPOINT" \
     --bucket "$S3_BACKUP_BUCKET" \
-    --prefix "$S3_BACKUP_PREFIX" \
-    --query "reverse(sort_by(Contents[?ends_with(Key, '$1')], &LastModified))[0].[Key, Size, LastModified]" \
+    --prefix "$1" \
+    --query "reverse(sort_by(Contents[?ends_with(Key, '$2')], &LastModified))[0].[Key, Size, LastModified]" \
     --output text
 }
 
@@ -85,8 +96,8 @@ chmod 0755 "$OUT_DIR"
 failures=0
 
 record() {
-  local scope=$1 suffix=$2 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at
-  if ! line=$(newest "$suffix"); then
+  local scope=$1 suffix=$2 prefix=$3 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at
+  if ! line=$(newest "$prefix" "$suffix"); then
     echo "scope=$scope listing_failed" >&2
     return 1
   fi
@@ -96,7 +107,7 @@ record() {
   fi
   IFS=$'\t' read -r key size last_modified <<<"$line"
 
-  relative=${key#"$S3_BACKUP_PREFIX"}
+  relative=${key#"$prefix"}
   # The key goes into JSON by printf; refuse anything that would need escaping
   # rather than write a document the backend would have to guess about.
   case "$relative" in
@@ -141,8 +152,8 @@ record() {
   echo "scope=$scope backup_id=$relative size=$size gzip_ok=$gzip_ok recorded"
 }
 
-record database .sql.gz || failures=$((failures + 1))
-record volume .tar || failures=$((failures + 1))
+record database .sql.gz "$DATABASE_PREFIX" || failures=$((failures + 1))
+record volume .tar "$VOLUME_PREFIX" || failures=$((failures + 1))
 
 # The backend accepts documents up to 14 days old; keep no more than that.
 find "$OUT_DIR" -maxdepth 1 -type f -name '*.json' -mtime +"$KEEP_DAYS" -delete