From 60f94f4bad301b7a22fc22d227ac61f01ae5c946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Keilyn=20Rodr=C3=ADguez=20Perojo?= <73491750+keilynrp@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:25:06 -0600 Subject: [PATCH] fix(ops): the backup recorder reaches each scope's own folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking production (#370, task 4.0) showed Dokploy writing database dumps under the database service's folder (ukip-dbukip-eqmmhw/pg/) and volume archives under the compose app's (…-danmq1_ukip-backend/static/). The recorder listed both scopes under one S3_BACKUP_PREFIX, so with the probe's prefix it would have found the dumps and reported no_object for volumes on every cycle. Its tests had assumed both under one folder. It now takes S3_BACKUP_PREFIX_DATABASE and S3_BACKUP_PREFIX_VOLUME, each falling back to S3_BACKUP_PREFIX, and refuses to run with a scope that has neither. backup_id stays relative to the prefix it was listed under, so no document carries a folder name; manual rows, recorded as full keys, still match by path suffix. Tests: each scope is listed under its own prefix, both are recorded relative to it and parse, a scope with no prefix is refused, and the shared prefix still covers both. Giving the volume scope the database prefix fails two of them. The runbook's §5c says where production keeps each scope and gives the credential's policy for exactly the two folders; task 4.0 is done with what it found (full keys; the last manual cycle was 2026-09-22). Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- .../tests/test_backup_evidence_recorder.py | 77 +++++++++++++++++++ docs/generated/repo_metrics.json | 2 +- docs/operating/BACKUP_RESTORE_RUNBOOK.md | 55 ++++++++++--- .../changes/automate-backup-evidence/tasks.md | 11 ++- scripts/ukip-backup-evidence-recorder.sh | 35 ++++++--- 6 files changed, 160 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c55c46fb..185d13b6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ![Rust](https://img.shields.io/badge/Rust-gRPC_Engine-000000?logo=rust&logoColor=white) ![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white) -![Tests](https://img.shields.io/badge/Tests-4404_collected-28A745?logo=pytest&logoColor=white) +![Tests](https://img.shields.io/badge/Tests-4408_collected-28A745?logo=pytest&logoColor=white) ![API Operations](https://img.shields.io/badge/API_Operations-427-blue) @@ -79,7 +79,7 @@ backend/ FastAPI API server domains/ 3 configurable schemas: default, science, healthcare scripts/ maintenance + backfills (nif_bayes, work_type, retrospective) + job worker/scheduler entrypoints - tests/ 333 test files, 4404 tests + tests/ 333 test files, 4408 tests frontend/ Next.js 16 App Router app/ 65 pages, 90+ components, 8 context providers @@ -147,7 +147,7 @@ UKIP manages major product and implementation decisions as architecture decision | Analytics | pandas, DuckDB, PyArrow, NumPy, SciPy | | Background Jobs | Durable PostgreSQL lease queue (broker-free, at-least-once) with worker/scheduler processes | -| Testing | pytest (4404 tests collected), Vitest (312 tests), Playwright | +| Testing | pytest (4408 tests collected), Vitest (312 tests), Playwright | | Deployment | Docker Compose, GHCR images, Dokploy-oriented production compose | | Monitoring | Sentry (opt-in), structured logging | @@ -284,7 +284,7 @@ cd frontend && npx tsc --noEmit ``` -**Current test stats:** 4404 backend tests collected across 333 test files. Frontend: 312 Vitest tests. +**Current test stats:** 4408 backend tests collected across 333 test files. Frontend: 312 Vitest tests. --- diff --git a/backend/tests/test_backup_evidence_recorder.py b/backend/tests/test_backup_evidence_recorder.py index 978998c8..946fe74f 100644 --- a/backend/tests/test_backup_evidence_recorder.py +++ b/backend/tests/test_backup_evidence_recorder.py @@ -199,3 +199,80 @@ def test_no_credential_or_key_path_reaches_the_output(tmp_path): assert "AKIAEXAMPLE" not in everything assert "bucket" not in json.dumps(_documents(env)), "documents carry no bucket name" assert PREFIX not in json.dumps(_documents(env)), "documents carry no prefix" + + +# Production keeps each scope in its own top-level folder (checked 2026-09-29): +# dumps under the database service's folder, volume archives under the compose +# app's. One shared prefix cannot reach both, so each scope may have its own. +DB_PREFIX = "ukip-dbukip-eqmmhw/pg/" +VOL_PREFIX = "ukip-app-danmq1_ukip-backend/static/" +SPLIT_DB_KEY = f"{DB_PREFIX}2026-09-25T03-00-00-076Z.sql.gz" +SPLIT_VOL_KEY = f"{VOL_PREFIX}ukip_static_data-2026-09-25T03-05-00-066Z.tar" + + +def _split_setup(tmp_path): + env = _setup(tmp_path, db_bytes=None) + objects = Path(env["STUB_OBJECTS"]) + (objects / Path(SPLIT_DB_KEY).name).write_bytes(DUMP) + (objects / Path(SPLIT_VOL_KEY).name).write_bytes(ARCHIVE) + del env["S3_BACKUP_PREFIX"] + env.update({ + "S3_BACKUP_PREFIX_DATABASE": DB_PREFIX, + "S3_BACKUP_PREFIX_VOLUME": VOL_PREFIX, + "STUB_DB_LISTING": f"{SPLIT_DB_KEY}\\t{len(DUMP)}\\t{_stamp(60)}", + "STUB_VOL_LISTING": f"{SPLIT_VOL_KEY}\\t{len(ARCHIVE)}\\t{_stamp(55)}", + }) + return env + + +def _listings(env) -> list[str]: + log = Path(env["STUB_LOG"]).read_text(encoding="utf-8").splitlines() + return [line for line in log if line.startswith("s3api list-objects-v2")] + + +def test_each_scope_is_listed_under_its_own_prefix(tmp_path): + env = _split_setup(tmp_path) + + completed = _run(env) + + assert completed.returncode == 0, completed.stderr + listings = _listings(env) + db_listing = next(line for line in listings if ".sql.gz" in line) + vol_listing = next(line for line in listings if ".tar" in line) + assert f"--prefix {DB_PREFIX} " in db_listing + assert f"--prefix {VOL_PREFIX} " in vol_listing + + +def test_split_prefixes_record_both_scopes_relative_to_their_own_prefix(tmp_path): + env = _split_setup(tmp_path) + + assert _run(env).returncode == 0 + docs = _documents(env) + ids = {doc["scope"]: doc["backup_id"] for doc in docs.values()} + assert ids == { + "database": "2026-09-25T03-00-00-076Z.sql.gz", + "volume": "ukip_static_data-2026-09-25T03-05-00-066Z.tar", + } + for document in docs.values(): + assert isinstance(_parse(document), EvidenceDocument) + assert DB_PREFIX not in json.dumps(document) and VOL_PREFIX not in json.dumps(document) + + +def test_a_scope_without_any_prefix_is_refused(tmp_path): + env = _split_setup(tmp_path) + del env["S3_BACKUP_PREFIX_VOLUME"] + + completed = _run(env) + + assert completed.returncode != 0 + assert "S3_BACKUP_PREFIX_VOLUME" in completed.stderr + assert _documents(env) == {} + + +def test_the_shared_prefix_still_covers_both_scopes(tmp_path): + # Installations that keep both scopes under one folder need no change. + env = _setup(tmp_path, db_bytes=DUMP, vol_bytes=ARCHIVE) + + assert _run(env).returncode == 0 + for listing in _listings(env): + assert f"--prefix {PREFIX} " in listing diff --git a/docs/generated/repo_metrics.json b/docs/generated/repo_metrics.json index 93ae56d3..27e03bbc 100644 --- a/docs/generated/repo_metrics.json +++ b/docs/generated/repo_metrics.json @@ -1,7 +1,7 @@ { "schema_version": 1, "metrics": { - "backend_tests_collected": 4404, + "backend_tests_collected": 4408, "backend_test_files": 333, "frontend_vitest_tests": 312, "api_operations": 427, diff --git a/docs/operating/BACKUP_RESTORE_RUNBOOK.md b/docs/operating/BACKUP_RESTORE_RUNBOOK.md index 01431d0d..9c4085b1 100644 --- a/docs/operating/BACKUP_RESTORE_RUNBOOK.md +++ b/docs/operating/BACKUP_RESTORE_RUNBOOK.md @@ -316,15 +316,52 @@ involved** (owner decision on #370). The only credential is a storage one on the host, and the trust boundary is root on that host, which already holds the database. -1. Create a **read-only** storage credential with `ListBucket` and `GetObject` - on the backup prefix only: no write, no delete, nothing outside the prefix. - It is separate from the probe's list-only credential. -2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600), with the - same variables as the probe's file: `AWS_ACCESS_KEY_ID`, - `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_BACKUP_ENDPOINT`, - `S3_BACKUP_BUCKET`, `S3_BACKUP_PREFIX`. Set `UKIP_BACKUP_ENVIRONMENT` if it - is not `production`; it must match the backend's, or every document is - rejected as `other_environment`. +**Where the backups are.** Dokploy writes each scope under its own top-level +folder (production, checked 2026-09-29): database dumps under the database +service's folder (`ukip-dbukip-eqmmhw/pg/`), volume archives under the compose +app's (`ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/`). +No single prefix reaches both, so the recorder takes one per scope. The probe's +`S3_BACKUP_PREFIX` covers the database folder only. + +1. Create a **read-only** storage credential: `ListBucket` limited to the two + folders and `GetObject` on them, nothing else. It is separate from the + probe's list-only credential. As an IAM policy (bucket name from + `/etc/ukip/backup-probe.env`): + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListTheTwoBackupFolders", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::", + "Condition": {"StringLike": {"s3:prefix": [ + "ukip-dbukip-eqmmhw/pg/*", + "ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*" + ]}} + }, + { + "Sid": "ReadTheBackups", + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": [ + "arn:aws:s3:::/ukip-dbukip-eqmmhw/pg/*", + "arn:aws:s3:::/ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*" + ] + } + ] + } + ``` + +2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600): + `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, + `S3_BACKUP_ENDPOINT`, `S3_BACKUP_BUCKET`, and either `S3_BACKUP_PREFIX` + (both scopes under one folder) or, as in production, + `S3_BACKUP_PREFIX_DATABASE` and `S3_BACKUP_PREFIX_VOLUME`. Set + `UKIP_BACKUP_ENVIRONMENT` if it is not `production`; it must match the + backend's, or every document is rejected as `other_environment`. 3. Install the script and the units: ```bash diff --git a/openspec/changes/automate-backup-evidence/tasks.md b/openspec/changes/automate-backup-evidence/tasks.md index 633fb964..191e5621 100644 --- a/openspec/changes/automate-backup-evidence/tasks.md +++ b/openspec/changes/automate-backup-evidence/tasks.md @@ -61,11 +61,18 @@ Section 4 is operator work and is what actually turns the automation on. ## 4. Operator actions (not code) -- [ ] 4.0 Check how the manual cycles recorded `backup_id` in production +- [x] 4.0 Check how the manual cycles recorded `backup_id` in production (`GET /ops/backups?environment=production`): full key or relative to the prefix. Suffix matching covers both; this confirms it. + Checked 2026-09-29: full keys from the bucket root, e.g. + `ukip-dbukip-eqmmhw/pg/2026-09-22T03-00-00-217Z.sql.gz`. It also showed + the two scopes in different top-level folders (volumes under + `…-danmq1_ukip-backend/static/`), which one `S3_BACKUP_PREFIX` could + not reach: the recorder now takes `S3_BACKUP_PREFIX_DATABASE` and + `S3_BACKUP_PREFIX_VOLUME`, with the shared prefix as the fallback. + The last manually recorded cycle was 2026-09-22. - [ ] 4.1 Create the read-only credential: `ListBucket` + `GetObject` on the - backup prefix only. + two backup folders only (policy in the runbook, §5c). - [ ] 4.2 Install the recorder and its timer on the production host. - [ ] 4.3 Confirm the first automatic cycle appears in `GET /ops/backups` with operator `system:backup-recorder`, and that `backup_freshness` stays `ok` diff --git a/scripts/ukip-backup-evidence-recorder.sh b/scripts/ukip-backup-evidence-recorder.sh index 33ebca23..be8348b0 100644 --- a/scripts/ukip-backup-evidence-recorder.sh +++ b/scripts/ukip-backup-evidence-recorder.sh @@ -13,9 +13,16 @@ # Credentials come from the systemd EnvironmentFile (root-owned, mode 600), # never from this file. Required environment: # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY read-only: ListBucket + GetObject -# on the backup prefix only +# on the backup prefixes only # AWS_DEFAULT_REGION bucket region -# S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET, S3_BACKUP_PREFIX +# S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET +# S3_BACKUP_PREFIX the folder both scopes live under, or instead: +# S3_BACKUP_PREFIX_DATABASE, S3_BACKUP_PREFIX_VOLUME +# one folder per scope, each overriding the shared +# one. Dokploy writes database dumps under the +# database service's folder and volume archives +# under the compose app's, which no single prefix +# reaches. # Optional: # UKIP_EVIDENCE_OUT_DIR default /var/lib/ukip/signals/backup-evidence # UKIP_BACKUP_ENVIRONMENT default production (must match the backend's) @@ -39,7 +46,11 @@ RECORDER="ukip-backup-evidence-recorder/1" KEEP_DAYS=14 : "${S3_BACKUP_ENDPOINT:?S3_BACKUP_ENDPOINT is required}" : "${S3_BACKUP_BUCKET:?S3_BACKUP_BUCKET is required}" -: "${S3_BACKUP_PREFIX:?S3_BACKUP_PREFIX is required}" +: "${S3_BACKUP_PREFIX:=}" +DATABASE_PREFIX=${S3_BACKUP_PREFIX_DATABASE:-$S3_BACKUP_PREFIX} +VOLUME_PREFIX=${S3_BACKUP_PREFIX_VOLUME:-$S3_BACKUP_PREFIX} +: "${DATABASE_PREFIX:?S3_BACKUP_PREFIX_DATABASE (or S3_BACKUP_PREFIX) is required}" +: "${VOLUME_PREFIX:?S3_BACKUP_PREFIX_VOLUME (or S3_BACKUP_PREFIX) is required}" RUNNER=${UKIP_AWS_RUNNER:-auto} if [ "$RUNNER" = "auto" ]; then @@ -65,14 +76,14 @@ aws_cli() { fi } -# newest → "keysizelast_modified", or nothing when there is none. -# The CLI paginates the whole listing before applying --query. +# newest → "keysizelast_modified", or nothing when +# there is none. The CLI paginates the whole listing before applying --query. newest() { aws_cli s3api list-objects-v2 \ --endpoint-url "$S3_BACKUP_ENDPOINT" \ --bucket "$S3_BACKUP_BUCKET" \ - --prefix "$S3_BACKUP_PREFIX" \ - --query "reverse(sort_by(Contents[?ends_with(Key, '$1')], &LastModified))[0].[Key, Size, LastModified]" \ + --prefix "$1" \ + --query "reverse(sort_by(Contents[?ends_with(Key, '$2')], &LastModified))[0].[Key, Size, LastModified]" \ --output text } @@ -85,8 +96,8 @@ chmod 0755 "$OUT_DIR" failures=0 record() { - local scope=$1 suffix=$2 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at - if ! line=$(newest "$suffix"); then + local scope=$1 suffix=$2 prefix=$3 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at + if ! line=$(newest "$prefix" "$suffix"); then echo "scope=$scope listing_failed" >&2 return 1 fi @@ -96,7 +107,7 @@ record() { fi IFS=$'\t' read -r key size last_modified <<<"$line" - relative=${key#"$S3_BACKUP_PREFIX"} + relative=${key#"$prefix"} # The key goes into JSON by printf; refuse anything that would need escaping # rather than write a document the backend would have to guess about. case "$relative" in @@ -141,8 +152,8 @@ record() { echo "scope=$scope backup_id=$relative size=$size gzip_ok=$gzip_ok recorded" } -record database .sql.gz || failures=$((failures + 1)) -record volume .tar || failures=$((failures + 1)) +record database .sql.gz "$DATABASE_PREFIX" || failures=$((failures + 1)) +record volume .tar "$VOLUME_PREFIX" || failures=$((failures + 1)) # The backend accepts documents up to 14 days old; keep no more than that. find "$OUT_DIR" -maxdepth 1 -type f -name '*.json' -mtime +"$KEEP_DAYS" -delete