diff --git a/README.md b/README.md index c55c46fb..185d13b6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ![Rust](https://img.shields.io/badge/Rust-gRPC_Engine-000000?logo=rust&logoColor=white) ![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white) -![Tests](https://img.shields.io/badge/Tests-4404_collected-28A745?logo=pytest&logoColor=white) +![Tests](https://img.shields.io/badge/Tests-4408_collected-28A745?logo=pytest&logoColor=white) ![API Operations](https://img.shields.io/badge/API_Operations-427-blue) @@ -79,7 +79,7 @@ backend/ FastAPI API server domains/ 3 configurable schemas: default, science, healthcare scripts/ maintenance + backfills (nif_bayes, work_type, retrospective) + job worker/scheduler entrypoints - tests/ 333 test files, 4404 tests + tests/ 333 test files, 4408 tests frontend/ Next.js 16 App Router app/ 65 pages, 90+ components, 8 context providers @@ -147,7 +147,7 @@ UKIP manages major product and implementation decisions as architecture decision | Analytics | pandas, DuckDB, PyArrow, NumPy, SciPy | | Background Jobs | Durable PostgreSQL lease queue (broker-free, at-least-once) with worker/scheduler processes | -| Testing | pytest (4404 tests collected), Vitest (312 tests), Playwright | +| Testing | pytest (4408 tests collected), Vitest (312 tests), Playwright | | Deployment | Docker Compose, GHCR images, Dokploy-oriented production compose | | Monitoring | Sentry (opt-in), structured logging | @@ -284,7 +284,7 @@ cd frontend && npx tsc --noEmit ``` -**Current test stats:** 4404 backend tests collected across 333 test files. Frontend: 312 Vitest tests. +**Current test stats:** 4408 backend tests collected across 333 test files. Frontend: 312 Vitest tests. --- diff --git a/backend/tests/test_backup_evidence_recorder.py b/backend/tests/test_backup_evidence_recorder.py index 978998c8..946fe74f 100644 --- a/backend/tests/test_backup_evidence_recorder.py +++ b/backend/tests/test_backup_evidence_recorder.py @@ -199,3 +199,80 @@ def test_no_credential_or_key_path_reaches_the_output(tmp_path): assert "AKIAEXAMPLE" not in everything assert "bucket" not in json.dumps(_documents(env)), "documents carry no bucket name" assert PREFIX not in json.dumps(_documents(env)), "documents carry no prefix" + + +# Production keeps each scope in its own top-level folder (checked 2026-09-29): +# dumps under the database service's folder, volume archives under the compose +# app's. One shared prefix cannot reach both, so each scope may have its own. +DB_PREFIX = "ukip-dbukip-eqmmhw/pg/" +VOL_PREFIX = "ukip-app-danmq1_ukip-backend/static/" +SPLIT_DB_KEY = f"{DB_PREFIX}2026-09-25T03-00-00-076Z.sql.gz" +SPLIT_VOL_KEY = f"{VOL_PREFIX}ukip_static_data-2026-09-25T03-05-00-066Z.tar" + + +def _split_setup(tmp_path): + env = _setup(tmp_path, db_bytes=None) + objects = Path(env["STUB_OBJECTS"]) + (objects / Path(SPLIT_DB_KEY).name).write_bytes(DUMP) + (objects / Path(SPLIT_VOL_KEY).name).write_bytes(ARCHIVE) + del env["S3_BACKUP_PREFIX"] + env.update({ + "S3_BACKUP_PREFIX_DATABASE": DB_PREFIX, + "S3_BACKUP_PREFIX_VOLUME": VOL_PREFIX, + "STUB_DB_LISTING": f"{SPLIT_DB_KEY}\\t{len(DUMP)}\\t{_stamp(60)}", + "STUB_VOL_LISTING": f"{SPLIT_VOL_KEY}\\t{len(ARCHIVE)}\\t{_stamp(55)}", + }) + return env + + +def _listings(env) -> list[str]: + log = Path(env["STUB_LOG"]).read_text(encoding="utf-8").splitlines() + return [line for line in log if line.startswith("s3api list-objects-v2")] + + +def test_each_scope_is_listed_under_its_own_prefix(tmp_path): + env = _split_setup(tmp_path) + + completed = _run(env) + + assert completed.returncode == 0, completed.stderr + listings = _listings(env) + db_listing = next(line for line in listings if ".sql.gz" in line) + vol_listing = next(line for line in listings if ".tar" in line) + assert f"--prefix {DB_PREFIX} " in db_listing + assert f"--prefix {VOL_PREFIX} " in vol_listing + + +def test_split_prefixes_record_both_scopes_relative_to_their_own_prefix(tmp_path): + env = _split_setup(tmp_path) + + assert _run(env).returncode == 0 + docs = _documents(env) + ids = {doc["scope"]: doc["backup_id"] for doc in docs.values()} + assert ids == { + "database": "2026-09-25T03-00-00-076Z.sql.gz", + "volume": "ukip_static_data-2026-09-25T03-05-00-066Z.tar", + } + for document in docs.values(): + assert isinstance(_parse(document), EvidenceDocument) + assert DB_PREFIX not in json.dumps(document) and VOL_PREFIX not in json.dumps(document) + + +def test_a_scope_without_any_prefix_is_refused(tmp_path): + env = _split_setup(tmp_path) + del env["S3_BACKUP_PREFIX_VOLUME"] + + completed = _run(env) + + assert completed.returncode != 0 + assert "S3_BACKUP_PREFIX_VOLUME" in completed.stderr + assert _documents(env) == {} + + +def test_the_shared_prefix_still_covers_both_scopes(tmp_path): + # Installations that keep both scopes under one folder need no change. + env = _setup(tmp_path, db_bytes=DUMP, vol_bytes=ARCHIVE) + + assert _run(env).returncode == 0 + for listing in _listings(env): + assert f"--prefix {PREFIX} " in listing diff --git a/docs/generated/repo_metrics.json b/docs/generated/repo_metrics.json index 93ae56d3..27e03bbc 100644 --- a/docs/generated/repo_metrics.json +++ b/docs/generated/repo_metrics.json @@ -1,7 +1,7 @@ { "schema_version": 1, "metrics": { - "backend_tests_collected": 4404, + "backend_tests_collected": 4408, "backend_test_files": 333, "frontend_vitest_tests": 312, "api_operations": 427, diff --git a/docs/operating/BACKUP_RESTORE_RUNBOOK.md b/docs/operating/BACKUP_RESTORE_RUNBOOK.md index 01431d0d..9c4085b1 100644 --- a/docs/operating/BACKUP_RESTORE_RUNBOOK.md +++ b/docs/operating/BACKUP_RESTORE_RUNBOOK.md @@ -316,15 +316,52 @@ involved** (owner decision on #370). The only credential is a storage one on the host, and the trust boundary is root on that host, which already holds the database. -1. Create a **read-only** storage credential with `ListBucket` and `GetObject` - on the backup prefix only: no write, no delete, nothing outside the prefix. - It is separate from the probe's list-only credential. -2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600), with the - same variables as the probe's file: `AWS_ACCESS_KEY_ID`, - `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_BACKUP_ENDPOINT`, - `S3_BACKUP_BUCKET`, `S3_BACKUP_PREFIX`. Set `UKIP_BACKUP_ENVIRONMENT` if it - is not `production`; it must match the backend's, or every document is - rejected as `other_environment`. +**Where the backups are.** Dokploy writes each scope under its own top-level +folder (production, checked 2026-09-29): database dumps under the database +service's folder (`ukip-dbukip-eqmmhw/pg/`), volume archives under the compose +app's (`ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/`). +No single prefix reaches both, so the recorder takes one per scope. The probe's +`S3_BACKUP_PREFIX` covers the database folder only. + +1. Create a **read-only** storage credential: `ListBucket` limited to the two + folders and `GetObject` on them, nothing else. It is separate from the + probe's list-only credential. As an IAM policy (bucket name from + `/etc/ukip/backup-probe.env`): + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListTheTwoBackupFolders", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::", + "Condition": {"StringLike": {"s3:prefix": [ + "ukip-dbukip-eqmmhw/pg/*", + "ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*" + ]}} + }, + { + "Sid": "ReadTheBackups", + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": [ + "arn:aws:s3:::/ukip-dbukip-eqmmhw/pg/*", + "arn:aws:s3:::/ukip-universal-knowledgeiintelligence-platform-danmq1_ukip-backend/static/*" + ] + } + ] + } + ``` + +2. Put it in `/etc/ukip/backup-evidence.env` (root-owned, mode 600): + `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, + `S3_BACKUP_ENDPOINT`, `S3_BACKUP_BUCKET`, and either `S3_BACKUP_PREFIX` + (both scopes under one folder) or, as in production, + `S3_BACKUP_PREFIX_DATABASE` and `S3_BACKUP_PREFIX_VOLUME`. Set + `UKIP_BACKUP_ENVIRONMENT` if it is not `production`; it must match the + backend's, or every document is rejected as `other_environment`. 3. Install the script and the units: ```bash diff --git a/openspec/changes/automate-backup-evidence/tasks.md b/openspec/changes/automate-backup-evidence/tasks.md index 633fb964..191e5621 100644 --- a/openspec/changes/automate-backup-evidence/tasks.md +++ b/openspec/changes/automate-backup-evidence/tasks.md @@ -61,11 +61,18 @@ Section 4 is operator work and is what actually turns the automation on. ## 4. Operator actions (not code) -- [ ] 4.0 Check how the manual cycles recorded `backup_id` in production +- [x] 4.0 Check how the manual cycles recorded `backup_id` in production (`GET /ops/backups?environment=production`): full key or relative to the prefix. Suffix matching covers both; this confirms it. + Checked 2026-09-29: full keys from the bucket root, e.g. + `ukip-dbukip-eqmmhw/pg/2026-09-22T03-00-00-217Z.sql.gz`. It also showed + the two scopes in different top-level folders (volumes under + `…-danmq1_ukip-backend/static/`), which one `S3_BACKUP_PREFIX` could + not reach: the recorder now takes `S3_BACKUP_PREFIX_DATABASE` and + `S3_BACKUP_PREFIX_VOLUME`, with the shared prefix as the fallback. + The last manually recorded cycle was 2026-09-22. - [ ] 4.1 Create the read-only credential: `ListBucket` + `GetObject` on the - backup prefix only. + two backup folders only (policy in the runbook, §5c). - [ ] 4.2 Install the recorder and its timer on the production host. - [ ] 4.3 Confirm the first automatic cycle appears in `GET /ops/backups` with operator `system:backup-recorder`, and that `backup_freshness` stays `ok` diff --git a/scripts/ukip-backup-evidence-recorder.sh b/scripts/ukip-backup-evidence-recorder.sh index 33ebca23..be8348b0 100644 --- a/scripts/ukip-backup-evidence-recorder.sh +++ b/scripts/ukip-backup-evidence-recorder.sh @@ -13,9 +13,16 @@ # Credentials come from the systemd EnvironmentFile (root-owned, mode 600), # never from this file. Required environment: # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY read-only: ListBucket + GetObject -# on the backup prefix only +# on the backup prefixes only # AWS_DEFAULT_REGION bucket region -# S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET, S3_BACKUP_PREFIX +# S3_BACKUP_ENDPOINT, S3_BACKUP_BUCKET +# S3_BACKUP_PREFIX the folder both scopes live under, or instead: +# S3_BACKUP_PREFIX_DATABASE, S3_BACKUP_PREFIX_VOLUME +# one folder per scope, each overriding the shared +# one. Dokploy writes database dumps under the +# database service's folder and volume archives +# under the compose app's, which no single prefix +# reaches. # Optional: # UKIP_EVIDENCE_OUT_DIR default /var/lib/ukip/signals/backup-evidence # UKIP_BACKUP_ENVIRONMENT default production (must match the backend's) @@ -39,7 +46,11 @@ RECORDER="ukip-backup-evidence-recorder/1" KEEP_DAYS=14 : "${S3_BACKUP_ENDPOINT:?S3_BACKUP_ENDPOINT is required}" : "${S3_BACKUP_BUCKET:?S3_BACKUP_BUCKET is required}" -: "${S3_BACKUP_PREFIX:?S3_BACKUP_PREFIX is required}" +: "${S3_BACKUP_PREFIX:=}" +DATABASE_PREFIX=${S3_BACKUP_PREFIX_DATABASE:-$S3_BACKUP_PREFIX} +VOLUME_PREFIX=${S3_BACKUP_PREFIX_VOLUME:-$S3_BACKUP_PREFIX} +: "${DATABASE_PREFIX:?S3_BACKUP_PREFIX_DATABASE (or S3_BACKUP_PREFIX) is required}" +: "${VOLUME_PREFIX:?S3_BACKUP_PREFIX_VOLUME (or S3_BACKUP_PREFIX) is required}" RUNNER=${UKIP_AWS_RUNNER:-auto} if [ "$RUNNER" = "auto" ]; then @@ -65,14 +76,14 @@ aws_cli() { fi } -# newest → "keysizelast_modified", or nothing when there is none. -# The CLI paginates the whole listing before applying --query. +# newest → "keysizelast_modified", or nothing when +# there is none. The CLI paginates the whole listing before applying --query. newest() { aws_cli s3api list-objects-v2 \ --endpoint-url "$S3_BACKUP_ENDPOINT" \ --bucket "$S3_BACKUP_BUCKET" \ - --prefix "$S3_BACKUP_PREFIX" \ - --query "reverse(sort_by(Contents[?ends_with(Key, '$1')], &LastModified))[0].[Key, Size, LastModified]" \ + --prefix "$1" \ + --query "reverse(sort_by(Contents[?ends_with(Key, '$2')], &LastModified))[0].[Key, Size, LastModified]" \ --output text } @@ -85,8 +96,8 @@ chmod 0755 "$OUT_DIR" failures=0 record() { - local scope=$1 suffix=$2 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at - if ! line=$(newest "$suffix"); then + local scope=$1 suffix=$2 prefix=$3 line key size last_modified relative doc_name doc tmp sha actual gzip_ok observed_at + if ! line=$(newest "$prefix" "$suffix"); then echo "scope=$scope listing_failed" >&2 return 1 fi @@ -96,7 +107,7 @@ record() { fi IFS=$'\t' read -r key size last_modified <<<"$line" - relative=${key#"$S3_BACKUP_PREFIX"} + relative=${key#"$prefix"} # The key goes into JSON by printf; refuse anything that would need escaping # rather than write a document the backend would have to guess about. case "$relative" in @@ -141,8 +152,8 @@ record() { echo "scope=$scope backup_id=$relative size=$size gzip_ok=$gzip_ok recorded" } -record database .sql.gz || failures=$((failures + 1)) -record volume .tar || failures=$((failures + 1)) +record database .sql.gz "$DATABASE_PREFIX" || failures=$((failures + 1)) +record volume .tar "$VOLUME_PREFIX" || failures=$((failures + 1)) # The backend accepts documents up to 14 days old; keep no more than that. find "$OUT_DIR" -maxdepth 1 -type f -name '*.json' -mtime +"$KEEP_DAYS" -delete