From 1af011aee494f91d2fc2f4dd1933847cef898af0 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Mon, 31 Aug 2026 02:35:14 +0200 Subject: [PATCH] feat: provision Amiary PostgreSQL and secure backups --- .github/CODEOWNERS | 5 + .github/workflows/amiary-ci.yml | 48 +++++ .github/workflows/manual-deploy.yml | 41 +++- README.md | 124 +++++++++-- bootstrap/amiary-apps.sql | 311 ++++++++++++++++++++++++++++ compose.host.yml | 9 +- compose.yml | 6 +- config/runtrace-pg_hba.conf | 10 + docs/postgresql-18-upgrade.md | 39 ++++ envs/canary/.env.db | 5 +- envs/production/.env.db | 8 +- envs/production/compose.yml | 6 +- scripts/preflight-postgres-major.sh | 43 ++++ scripts/run-runtrace-backup.sh | 18 +- scripts/test-amiary-bootstrap.sh | 154 ++++++++++++++ scripts/test-runtrace-backup.sh | 13 +- scripts/test-runtrace-tls-policy.sh | 26 ++- scripts/test-shared-restore.sh | 171 +++++++++++++++ scripts/validate-amiary-config.sh | 155 ++++++++++++++ scripts/validate-postgres-config.sh | 34 ++- scripts/verify-runtrace-restore.sh | 77 ++++++- 21 files changed, 1249 insertions(+), 54 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/amiary-ci.yml create mode 100644 bootstrap/amiary-apps.sql create mode 100644 docs/postgresql-18-upgrade.md create mode 100755 scripts/preflight-postgres-major.sh create mode 100755 scripts/test-amiary-bootstrap.sh create mode 100755 scripts/test-shared-restore.sh create mode 100755 scripts/validate-amiary-config.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..14bc64b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +* @kaanyagci + +/bootstrap/ @kaanyagci +/config/ @kaanyagci +/.github/ @kaanyagci diff --git a/.github/workflows/amiary-ci.yml b/.github/workflows/amiary-ci.yml new file mode 100644 index 0000000..6d143dd --- /dev/null +++ b/.github/workflows/amiary-ci.yml @@ -0,0 +1,48 @@ +name: Amiary PostgreSQL CI + +on: + pull_request: + push: + branches: [main] + paths: + - ".github/workflows/amiary-ci.yml" + - ".github/workflows/manual-deploy.yml" + - "bootstrap/amiary-apps.sql" + - "config/runtrace-pg_hba.conf" + - "envs/production/.env.db" + - "envs/production/compose.yml" + - "scripts/run-runtrace-backup.sh" + - "scripts/run-runtrace-backup-loop.sh" + - "scripts/test-amiary-bootstrap.sh" + - "scripts/test-runtrace-backup.sh" + - "scripts/test-shared-restore.sh" + - "scripts/preflight-postgres-major.sh" + - "scripts/validate-amiary-config.sh" + - "scripts/validate-postgres-config.sh" + - "scripts/verify-runtrace-restore.sh" + - "README.md" + +permissions: + contents: read + +concurrency: + group: amiary-postgres-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + policy-and-integration: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - name: Validate repository contracts + run: scripts/validate-postgres-config.sh + - name: Check shell scripts + run: shellcheck scripts/*.sh + - name: Exercise PostgreSQL 18 role isolation + run: scripts/test-amiary-bootstrap.sh + - name: Exercise five-database backup contract + run: scripts/test-runtrace-backup.sh + - name: Exercise ownership and ACL restore contract + run: scripts/test-shared-restore.sh diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 32befcb..2ef286c 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -19,7 +19,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.0.0 - name: Configure SSH key shell: bash @@ -30,6 +30,12 @@ jobs: printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > "${HOME}/.ssh/id_ed25519" chmod 600 "${HOME}/.ssh/id_ed25519" + - name: Trust configured SSH host key + uses: Makepad-fr/add-ssh-host-key-action@1b337540ffcece7215f38921dd2fa77ae7bdcbbd + with: + server_ip_address: ${{ secrets.DEPLOY_SSH_HOST }} + known_host_entry: ${{ secrets.DEPLOY_SSH_KNOWN_HOSTS }} + - name: Prepare deployment bundle shell: bash env: @@ -39,6 +45,8 @@ jobs: DEPLOY_VIF_DB_NAME: ${{ secrets.DEPLOY_VIF_DB_NAME }} DEPLOY_VIF_DB_USER: ${{ secrets.DEPLOY_VIF_DB_USER }} DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} + DEPLOY_STORAGEBOX_TRANSPORT_ENCRYPTION_CONFIRMED: ${{ secrets.DEPLOY_STORAGEBOX_TRANSPORT_ENCRYPTION_CONFIRMED }} + DEPLOY_STORAGEBOX_AT_REST_ENCRYPTION_CONFIRMED: ${{ secrets.DEPLOY_STORAGEBOX_AT_REST_ENCRYPTION_CONFIRMED }} run: | set -euo pipefail deploy_env="${{ inputs.environment }}" @@ -47,6 +55,10 @@ jobs: if [[ "${deploy_env}" == "production" ]]; then : "${DEPLOY_VIF_DB_NETWORK:?set DEPLOY_VIF_DB_NETWORK production environment secret}" : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" + [[ "${DEPLOY_STORAGEBOX_TRANSPORT_ENCRYPTION_CONFIRMED:-}" == true ]] \ + || { echo "Production requires an explicit authenticated encrypted Storage Box transport confirmation." >&2; exit 1; } + [[ "${DEPLOY_STORAGEBOX_AT_REST_ENCRYPTION_CONFIRMED:-}" == true ]] \ + || { echo "Production requires an explicit Storage Box encryption-at-rest confirmation." >&2; exit 1; } DEPLOY_VIF_DB_NAME="${DEPLOY_VIF_DB_NAME:-vif}" DEPLOY_VIF_DB_USER="${DEPLOY_VIF_DB_USER:-vif}" fi @@ -56,6 +68,7 @@ jobs: cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh" cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh" + cp scripts/preflight-postgres-major.sh "${bundle_root}/scripts/preflight-postgres-major.sh" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" < 'amiary_migrator' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_api_prod, amiary_worker_prod, amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE amiary TO amiary_migrator, amiary_api_prod, amiary_worker_prod; +ALTER DATABASE amiary SET timezone TO 'UTC'; + +SELECT 'CREATE DATABASE amiary_canary OWNER amiary_canary_migrator' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'amiary_canary') \gexec +SELECT 'ALTER DATABASE amiary_canary OWNER TO amiary_canary_migrator' +WHERE EXISTS ( + SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'amiary_canary' AND r.rolname <> 'amiary_canary_migrator' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_migrator, amiary_api_prod, amiary_worker_prod, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE amiary_canary TO amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +ALTER DATABASE amiary_canary SET timezone TO 'UTC'; + +-- Preserve the dedicated Keycloak role/database pair. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_amiary_app') THEN + CREATE ROLE keycloak_amiary_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; +END; +$$; +ALTER ROLE keycloak_amiary_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS PASSWORD :'keycloak_amiary_app_password'; +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM keycloak_amiary_app; +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM keycloak_amiary_app; +SELECT 'CREATE DATABASE keycloak_amiary OWNER keycloak_amiary_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_amiary') \gexec +SELECT 'ALTER DATABASE keycloak_amiary OWNER TO keycloak_amiary_app' +WHERE EXISTS ( + SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'keycloak_amiary' AND r.rolname <> 'keycloak_amiary_app' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE keycloak_amiary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE keycloak_amiary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_migrator, amiary_api_prod, amiary_worker_prod, amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE keycloak_amiary TO keycloak_amiary_app; +ALTER DATABASE keycloak_amiary SET timezone TO 'UTC'; + +-- The scheduled backup login never receives superuser or write privileges. It +-- connects only to the five protected databases and pg_dump explicitly uses +-- SET ROLE makepad_backup_reader. That NOLOGIN role has read-only data access +-- and BYPASSRLS (required for a complete forced-RLS backup); the login is +-- NOINHERIT and has neither capability at all other times. +SELECT format('REVOKE ALL PRIVILEGES ON DATABASE %I FROM makepad_backup', database.datname) +FROM pg_database database +WHERE database.datname IN ('runtrace', 'keycloak_runtrace', 'amiary', 'amiary_canary', 'keycloak_amiary') +ORDER BY database.datname +\gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO makepad_backup', database.datname) +FROM pg_database database +WHERE database.datname IN ('runtrace', 'keycloak_runtrace', 'amiary', 'amiary_canary', 'keycloak_amiary') +ORDER BY database.datname +\gexec + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('amiary-bootstrap')); diff --git a/compose.host.yml b/compose.host.yml index 6ab3ba7..153dcac 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -1,11 +1,12 @@ services: postgres: - image: ${POSTGRES_IMAGE:-postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777} + image: ${POSTGRES_IMAGE:-postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2} network_mode: host environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_DB: ${POSTGRES_DB:-postgres} POSTGRES_PASSWORD_FILE: /run/secrets/postgres_superuser_password + PGDATA: /var/lib/postgresql/data command: - postgres - -c @@ -37,7 +38,7 @@ services: max-file: "${MAKEPAD_POSTGRES_LOG_MAX_FILES:-5}" runtrace_backup: - image: ${POSTGRES_IMAGE:-postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777} + image: ${POSTGRES_IMAGE:-postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2} user: "70:70" network_mode: host command: @@ -45,10 +46,10 @@ services: environment: PGHOST: 127.0.0.1 PGPORT: "5432" - PGUSER: ${POSTGRES_USER:-postgres} + PGUSER: makepad_backup PGSSLMODE: verify-full PGSSLROOTCERT: /etc/postgresql/ca.crt - POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password RUNTRACE_BACKUP_ROOT: /backups RUNTRACE_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS:-21600} RUNTRACE_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETRY_SECONDS:-300} diff --git a/compose.yml b/compose.yml index 2cc8077..c5d09f8 100644 --- a/compose.yml +++ b/compose.yml @@ -1,6 +1,10 @@ services: postgres: - image: ${POSTGRES_IMAGE:-postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777} + image: ${POSTGRES_IMAGE:-postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2} + environment: + # Keep the data path explicit across the PostgreSQL 18 official-image + # layout change. A major-version preflight prevents mounting PG16 data. + PGDATA: /var/lib/postgresql/data command: - postgres - -c diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 55be5a8..503948c 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -3,6 +3,16 @@ local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject +hostnossl amiary all all reject +hostnossl amiary_canary all all reject +hostnossl keycloak_amiary all all reject hostssl runtrace all all scram-sha-256 hostssl keycloak_runtrace all all scram-sha-256 +hostssl amiary all all scram-sha-256 +hostssl amiary_canary all all scram-sha-256 +hostssl keycloak_amiary all all scram-sha-256 +# After their exact database rules above, Amiary identities are denied from +# every unrelated database even where that database still grants PUBLIC +# CONNECT. This is deliberately before the shared compatibility rule. +host all amiary_migrator,amiary_api_prod,amiary_worker_prod,amiary_canary_migrator,amiary_api_canary,amiary_worker_canary,keycloak_amiary_app,makepad_backup all reject host all all all scram-sha-256 diff --git a/docs/postgresql-18-upgrade.md b/docs/postgresql-18-upgrade.md new file mode 100644 index 0000000..5d8be5c --- /dev/null +++ b/docs/postgresql-18-upgrade.md @@ -0,0 +1,39 @@ +# PostgreSQL 18 major-version upgrade + +The repository is pinned to PostgreSQL 18. `scripts/preflight-postgres-major.sh` +refuses to start that image against a data directory initialized by an older +major version. A normal deploy is not an upgrade mechanism. + +## Required gate + +Schedule a maintenance window and assign an operator plus reviewer. Before +stopping writers, require all of the following: + +- a fresh certificate-verified logical backup of every database, including + `amiary`, `amiary_canary`, and `keycloak_amiary`; +- verified checksums and `pg_restore --list` for every custom-format archive; +- a successful isolated restore drill using the PostgreSQL 18 client/server; +- a separately protected globals export and an inventory of extensions; +- measured rollback time within the four-hour RTO; and +- a filesystem-level snapshot of the PostgreSQL 16 data directory retained + unchanged until the PostgreSQL 18 validation window closes. + +## Cutover outline + +1. Disable application writes and stop all database clients and backup jobs. +2. Take and verify a final logical backup and globals export over + certificate-verified TLS. Treat globals and dumps as secrets. +3. Move the PostgreSQL 16 data directory to a narrow, timestamped rollback + path. Never overwrite or delete it during the cutover. +4. Create a new empty directory at the configured data path with the existing + PostgreSQL uid/gid and mode, then run the major-version preflight. +5. Start the pinned PostgreSQL 18 image, restore reviewed globals and all + database archives, and re-run every idempotent application bootstrap. +6. Run ownership, grants, RLS isolation, migration checksum, TLS/HBA, backup, + application smoke, and latency checks before enabling writers. +7. Re-enable clients gradually. Roll back to the untouched PostgreSQL 16 + directory if any acceptance gate fails. + +Record image digests, backup identifiers, checksums, commands, timings, +reviewer approval, and restore evidence. Do not delete the PostgreSQL 16 +rollback snapshot until the retention decision is explicitly reviewed. diff --git a/envs/canary/.env.db b/envs/canary/.env.db index 43296e7..3008079 100644 --- a/envs/canary/.env.db +++ b/envs/canary/.env.db @@ -1,9 +1,10 @@ # Canary PostgreSQL settings -POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +POSTGRES_IMAGE=postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2 +MAKEPAD_POSTGRES_EXPECTED_DATA_MAJOR=18 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres-canary MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-canary-superuser-password MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v1 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v1 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v4 diff --git a/envs/production/.env.db b/envs/production/.env.db index fbe71c5..7e97a93 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -1,14 +1,16 @@ # Production PostgreSQL settings -POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +POSTGRES_IMAGE=postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2 +MAKEPAD_POSTGRES_EXPECTED_DATA_MAJOR=18 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-superuser-password MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_tls_cert_v1 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v1 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v4 MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt -MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=/var/lib/makepad/postgres-backups/runtrace +MAKEPAD_POSTGRES_STORAGEBOX_MOUNT=/mnt/makepad-storagebox +MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=/mnt/makepad-storagebox/postgres MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETRY_SECONDS=300 diff --git a/envs/production/compose.yml b/envs/production/compose.yml index 76cdbe4..5278593 100644 --- a/envs/production/compose.yml +++ b/envs/production/compose.yml @@ -45,17 +45,17 @@ services: memory: ${MAKEPAD_POSTGRES_MEMORY_RESERVATION:-512M} runtrace_backup: - image: ${POSTGRES_IMAGE:-postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777} + image: ${POSTGRES_IMAGE:-postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2} user: "70:70" command: - /usr/local/bin/run-runtrace-backup-loop.sh environment: PGHOST: makepad-postgres PGPORT: "5432" - PGUSER: ${POSTGRES_USER:-postgres} + PGUSER: makepad_backup PGSSLMODE: verify-full PGSSLROOTCERT: /etc/postgresql/ca.crt - POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password RUNTRACE_BACKUP_ROOT: /backups RUNTRACE_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS:-21600} RUNTRACE_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETRY_SECONDS:-300} diff --git a/scripts/preflight-postgres-major.sh b/scripts/preflight-postgres-major.sh new file mode 100755 index 0000000..3b24812 --- /dev/null +++ b/scripts/preflight-postgres-major.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +data_path=${1:?Usage: preflight-postgres-major.sh [expected-major]} +expected_major=${2:-18} + +if [[ "${data_path}" != /* || "${data_path}" == / || "${data_path}" == /var || "${data_path}" == /var/lib ]]; then + echo "PostgreSQL data path must be a narrow absolute path." >&2 + exit 1 +fi +if [[ ! "${expected_major}" =~ ^[0-9]+$ ]]; then + echo "Expected PostgreSQL major must be numeric." >&2 + exit 1 +fi +if [[ -L "${data_path}" ]]; then + echo "PostgreSQL data path must not be a symlink: ${data_path}" >&2 + exit 1 +fi +if [[ -e "${data_path}" && ! -d "${data_path}" ]]; then + echo "PostgreSQL data path is not a directory: ${data_path}" >&2 + exit 1 +fi + +version_file=${data_path}/PG_VERSION +if [[ ! -e "${version_file}" ]]; then + # A missing directory or genuinely empty/new data directory is initialized by + # the pinned image. Non-empty unrecognized directories are never accepted. + if [[ -d "${data_path}" && -n "$(find "${data_path}" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then + echo "Non-empty PostgreSQL data path has no PG_VERSION marker: ${data_path}" >&2 + exit 1 + fi + exit 0 +fi +if [[ ! -f "${version_file}" || -L "${version_file}" ]]; then + echo "PostgreSQL PG_VERSION must be a regular non-symlink file." >&2 + exit 1 +fi + +actual_major=$(tr -d '[:space:]' < "${version_file}") +if [[ "${actual_major}" != "${expected_major}" ]]; then + echo "Refusing PostgreSQL ${expected_major} against data initialized by PostgreSQL ${actual_major}. Complete docs/postgresql-18-upgrade.md first." >&2 + exit 1 +fi diff --git a/scripts/run-runtrace-backup.sh b/scripts/run-runtrace-backup.sh index 7057083..88f9bc9 100755 --- a/scripts/run-runtrace-backup.sh +++ b/scripts/run-runtrace-backup.sh @@ -4,11 +4,11 @@ set -eu umask 077 backup_root=${RUNTRACE_BACKUP_ROOT:-/backups} -password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} +password_file=${POSTGRES_BACKUP_PASSWORD_FILE:-/run/secrets/postgres_backup_password} retention_days=${RUNTRACE_BACKUP_RETENTION_DAYS:-35} pg_host=${PGHOST:-makepad-postgres} pg_port=${PGPORT:-5432} -pg_user=${PGUSER:-postgres} +pg_user=${PGUSER:-makepad_backup} case "${retention_days}" in ''|*[!0-9]*) @@ -68,27 +68,27 @@ export PGSSLMODE="${PGSSLMODE:-verify-full}" export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" mkdir "${partial_dir}" -for database in runtrace keycloak_runtrace; do +databases='runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary' +for database in ${databases}; do dump_path=${partial_dir}/${database}.dump pg_dump \ --host="${pg_host}" \ --port="${pg_port}" \ --username="${pg_user}" \ + --role=makepad_backup_reader \ --dbname="${database}" \ --format=custom \ --compress=9 \ - --no-owner \ - --no-acl \ --file="${dump_path}" pg_restore --list "${dump_path}" >/dev/null done ( cd "${partial_dir}" - sha256sum runtrace.dump keycloak_runtrace.dump > SHA256SUMS + sha256sum runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump > SHA256SUMS ) cat > "${partial_dir}/metadata.json" < "${status_tmp}" +printf '{"createdAt":"%s","backup":"%s","databases":["runtrace","keycloak_runtrace","amiary","amiary_canary","keycloak_amiary"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" chmod 600 "${status_tmp}" mv "${status_tmp}" "${backup_root}/last-success.json" @@ -110,4 +110,4 @@ find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' - done ' sh {} + -echo "Runtrace PostgreSQL backup completed: ${final_dir}" +echo "Shared PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/test-amiary-bootstrap.sh b/scripts/test-amiary-bootstrap.sh new file mode 100755 index 0000000..3668865 --- /dev/null +++ b/scripts/test-amiary-bootstrap.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 1; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) +container="amiary-postgres-bootstrap-test-$$" + +cleanup() { + docker rm -f "${container}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run -d --rm --name "${container}" \ + -e POSTGRES_PASSWORD=test-superuser-password \ + -v "${repo_root}/bootstrap/amiary-apps.sql:/bootstrap/amiary-apps.sql:ro" \ + postgres:18@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 >/dev/null + +for _ in $(seq 1 30); do + if docker exec "${container}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "${container}" pg_isready -U postgres -d postgres >/dev/null +docker exec "${container}" createdb -U postgres runtrace +docker exec "${container}" createdb -U postgres keycloak_runtrace + +bootstrap() { + docker exec "${container}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v amiary_migrator_password=test-prod-migrator-password \ + -v amiary_api_prod_password=test-prod-api-password \ + -v amiary_worker_prod_password=test-prod-worker-password \ + -v amiary_canary_migrator_password=test-canary-migrator-password \ + -v amiary_api_canary_password=test-canary-api-password \ + -v amiary_worker_canary_password=test-canary-worker-password \ + -v keycloak_amiary_app_password=test-keycloak-password \ + -v makepad_backup_password=test-backup-password \ + -f /bootstrap/amiary-apps.sql >/dev/null +} + +bootstrap + +# Introduce representative ownership, membership, and role-attribute drift. A +# second run must converge back to the declared least-privilege state. +docker exec "${container}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null <<'SQL' +ALTER ROLE amiary_api LOGIN SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION BYPASSRLS; +ALTER ROLE amiary_api_prod SUPERUSER CREATEDB CREATEROLE NOINHERIT REPLICATION BYPASSRLS; +ALTER ROLE amiary_security_definer LOGIN SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION; +ALTER DATABASE amiary OWNER TO amiary_api_prod; +GRANT ALL PRIVILEGES ON DATABASE amiary_canary TO amiary_api_prod; +GRANT amiary_security_definer TO amiary_api_prod WITH ADMIN TRUE, INHERIT TRUE, SET TRUE; +CREATE ROLE unrelated_application_role NOLOGIN; +GRANT unrelated_application_role TO amiary_api_prod; +GRANT unrelated_application_role TO makepad_backup; +GRANT unrelated_application_role TO makepad_backup_reader; +ALTER ROLE makepad_backup SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION BYPASSRLS; +SQL + +bootstrap + +roles=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT rolname, rolcanlogin, rolsuper, rolcreatedb, rolcreaterole, rolinherit, rolreplication, rolbypassrls FROM pg_roles WHERE rolname IN ('amiary_api','amiary_worker','amiary_security_definer','amiary_migrator','amiary_api_prod','amiary_worker_prod','amiary_canary_migrator','amiary_api_canary','amiary_worker_canary','keycloak_amiary_app','makepad_backup','makepad_backup_reader') ORDER BY rolname") +expected_roles=$'amiary_api|f|f|f|f|f|f|f\namiary_api_canary|t|f|f|f|t|f|f\namiary_api_prod|t|f|f|f|t|f|f\namiary_canary_migrator|t|f|f|f|f|f|f\namiary_migrator|t|f|f|f|f|f|f\namiary_security_definer|f|f|f|f|f|f|t\namiary_worker|f|f|f|f|f|f|f\namiary_worker_canary|t|f|f|f|t|f|f\namiary_worker_prod|t|f|f|f|t|f|f\nkeycloak_amiary_app|t|f|f|f|f|f|f\nmakepad_backup|t|f|f|f|f|f|f\nmakepad_backup_reader|f|f|f|f|t|f|t' +test "${roles}" = "${expected_roles}" || { printf 'unexpected role policy:\n%s\n' "${roles}" >&2; exit 1; } + +memberships=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT granted.rolname, member.rolname, membership.admin_option, membership.inherit_option, membership.set_option FROM pg_auth_members membership JOIN pg_roles granted ON granted.oid = membership.roleid JOIN pg_roles member ON member.oid = membership.member WHERE granted.rolname IN ('amiary_api','amiary_worker','amiary_security_definer','pg_read_all_data','makepad_backup_reader') AND member.rolname IN ('amiary_api_prod','amiary_api_canary','amiary_worker_prod','amiary_worker_canary','amiary_migrator','amiary_canary_migrator','makepad_backup','makepad_backup_reader') ORDER BY granted.rolname, member.rolname") +expected_memberships=$'amiary_api|amiary_api_canary|f|t|f\namiary_api|amiary_api_prod|f|t|f\namiary_security_definer|amiary_canary_migrator|f|f|t\namiary_security_definer|amiary_migrator|f|f|t\namiary_worker|amiary_worker_canary|f|t|f\namiary_worker|amiary_worker_prod|f|t|f\nmakepad_backup_reader|makepad_backup|f|f|t\npg_read_all_data|makepad_backup_reader|f|t|f' +test "${memberships}" = "${expected_memberships}" || { printf 'unexpected role membership policy:\n%s\n' "${memberships}" >&2; exit 1; } +undeclared_memberships=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_auth_members membership JOIN pg_roles member ON member.oid = membership.member WHERE member.rolname IN ('amiary_migrator','amiary_api_prod','amiary_worker_prod','amiary_canary_migrator','amiary_api_canary','amiary_worker_canary','keycloak_amiary_app','makepad_backup','makepad_backup_reader') AND NOT EXISTS (SELECT 1 FROM (VALUES ('amiary_api','amiary_api_prod'),('amiary_api','amiary_api_canary'),('amiary_worker','amiary_worker_prod'),('amiary_worker','amiary_worker_canary'),('amiary_security_definer','amiary_migrator'),('amiary_security_definer','amiary_canary_migrator'),('pg_read_all_data','makepad_backup_reader'),('makepad_backup_reader','makepad_backup')) allowed(granted_role,member_role) JOIN pg_roles granted ON granted.rolname = allowed.granted_role WHERE granted.oid = membership.roleid AND allowed.member_role = member.rolname)") +test "${undeclared_memberships}" = 0 || { echo "Amiary login retains an undeclared role membership" >&2; exit 1; } + +databases=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT database.datname, owner.rolname FROM pg_database database JOIN pg_roles owner ON owner.oid = database.datdba WHERE database.datname IN ('amiary','amiary_canary','keycloak_amiary') ORDER BY database.datname") +expected_databases=$'amiary|amiary_migrator\namiary_canary|amiary_canary_migrator\nkeycloak_amiary|keycloak_amiary_app' +test "${databases}" = "${expected_databases}" || { printf 'unexpected database ownership:\n%s\n' "${databases}" >&2; exit 1; } + +runtime_database_privileges=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "WITH expected(role_name, database_name, may_connect) AS (VALUES ('amiary_api_prod','amiary',true), ('amiary_api_prod','amiary_canary',false), ('amiary_worker_prod','amiary',true), ('amiary_worker_prod','amiary_canary',false), ('amiary_api_canary','amiary',false), ('amiary_api_canary','amiary_canary',true), ('amiary_worker_canary','amiary',false), ('amiary_worker_canary','amiary_canary',true)) SELECT count(*) FROM expected WHERE has_database_privilege(role_name, database_name, 'CONNECT') IS DISTINCT FROM may_connect OR has_database_privilege(role_name, database_name, 'CREATE') OR has_database_privilege(role_name, database_name, 'TEMPORARY')") +test "${runtime_database_privileges}" = 0 || { echo "runtime database privileges are too broad" >&2; exit 1; } + +shared_role_connect=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM (VALUES ('amiary_api'), ('amiary_worker'), ('amiary_security_definer')) AS roles(role_name) CROSS JOIN (VALUES ('amiary'), ('amiary_canary'), ('keycloak_amiary')) AS databases(database_name) WHERE has_database_privilege(role_name, database_name, 'CONNECT')") +test "${shared_role_connect}" = 0 || { echo "NOLOGIN capability role unexpectedly has database CONNECT" >&2; exit 1; } + +connect_as() { + local role=$1 + local password=$2 + local database=$3 + docker exec -e PGPASSWORD="${password}" "${container}" \ + psql -h 127.0.0.1 -At -v ON_ERROR_STOP=1 -U "${role}" -d "${database}" -c 'SELECT current_user' 2>/dev/null +} + +test "$(connect_as amiary_api_prod test-prod-api-password amiary)" = amiary_api_prod +test "$(connect_as amiary_worker_prod test-prod-worker-password amiary)" = amiary_worker_prod +test "$(connect_as amiary_api_canary test-canary-api-password amiary_canary)" = amiary_api_canary +test "$(connect_as amiary_worker_canary test-canary-worker-password amiary_canary)" = amiary_worker_canary +test "$(connect_as keycloak_amiary_app test-keycloak-password keycloak_amiary)" = keycloak_amiary_app +test "$(connect_as makepad_backup test-backup-password amiary)" = makepad_backup + +docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'SET ROLE makepad_backup_reader; SELECT current_role' >/dev/null +if docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'CREATE TABLE public.backup_privilege_escape (id bigint)' >/dev/null 2>&1; then + echo "backup login can create schema objects" >&2 + exit 1 +fi +if docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'ALTER ROLE makepad_backup SUPERUSER' >/dev/null 2>&1; then + echo "backup login can alter roles" >&2 + exit 1 +fi + +if connect_as amiary_api_prod test-prod-api-password amiary_canary >/dev/null; then + echo "production API login connected to canary" >&2 + exit 1 +fi +if connect_as amiary_worker_canary test-canary-worker-password amiary >/dev/null; then + echo "canary worker login connected to production" >&2 + exit 1 +fi +if connect_as keycloak_amiary_app test-keycloak-password amiary >/dev/null; then + echo "Keycloak login connected to the Amiary application database" >&2 + exit 1 +fi + +if docker exec -e PGPASSWORD=test-prod-api-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_api_prod -d amiary \ + -c 'SET ROLE amiary_security_definer' >/dev/null 2>&1; then + echo "API login can SET ROLE to the RLS-bypass owner" >&2 + exit 1 +fi + +docker exec -e PGPASSWORD=test-prod-migrator-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_migrator -d amiary \ + -c 'SET ROLE amiary_security_definer' >/dev/null + +if docker exec -e PGPASSWORD=test-prod-api-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_api_prod -d amiary \ + -c 'CREATE TABLE public.privilege_escape_test (id bigint)' >/dev/null 2>&1; then + echo "API login can create schema objects" >&2 + exit 1 +fi + +docker exec -e PGPASSWORD=test-prod-migrator-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_migrator -d amiary \ + -c 'CREATE TABLE public.migration_permission_test (id bigint); DROP TABLE public.migration_permission_test' >/dev/null + +echo "Amiary PostgreSQL bootstrap is idempotent, drift-repairing, and least-privilege." diff --git a/scripts/test-runtrace-backup.sh b/scripts/test-runtrace-backup.sh index 5f5d0ca..103dac9 100755 --- a/scripts/test-runtrace-backup.sh +++ b/scripts/test-runtrace-backup.sh @@ -19,13 +19,16 @@ cat > "${work_dir}/bin/pg_dump" <<'EOF' set -eu output= database= +role= for argument in "$@"; do case "${argument}" in --file=*) output=${argument#--file=} ;; --dbname=*) database=${argument#--dbname=} ;; + --role=*) role=${argument#--role=} ;; esac done [ -n "${output}" ] && [ -n "${database}" ] +[ "${role}" = makepad_backup_reader ] printf 'validated dump for %s\n' "${database}" > "${output}" EOF cat > "${work_dir}/bin/pg_restore" <<'EOF' @@ -38,20 +41,24 @@ chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" PATH="${work_dir}/bin:${PATH}" \ RUNTRACE_BACKUP_ROOT="${work_dir}/backups" \ -POSTGRES_SUPERUSER_PASSWORD_FILE="${work_dir}/password" \ +POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ +PGUSER=makepad_backup \ PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ sh "${repo_root}/scripts/run-runtrace-backup.sh" latest=$(readlink "${work_dir}/backups/latest") backup_dir=${work_dir}/backups/${latest} -for expected in runtrace.dump keycloak_runtrace.dump SHA256SUMS metadata.json; do +for expected in runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump SHA256SUMS metadata.json; do test -s "${backup_dir}/${expected}" done ( cd "${backup_dir}" - sha256sum --check SHA256SUMS >/dev/null + sha256sum -c SHA256SUMS >/dev/null ) test -s "${work_dir}/backups/last-success.json" +grep -q '"amiary"' "${backup_dir}/metadata.json" +grep -q '"amiary_canary"' "${backup_dir}/metadata.json" +grep -q '"keycloak_amiary"' "${backup_dir}/metadata.json" RUNTRACE_BACKUP_ROOT="${work_dir}/backups" \ RUNTRACE_BACKUP_INTERVAL_SECONDS=300 \ diff --git a/scripts/test-runtrace-tls-policy.sh b/scripts/test-runtrace-tls-policy.sh index fe63821..1e06a61 100755 --- a/scripts/test-runtrace-tls-policy.sh +++ b/scripts/test-runtrace-tls-policy.sh @@ -10,7 +10,7 @@ done script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) repo_root=$(cd "${script_dir}/.." && pwd) -postgres_image=${POSTGRES_IMAGE:-postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777} +postgres_image=${POSTGRES_IMAGE:-postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2} suffix="${RANDOM}-$$" container_name="runtrace-postgres-tls-${suffix}" network_name="runtrace-postgres-tls-${suffix}" @@ -94,6 +94,9 @@ if [[ "${ready}" != "true" ]]; then exit 1 fi +docker exec "${container_name}" psql -v ON_ERROR_STOP=1 -U runtrace_app -d runtrace \ + -c "CREATE ROLE makepad_backup LOGIN PASSWORD 'backup-${password}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; GRANT CONNECT ON DATABASE runtrace TO makepad_backup" >/dev/null + tls_result=$(docker run --rm --network "${network_name}" \ -e "PGPASSWORD=${password}" \ -v "${temp_dir}/ca.crt:/run/test-ca.crt:ro" \ @@ -114,6 +117,27 @@ if docker run --rm --network "${network_name}" \ exit 1 fi +backup_tls_result=$(docker run --rm --network "${network_name}" \ + -e "PGPASSWORD=backup-${password}" \ + -v "${temp_dir}/ca.crt:/run/test-ca.crt:ro" \ + "${postgres_image}" psql \ + "host=${server_name} user=makepad_backup dbname=runtrace sslmode=verify-full sslrootcert=/run/test-ca.crt" \ + -Atc "select current_user") +if [[ "${backup_tls_result}" != "makepad_backup" ]]; then + echo "Dedicated backup login could not reach an approved database over verified TLS." >&2 + exit 1 +fi + +if docker run --rm --network "${network_name}" \ + -e "PGPASSWORD=backup-${password}" \ + -v "${temp_dir}/ca.crt:/run/test-ca.crt:ro" \ + "${postgres_image}" psql \ + "host=${server_name} user=makepad_backup dbname=postgres sslmode=verify-full sslrootcert=/run/test-ca.crt" \ + -Atc "select 1" >/dev/null 2>&1; then + echo "Dedicated backup login reached an unrelated database." >&2 + exit 1 +fi + legacy_result=$(docker run --rm --network "${network_name}" \ -e "PGPASSWORD=${password}" \ "${postgres_image}" psql \ diff --git a/scripts/test-shared-restore.sh b/scripts/test-shared-restore.sh new file mode 100755 index 0000000..a48d3e8 --- /dev/null +++ b/scripts/test-shared-restore.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 1; } +command -v sha256sum >/dev/null 2>&1 || { echo "sha256sum is required" >&2; exit 1; } + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +suffix=$$ +network="amiary-pg-restore-${suffix}" +server="amiary-pg-restore-${suffix}" +postgres_image='postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2' +password='disposable-restore-password' + +cleanup() { + docker rm -f "${server}" >/dev/null 2>&1 || true + docker network rm "${network}" >/dev/null 2>&1 || true + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +docker network create "${network}" >/dev/null +docker run -d --rm --name "${server}" --network "${network}" --network-alias postgres-restore \ + -e POSTGRES_PASSWORD="${password}" "${postgres_image}" >/dev/null +for _ in $(seq 1 30); do + if docker exec "${server}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "${server}" pg_isready -U postgres -d postgres >/dev/null + +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null <<'SQL' +CREATE ROLE runtrace_app NOLOGIN; +CREATE ROLE keycloak_runtrace_app NOLOGIN; +CREATE ROLE amiary_migrator NOLOGIN; +CREATE ROLE amiary_canary_migrator NOLOGIN; +CREATE ROLE amiary_security_definer NOLOGIN BYPASSRLS; +CREATE ROLE amiary_api NOLOGIN; +CREATE ROLE amiary_worker NOLOGIN; +CREATE ROLE keycloak_amiary_app NOLOGIN; +CREATE ROLE makepad_backup LOGIN PASSWORD 'disposable-backup-password' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +CREATE ROLE makepad_backup_reader NOLOGIN + NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS; +GRANT pg_read_all_data TO makepad_backup_reader WITH ADMIN FALSE, INHERIT TRUE, SET FALSE; +GRANT makepad_backup_reader TO makepad_backup WITH ADMIN FALSE, INHERIT FALSE, SET TRUE; +SQL + +for database in runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary; do + docker exec "${server}" createdb -U postgres "${database}" +done +for database in runtrace_restore_test keycloak_runtrace_restore_test amiary_restore_test amiary_canary_restore_test keycloak_amiary_restore_test; do + docker exec "${server}" createdb -U postgres "${database}" +done + +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d runtrace >/dev/null <<'SQL' +CREATE TABLE public.runtrace_state (id bigint PRIMARY KEY); +ALTER TABLE public.runtrace_state OWNER TO runtrace_app; +SQL +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d keycloak_runtrace >/dev/null <<'SQL' +CREATE TABLE public.realm (id text PRIMARY KEY); +ALTER TABLE public.realm OWNER TO keycloak_runtrace_app; +SQL +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d keycloak_amiary >/dev/null <<'SQL' +CREATE TABLE public.realm (id text PRIMARY KEY); +ALTER TABLE public.realm OWNER TO keycloak_amiary_app; +SQL + +for database in amiary amiary_canary; do + docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d "${database}" >/dev/null <<'SQL' +CREATE SCHEMA amiary AUTHORIZATION amiary_migrator; +CREATE TABLE amiary.persons (account_id uuid NOT NULL, id uuid PRIMARY KEY); +ALTER TABLE amiary.persons OWNER TO amiary_migrator; +ALTER TABLE amiary.persons ENABLE ROW LEVEL SECURITY; +ALTER TABLE amiary.persons FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON amiary.persons USING (true) WITH CHECK (true); +GRANT USAGE ON SCHEMA amiary TO amiary_api, amiary_worker, amiary_security_definer; +GRANT SELECT, INSERT, UPDATE, DELETE ON amiary.persons TO amiary_api; + +CREATE FUNCTION amiary.resolve_carddav_credential(text) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.claim_graph_events(integer) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +ALTER FUNCTION amiary.resolve_carddav_credential(text) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.claim_graph_events(integer) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) OWNER TO amiary_security_definer; +REVOKE ALL ON FUNCTION amiary.resolve_carddav_credential(text) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.claim_graph_events(integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION amiary.resolve_carddav_credential(text) TO amiary_api; +GRANT EXECUTE ON FUNCTION amiary.claim_graph_events(integer) TO amiary_worker; +GRANT EXECUTE ON FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) TO amiary_worker; +GRANT EXECUTE ON FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) TO amiary_worker; +SQL +done + +for database in runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary; do + docker exec "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -c "GRANT CONNECT ON DATABASE ${database} TO makepad_backup" >/dev/null + docker run --rm --user "$(id -u):$(id -g)" --network "${network}" \ + -e PGPASSWORD=disposable-backup-password -v "${work_dir}:/backups" "${postgres_image}" \ + pg_dump -h postgres-restore -U makepad_backup --role=makepad_backup_reader -d "${database}" -Fc -f "/backups/${database}.dump" +done + +if docker exec -e PGPASSWORD=disposable-backup-password "${server}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c "INSERT INTO amiary.persons(account_id,id) VALUES ('00000000-0000-4000-8000-000000000001','00000000-0000-4000-8000-000000000002')" >/dev/null 2>&1; then + echo "backup login unexpectedly wrote application data" >&2 + exit 1 +fi +( + cd "${work_dir}" + sha256sum runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump > SHA256SUMS +) +printf '{"fixture":"disposable","databases":["runtrace","keycloak_runtrace","amiary","amiary_canary","keycloak_amiary"]}\n' > "${work_dir}/metadata.json" + +cat > "${work_dir}/pg_service.conf" <&2; exit 1; } +done + +common_roles=(amiary_api amiary_worker) +for role in "${common_roles[@]}"; do + grep -Fq "CREATE ROLE ${role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS" "${sql}" \ + || { echo "missing constrained NOLOGIN capability role: ${role}" >&2; exit 1; } + grep -Fq "ALTER ROLE ${role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair capability-role drift: ${role}" >&2; exit 1; } +done + +grep -Fq "CREATE ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "missing constrained RLS-bypass definer role" >&2; exit 1; } +grep -Fq "ALTER ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair security-definer role drift" >&2; exit 1; } +grep -Fq "CREATE ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "missing constrained backup reader role" >&2; exit 1; } +grep -Fq "ALTER ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair backup reader role drift" >&2; exit 1; } + +login_roles=( + amiary_migrator + amiary_api_prod + amiary_worker_prod + amiary_canary_migrator + amiary_api_canary + amiary_worker_canary + keycloak_amiary_app + makepad_backup +) +password_variables=( + amiary_migrator_password + amiary_api_prod_password + amiary_worker_prod_password + amiary_canary_migrator_password + amiary_api_canary_password + amiary_worker_canary_password + keycloak_amiary_app_password + makepad_backup_password +) + +for index in "${!login_roles[@]}"; do + role=${login_roles[${index}]} + variable=${password_variables[${index}]} + grep -Fq "CREATE ROLE ${role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE" "${sql}" \ + || { echo "missing constrained login role: ${role}" >&2; exit 1; } + grep -Fq "ALTER ROLE ${role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE" "${sql}" \ + || { echo "bootstrap does not repair login-role drift: ${role}" >&2; exit 1; } + grep -Fq "PASSWORD :'${variable}'" "${sql}" \ + || { echo "role password is not supplied by psql variable: ${role}" >&2; exit 1; } + grep -Fq "NULLIF(btrim(:'${variable}'), '')" "${sql}" \ + || { echo "bootstrap does not reject an empty password: ${variable}" >&2; exit 1; } +done + +membership_contracts=( + "('amiary_api', 'amiary_api_prod', true, false)" + "('amiary_api', 'amiary_api_canary', true, false)" + "('amiary_worker', 'amiary_worker_prod', true, false)" + "('amiary_worker', 'amiary_worker_canary', true, false)" + "('amiary_security_definer', 'amiary_migrator', false, true)" + "('amiary_security_definer', 'amiary_canary_migrator', false, true)" + "('pg_read_all_data', 'makepad_backup_reader', true, false)" + "('makepad_backup_reader', 'makepad_backup', false, true)" +) +for contract in "${membership_contracts[@]}"; do + grep -Fq "${contract}" "${sql}" || { echo "missing membership contract: ${contract}" >&2; exit 1; } +done + +grep -Fq "'GRANT %I TO %I WITH ADMIN FALSE, INHERIT %s, SET %s'" "${sql}" +grep -Fq "SELECT format('REVOKE %I FROM %I', granted.rolname, member.rolname)" "${sql}" +grep -Fq "WHERE member.rolname IN (" "${sql}" + +for database in amiary amiary_canary keycloak_amiary; do + grep -Eq "^hostnossl[[:space:]]+${database}[[:space:]]+all[[:space:]]+all[[:space:]]+reject$" "${hba}" \ + || { echo "missing plaintext rejection for ${database}" >&2; exit 1; } + grep -Eq "^hostssl[[:space:]]+${database}[[:space:]]+all[[:space:]]+all[[:space:]]+scram-sha-256$" "${hba}" \ + || { echo "missing TLS/SCRAM rule for ${database}" >&2; exit 1; } + grep -Fq "REVOKE ALL PRIVILEGES ON DATABASE ${database} FROM PUBLIC;" "${sql}" \ + || { echo "missing public database privilege revocation for ${database}" >&2; exit 1; } +done +grep -Eq '^host[[:space:]]+all[[:space:]]+amiary_migrator,amiary_api_prod,amiary_worker_prod,amiary_canary_migrator,amiary_api_canary,amiary_worker_canary,keycloak_amiary_app,makepad_backup[[:space:]]+all[[:space:]]+reject$' "${hba}" \ + || { echo "missing cross-database HBA rejection for Amiary login roles" >&2; exit 1; } + +grep -Fq "CREATE DATABASE amiary OWNER amiary_migrator" "${sql}" +grep -Fq "ALTER DATABASE amiary OWNER TO amiary_migrator" "${sql}" +grep -Fq "GRANT CONNECT ON DATABASE amiary TO amiary_migrator, amiary_api_prod, amiary_worker_prod;" "${sql}" +grep -Fq "CREATE DATABASE amiary_canary OWNER amiary_canary_migrator" "${sql}" +grep -Fq "ALTER DATABASE amiary_canary OWNER TO amiary_canary_migrator" "${sql}" +grep -Fq "GRANT CONNECT ON DATABASE amiary_canary TO amiary_canary_migrator, amiary_api_canary, amiary_worker_canary;" "${sql}" +grep -Fq "CREATE DATABASE keycloak_amiary OWNER keycloak_amiary_app" "${sql}" +grep -Fq "ALTER DATABASE keycloak_amiary OWNER TO keycloak_amiary_app" "${sql}" + +if grep -Eq "(CREATE|ALTER) DATABASE (amiary|amiary_canary) OWNER (amiary_api_|amiary_worker_)" "${sql}"; then + echo "API or worker role must not own an Amiary database" >&2 + exit 1 +fi + +grep -Fq "pg_advisory_lock" "${sql}" +grep -Fq "pg_advisory_unlock" "${sql}" +grep -Fq "bootstrap/amiary-apps.sql" "${readme}" + +readme_roles=( + amiary_migrator + amiary_api_prod + amiary_worker_prod + amiary_canary_migrator + amiary_api_canary + amiary_worker_canary + amiary_api + amiary_worker + amiary_security_definer + keycloak_amiary_app + makepad_backup + makepad_backup_reader +) +for role in "${readme_roles[@]}"; do + grep -Fq "${role}" "${readme}" || { echo "README does not document role: ${role}" >&2; exit 1; } +done + +environment_variables=( + AMIARY_MIGRATOR_DB_PASSWORD + AMIARY_API_DB_PASSWORD + AMIARY_WORKER_DB_PASSWORD + AMIARY_CANARY_MIGRATOR_DB_PASSWORD + AMIARY_CANARY_API_DB_PASSWORD + AMIARY_CANARY_WORKER_DB_PASSWORD + KEYCLOAK_AMIARY_DB_PASSWORD + MAKEPAD_BACKUP_DB_PASSWORD +) +for variable in "${environment_variables[@]}"; do + grep -Fq "\${${variable}:?" "${readme}" || { echo "README does not fail fast for ${variable}" >&2; exit 1; } +done + +for role in amiary_migrator amiary_api_prod amiary_worker_prod amiary_canary_migrator amiary_api_canary amiary_worker_canary keycloak_amiary_app; do + grep -Fq "postgres://${role}:@:5432/" "${readme}" \ + || { echo "README is missing certificate-verified DSN for ${role}" >&2; exit 1; } +done +grep -Fq "sslmode=verify-full&sslrootcert=/run/secrets/database_ca" "${readme}" + +if grep -Eq "(password|secret)[[:space:]]*=[[:space:]]*['\"][^:'\"]+['\"]" "${sql}"; then + echo "possible literal secret in Amiary bootstrap" >&2 + exit 1 +fi + +echo "Amiary PostgreSQL provisioning contract is valid." diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 5f7d504..ada3c99 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -130,7 +130,8 @@ require("name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK}" in production_compose, "Produ for required in ("target: 5432", "published: 5432", "protocol: tcp", "mode: host"): require(required in production_compose, f"Production Compose must publish PostgreSQL for DB VM clients: {required}") require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy workflow must reject root SSH users.") -require("postgres:16-alpine@sha256:" in base_compose, "Base Compose must pin PostgreSQL to an immutable digest.") +require("postgres:18-alpine@sha256:" in base_compose, "Base Compose must pin PostgreSQL 18 to an immutable digest.") +require("PGDATA: /var/lib/postgresql/data" in base_compose, "Base Compose must explicitly retain the mounted PGDATA path under PostgreSQL 18.") require("pg_isready" in base_compose, "Base Compose must define a PostgreSQL healthcheck.") for required in ( "network_mode: host", @@ -143,6 +144,8 @@ for required in ( "runtrace_backup:", "PGSSLMODE: verify-full", "PGHOST: 127.0.0.1", + "PGUSER: makepad_backup", + "POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password", ): require(required in host_compose, f"Host Compose is missing production control: {required}") for required in ( @@ -159,7 +162,8 @@ for database in ("runtrace", "keycloak_runtrace"): require(re.search(rf"^hostssl\s+{database}\s+all\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must require TLS and SCRAM for {database}.") for label, content in (("canary", canary_env), ("production", production_env)): require("POSTGRES_PASSWORD=" not in content, f"{label} database environment must not contain POSTGRES_PASSWORD.") - require("POSTGRES_IMAGE=postgres:16-alpine@sha256:" in content, f"{label} database environment must pin POSTGRES_IMAGE.") + require("POSTGRES_IMAGE=postgres:18-alpine@sha256:" in content, f"{label} database environment must pin PostgreSQL 18.") + require("MAKEPAD_POSTGRES_EXPECTED_DATA_MAJOR=18" in content, f"{label} database environment must gate the data directory at PostgreSQL 18.") require("MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=" in content, f"{label} database environment must define the host password-file path.") require("MAKEPAD_POSTGRES_TLS_CERT_CONFIG=" in content, f"{label} database environment must name the TLS certificate config.") require("MAKEPAD_POSTGRES_TLS_KEY_SECRET=" in content, f"{label} database environment must name the TLS private-key secret.") @@ -171,6 +175,7 @@ for label, content in (("canary", canary_compose), ("production", production_com require("ensure_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate encrypted database overlay networks.") require("--opt encrypted" in manual_deploy, "Manual deploy must create database overlay networks with encryption.") require("postgres_root_password_file" in manual_deploy, "Manual deploy must load the PostgreSQL superuser password from the host file.") +require("preflight-postgres-major.sh" in manual_deploy, "Manual deploy must reject a PostgreSQL data-major mismatch before stack deployment.") require('docker config inspect "${postgres_tls_cert_config}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS certificate config.") require('docker secret inspect "${postgres_tls_key_secret}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS private-key secret.") require('docker config create --label "content-sha256=${hba_sha256}"' in manual_deploy, "Manual deploy must create a content-labelled Runtrace HBA config.") @@ -188,6 +193,8 @@ for required in ( "runtrace_backup:", "PGSSLMODE: verify-full", "PGSSLROOTCERT: /etc/postgresql/ca.crt", + "PGUSER: makepad_backup", + "POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password", "RUNTRACE_BACKUP_INTERVAL_SECONDS", "RUNTRACE_BACKUP_RETENTION_DAYS", "runtrace_backup_script", @@ -199,6 +206,7 @@ for required in ( require(required in production_compose, f"Production Compose is missing Runtrace backup control: {required}") for required in ( "MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=", + "MAKEPAD_POSTGRES_STORAGEBOX_MOUNT=/mnt/makepad-storagebox", "MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=", "MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=", "MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS=21600", @@ -206,22 +214,32 @@ for required in ( ): require(required in production_env, f"Production environment is missing Runtrace backup setting: {required}") for required in ( - "for database in runtrace keycloak_runtrace", + "databases='runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary'", 'PGSSLMODE="${PGSSLMODE:-verify-full}"', "pg_restore --list", - "sha256sum runtrace.dump keycloak_runtrace.dump", + "--role=makepad_backup_reader", + "sha256sum runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump", "last-success.json", ): require(required in runtrace_backup, f"Runtrace backup script is missing: {required}") +require("POSTGRES_SUPERUSER_PASSWORD_FILE" not in runtrace_backup, "Backup script must not consume the PostgreSQL superuser credential.") require("healthcheck" in runtrace_backup_loop and "interval_seconds * 2" in runtrace_backup_loop, "Runtrace backup health check must enforce freshness.") for required in ( "replace-nonproduction-restore-targets", "--single-transaction", "runtrace_state", "public.realm", - "sha256sum --check", + "amiary.persons", + "amiary_security_definer", + "relforcerowsecurity", + "AMIARY_RESTORE_SERVICE", + "AMIARY_CANARY_RESTORE_SERVICE", + "KEYCLOAK_AMIARY_RESTORE_SERVICE", + "sha256sum -c", ): require(required in runtrace_restore, f"Runtrace restore verifier is missing: {required}") +require("--no-owner" not in runtrace_backup and "--no-acl" not in runtrace_backup, "Backups must preserve ownership and ACL metadata.") +require("--no-owner" not in runtrace_restore and "--no-acl" not in runtrace_restore, "Restore drills must apply preserved ownership and ACL metadata.") require("run-runtrace-backup.sh" in runtrace_backup_test, "Runtrace backup contract test must execute the real backup script.") for required in ( 'cp scripts/run-runtrace-backup.sh', @@ -229,9 +247,13 @@ for required in ( "backup_directory_mode", "backup_password_mode", "postgres_ca_mode", + "mountpoint --quiet", + "canonical_storagebox", ): require(required in manual_deploy, f"Manual deploy is missing backup preflight control: {required}") require("same physical data disk is not a disaster-recovery backup" in normalized_readme, "README must require off-host Runtrace backup replication.") +require("DEPLOY_STORAGEBOX_TRANSPORT_ENCRYPTION_CONFIRMED" in manual_deploy, "Production deploy must require encrypted Storage Box transport confirmation.") +require("DEPLOY_STORAGEBOX_AT_REST_ENCRYPTION_CONFIRMED" in manual_deploy, "Production deploy must require Storage Box encryption-at-rest confirmation.") require("scripts/verify-runtrace-restore.sh" in normalized_readme, "README must document destructive non-production restore verification.") require("DEPLOY_VIF_DB_NETWORK production environment secret" in manual_deploy, "Manual deploy workflow must require VIF network secret only for production.") require('if [[ "${deploy_env}" == "production" ]]; then' in manual_deploy, "Manual deploy workflow must gate VIF setup to production.") @@ -317,3 +339,5 @@ require("openpanel_app" in normalized_readme, "README must document the OpenPane require("postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable" in readme, "README must document the OpenPanel DB VM host connection URI.") require("${OPENPANEL_DB_PASSWORD:?" in readme, "README bootstrap command must fail fast for OPENPANEL_DB_PASSWORD.") PY + +bash "${script_dir}/validate-amiary-config.sh" diff --git a/scripts/verify-runtrace-restore.sh b/scripts/verify-runtrace-restore.sh index c943415..f015290 100755 --- a/scripts/verify-runtrace-restore.sh +++ b/scripts/verify-runtrace-restore.sh @@ -5,6 +5,9 @@ backup_dir=${1:?Usage: verify-runtrace-restore.sh } : "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" : "${RUNTRACE_RESTORE_SERVICE:?RUNTRACE_RESTORE_SERVICE must name an empty non-production database}" : "${KEYCLOAK_RUNTRACE_RESTORE_SERVICE:?KEYCLOAK_RUNTRACE_RESTORE_SERVICE must name an empty non-production database}" +: "${AMIARY_RESTORE_SERVICE:?AMIARY_RESTORE_SERVICE must name an empty non-production database}" +: "${AMIARY_CANARY_RESTORE_SERVICE:?AMIARY_CANARY_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_AMIARY_RESTORE_SERVICE:?KEYCLOAK_AMIARY_RESTORE_SERVICE must name an empty non-production database}" : "${RUNTRACE_RESTORE_CONFIRM:?set RUNTRACE_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" if [[ "${RUNTRACE_RESTORE_CONFIRM}" != "replace-nonproduction-restore-targets" ]]; then @@ -20,7 +23,7 @@ if [[ ! -d "${backup_dir}" || -L "${backup_dir}" ]]; then exit 1 fi -for required in runtrace.dump keycloak_runtrace.dump SHA256SUMS metadata.json; do +for required in runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump SHA256SUMS metadata.json; do if [[ ! -s "${backup_dir}/${required}" ]]; then echo "Backup artifact is missing or empty: ${required}" >&2 exit 1 @@ -28,7 +31,7 @@ for required in runtrace.dump keycloak_runtrace.dump SHA256SUMS metadata.json; d done ( cd "${backup_dir}" - sha256sum --check SHA256SUMS + sha256sum -c SHA256SUMS ) restore_dump() { @@ -38,8 +41,6 @@ restore_dump() { --dbname="service=${service}" \ --clean \ --if-exists \ - --no-owner \ - --no-acl \ --exit-on-error \ --single-transaction \ "${dump}" @@ -47,14 +48,76 @@ restore_dump() { restore_dump "${RUNTRACE_RESTORE_SERVICE}" "${backup_dir}/runtrace.dump" restore_dump "${KEYCLOAK_RUNTRACE_RESTORE_SERVICE}" "${backup_dir}/keycloak_runtrace.dump" +restore_dump "${AMIARY_RESTORE_SERVICE}" "${backup_dir}/amiary.dump" +restore_dump "${AMIARY_CANARY_RESTORE_SERVICE}" "${backup_dir}/amiary_canary.dump" +restore_dump "${KEYCLOAK_AMIARY_RESTORE_SERVICE}" "${backup_dir}/keycloak_amiary.dump" runtrace_table=$(psql "service=${RUNTRACE_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ "SELECT to_regclass('public.runtrace_state') IS NOT NULL;") keycloak_table=$(psql "service=${KEYCLOAK_RUNTRACE_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ "SELECT to_regclass('public.realm') IS NOT NULL;") -if [[ "${runtrace_table}" != "t" || "${keycloak_table}" != "t" ]]; then - echo "Restored databases are missing the Runtrace or Keycloak durable state tables." >&2 +amiary_table=$(psql "service=${AMIARY_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('amiary.persons') IS NOT NULL;") +amiary_canary_table=$(psql "service=${AMIARY_CANARY_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('amiary.persons') IS NOT NULL;") +keycloak_amiary_table=$(psql "service=${KEYCLOAK_AMIARY_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.realm') IS NOT NULL;") +if [[ "${runtrace_table}" != "t" || "${keycloak_table}" != "t" || \ + "${amiary_table}" != "t" || "${amiary_canary_table}" != "t" || \ + "${keycloak_amiary_table}" != "t" ]]; then + echo "Restored databases are missing required Runtrace, Amiary, or Keycloak durable tables." >&2 + exit 1 +fi + +amiary_security_contract=$(psql "service=${AMIARY_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc " + SELECT count(*) = 0 + FROM ( + VALUES + ('resolve_carddav_credential', 'amiary_security_definer', true), + ('claim_graph_events', 'amiary_security_definer', true), + ('finish_graph_event', 'amiary_security_definer', true), + ('fetch_graph_projection', 'amiary_security_definer', true) + ) expected(function_name, owner_name, security_definer) + WHERE NOT EXISTS ( + SELECT 1 + FROM pg_proc function + JOIN pg_namespace namespace ON namespace.oid = function.pronamespace + JOIN pg_roles owner ON owner.oid = function.proowner + WHERE namespace.nspname = 'amiary' + AND function.proname = expected.function_name + AND owner.rolname = expected.owner_name + AND function.prosecdef = expected.security_definer + ); + SELECT count(*) = 0 + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'amiary' + AND relation.relkind IN ('r', 'p') + AND NOT relation.relispartition + AND ( + relation.relname = 'accounts' + OR EXISTS ( + SELECT 1 FROM pg_attribute attribute + WHERE attribute.attrelid = relation.oid + AND attribute.attname = 'account_id' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ) + ) + AND (NOT relation.relrowsecurity OR NOT relation.relforcerowsecurity); + SELECT + has_schema_privilege('amiary_api', 'amiary', 'USAGE') + AND has_table_privilege('amiary_api', 'amiary.persons', 'SELECT,INSERT,UPDATE,DELETE') + AND NOT has_table_privilege('amiary_api', 'amiary.persons', 'TRUNCATE') + AND has_function_privilege('amiary_api', 'amiary.resolve_carddav_credential(text)', 'EXECUTE') + AND NOT has_table_privilege('amiary_worker', 'amiary.persons', 'SELECT') + AND has_function_privilege('amiary_worker', 'amiary.claim_graph_events(integer)', 'EXECUTE') + AND has_function_privilege('amiary_worker', 'amiary.fetch_graph_projection(bigint,uuid,bigint,text,uuid)', 'EXECUTE') + AND NOT has_function_privilege('amiary_api', 'amiary.claim_graph_events(integer)', 'EXECUTE'); +") +if [[ "${amiary_security_contract}" != $'t\nt\nt' ]]; then + echo "Restored Amiary function ownership or forced-RLS contract is invalid." >&2 exit 1 fi -echo "Runtrace and Keycloak restore verification completed against non-production targets." +echo "Shared PostgreSQL restore verification completed against non-production targets."