From 2df98f091da76cec9d324ce316877a156d798610 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 16:39:18 +0200 Subject: [PATCH 01/20] feat: add isolated Brio staging databases --- .github/actionlint.yaml | 3 + .github/workflows/ci.yml | 37 ++++ .github/workflows/manual-deploy.yml | 213 ++++++++++++++++++-- README.md | 190 +++++++++++++++++- bootstrap/brio-staging-app.sql | 82 ++++++++ bootstrap/keycloak-brio-staging.sql | 83 ++++++++ compose.host.yml | 49 +++++ compose.yml | 4 + config/runtrace-pg_hba.conf | 10 + envs/canary/.env.db | 20 +- envs/canary/compose.yml | 74 +++++++ envs/production/.env.db | 11 +- envs/production/compose.yml | 66 ++++++ scripts/run-brio-encrypted-backup-loop.sh | 44 ++++ scripts/run-brio-encrypted-backup.sh | 233 ++++++++++++++++++++++ scripts/test-brio-bootstrap.sh | 149 ++++++++++++++ scripts/test-brio-encrypted-backup.sh | 144 +++++++++++++ scripts/test-brio-encrypted-restore.sh | 155 ++++++++++++++ scripts/validate-postgres-config.sh | 209 +++++++++++++++++++ scripts/verify-brio-encrypted-restore.sh | 192 ++++++++++++++++++ 20 files changed, 1943 insertions(+), 25 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/ci.yml create mode 100644 bootstrap/brio-staging-app.sql create mode 100644 bootstrap/keycloak-brio-staging.sql create mode 100755 scripts/run-brio-encrypted-backup-loop.sh create mode 100755 scripts/run-brio-encrypted-backup.sh create mode 100755 scripts/test-brio-bootstrap.sh create mode 100755 scripts/test-brio-encrypted-backup.sh create mode 100755 scripts/test-brio-encrypted-restore.sh create mode 100755 scripts/verify-brio-encrypted-restore.sh diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..628281c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - makepad diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0583c16 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Validate PostgreSQL deployment contract + run: ./scripts/validate-postgres-config.sh + - name: Check deployment shell scripts + run: >- + shellcheck + scripts/run-brio-encrypted-backup.sh + scripts/run-brio-encrypted-backup-loop.sh + scripts/verify-brio-encrypted-restore.sh + scripts/test-brio-bootstrap.sh + scripts/test-brio-encrypted-backup.sh + scripts/test-brio-encrypted-restore.sh + - name: Test idempotent Brio bootstraps + run: ./scripts/test-brio-bootstrap.sh + - name: Test Brio encrypted backup publication + run: ./scripts/test-brio-encrypted-backup.sh + - name: Test Brio encrypted restore safeguards + run: ./scripts/test-brio-encrypted-restore.sh + - name: Test backup contracts in the pinned runtime image + run: | + backup_image=$(grep '^BRIO_BACKUP_IMAGE=' envs/canary/.env.db | cut -d= -f2-) + docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-backup.sh + docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-restore.sh diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 32befcb..4280f1a 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -23,12 +23,19 @@ jobs: - name: Configure SSH key shell: bash + env: + DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + DEPLOY_SSH_KNOWN_HOSTS: ${{ secrets.DEPLOY_SSH_KNOWN_HOSTS }} run: | set -euo pipefail + : "${DEPLOY_SSH_PRIVATE_KEY:?set DEPLOY_SSH_PRIVATE_KEY}" + : "${DEPLOY_SSH_KNOWN_HOSTS:?set DEPLOY_SSH_KNOWN_HOSTS}" mkdir -p "${HOME}/.ssh" chmod 700 "${HOME}/.ssh" - printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > "${HOME}/.ssh/id_ed25519" + printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" > "${HOME}/.ssh/id_ed25519" chmod 600 "${HOME}/.ssh/id_ed25519" + printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${HOME}/.ssh/known_hosts" + chmod 600 "${HOME}/.ssh/known_hosts" - name: Prepare deployment bundle shell: bash @@ -39,11 +46,19 @@ jobs: DEPLOY_VIF_DB_NAME: ${{ secrets.DEPLOY_VIF_DB_NAME }} DEPLOY_VIF_DB_USER: ${{ secrets.DEPLOY_VIF_DB_USER }} DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} + DEPLOY_BRIO_STAGING_DB_NETWORK: ${{ secrets.DEPLOY_BRIO_STAGING_DB_NETWORK }} run: | set -euo pipefail deploy_env="${{ inputs.environment }}" : "${DEPLOY_CATWLK_DB_NETWORK:?set DEPLOY_CATWLK_DB_NETWORK environment secret}" : "${DEPLOY_LE_PETIT_COIN_DB_NETWORK:?set DEPLOY_LE_PETIT_COIN_DB_NETWORK environment secret}" + if [[ "${deploy_env}" == "canary" ]]; then + : "${DEPLOY_BRIO_STAGING_DB_NETWORK:?set DEPLOY_BRIO_STAGING_DB_NETWORK canary environment secret}" + if [[ "${DEPLOY_BRIO_STAGING_DB_NETWORK}" != "makepad_brio_staging_db" ]]; then + echo "DEPLOY_BRIO_STAGING_DB_NETWORK must be makepad_brio_staging_db." >&2 + exit 1 + fi + fi if [[ "${deploy_env}" == "production" ]]; then : "${DEPLOY_VIF_DB_NETWORK:?set DEPLOY_VIF_DB_NETWORK production environment secret}" : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" @@ -56,12 +71,19 @@ jobs: cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh" cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh" + cp scripts/run-brio-encrypted-backup.sh "${bundle_root}/scripts/run-brio-encrypted-backup.sh" + cp scripts/run-brio-encrypted-backup-loop.sh "${bundle_root}/scripts/run-brio-encrypted-backup-loop.sh" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <> "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <> "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <&2 + exit 1 + fi + postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") + if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then + echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 + exit 1 + fi + for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then + echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 + exit 1 + fi + done + if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then + echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 + exit 1 + fi + brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") + brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") + if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then + echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then + echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 + exit 1 + fi + brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") + brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") + if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then + echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then + echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 + exit 1 + fi + brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") + brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") + if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then + echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 + exit 1 + fi + if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then + echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 + exit 1 + fi + server_certificate=$(mktemp) + cleanup_server_certificate() { rm -f "${server_certificate}"; } + trap cleanup_server_certificate EXIT + docker config inspect "${postgres_tls_cert_config}" --format '{{printf "%s" .Spec.Data}}' > "${server_certificate}" + if ! openssl x509 -in "${server_certificate}" -noout -checkend 604800 >/dev/null; then + echo "PostgreSQL TLS certificate is invalid or expires in less than seven days." >&2 + exit 1 + fi + if ! openssl verify -purpose sslserver -CAfile "${postgres_ca_cert_file}" -untrusted "${server_certificate}" "${server_certificate}" >/dev/null; then + echo "PostgreSQL TLS certificate does not chain to the configured CA." >&2 + exit 1 + fi + if [[ "${deploy_env}" == "canary" ]] && ! openssl x509 -in "${server_certificate}" -noout -checkhost makepad-postgres-brio-staging >/dev/null; then + echo "Canary PostgreSQL TLS certificate does not cover makepad-postgres-brio-staging." >&2 + exit 1 + fi if [[ "${deploy_env}" == "production" ]]; then if [[ ! -d "${runtrace_backup_path}" || -L "${runtrace_backup_path}" ]]; then echo "Runtrace backup path must be a pre-provisioned non-symlink directory: ${runtrace_backup_path}" >&2 @@ -173,15 +282,6 @@ jobs: echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2 exit 1 fi - if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then - echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2 - exit 1 - fi - postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") - if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then - echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 - exit 1 - fi fi hba_path="${remote_dir}/config/runtrace-pg_hba.conf" hba_sha256=$(sha256sum "${hba_path}" | awk '{print $1}') @@ -200,6 +300,13 @@ jobs: : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" fi + if [[ "${brio_staging_enabled}" == "1" ]]; then + : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" + if [[ "${brio_staging_db_network}" != "makepad_brio_staging_db" ]]; then + echo "Brio deployment bundle must use makepad_brio_staging_db." >&2 + exit 1 + fi + fi ensure_encrypted_overlay_network() { local network_name=$1 @@ -217,12 +324,29 @@ jobs: docker network create --driver overlay --attachable --opt encrypted "${network_name}" >/dev/null } + ensure_internal_encrypted_overlay_network() { + local network_name=$1 + if ! docker network inspect "${network_name}" >/dev/null 2>&1; then + docker network create --driver overlay --attachable --internal --opt encrypted "${network_name}" >/dev/null + fi + local details + details=$(docker network inspect "${network_name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') + if [[ "${details}" != "overlay swarm true true true" ]]; then + echo "Brio database network ${network_name} must be an internal, encrypted, attachable Swarm overlay; got ${details}." >&2 + exit 1 + fi + } + ensure_encrypted_overlay_network "${db_network}" ensure_encrypted_overlay_network "${le_petit_coin_db_network}" if [[ "${vif_enabled}" == "1" ]]; then ensure_encrypted_overlay_network "${vif_db_network}" export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" fi + if [[ "${brio_staging_enabled}" == "1" ]]; then + ensure_internal_encrypted_overlay_network "${brio_staging_db_network}" + export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_staging_db_network}" + fi export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" @@ -235,6 +359,69 @@ jobs: docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" + wait_for_service_convergence() { + local service_name=$1 + local expected_image=$2 + local update_state desired running_snapshot running_count wrong_image + for attempt in $(seq 1 60); do + if ! docker service inspect "${service_name}" >/dev/null 2>&1; then + sleep 2 + continue + fi + update_state=$(docker service inspect "${service_name}" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{else}}none{{end}}') + case "${update_state}" in + paused|rollback_started|rollback_paused|rollback_completed) + echo "Service ${service_name} update did not complete successfully: ${update_state}." >&2 + docker service ps --no-trunc "${service_name}" >&2 + return 1 + ;; + updating) + sleep 2 + continue + ;; + esac + desired=$(docker service inspect "${service_name}" --format '{{.Spec.Mode.Replicated.Replicas}}') + running_snapshot=$(docker service ps --no-trunc --filter desired-state=running --format '{{.Image}} {{.CurrentState}}' "${service_name}") + running_count=$(printf '%s\n' "${running_snapshot}" | awk '$2 == "Running" {count++} END {print count + 0}') + wrong_image=$(printf '%s\n' "${running_snapshot}" | awk -v expected="${expected_image}" '$2 == "Running" && $1 != expected {print $1; exit}') + if [[ "${running_count}" == "${desired}" && -z "${wrong_image}" && ( "${update_state}" == "completed" || "${update_state}" == "none" ) ]]; then + return 0 + fi + sleep 2 + done + echo "Service ${service_name} did not converge to ${expected_image}." >&2 + docker service ps --no-trunc "${service_name}" >&2 || true + return 1 + } + + wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}" + if [[ "${deploy_env}" == "canary" ]]; then + wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}" + else + wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}" + fi + + if [[ "${brio_staging_enabled}" == "1" ]]; then + brio_tls_ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --network "${brio_staging_db_network}" \ + -e PGSSLMODE=verify-full \ + -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${postgres_ca_cert_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-brio-staging -U "${postgres_root_user}" -d postgres -Atc "select 1" >/dev/null 2>&1; then + brio_tls_ready=1 + break + fi + sleep 2 + done + if [[ "${brio_tls_ready}" != "1" ]]; then + echo "PostgreSQL did not pass sslmode=verify-full using makepad-postgres-brio-staging within 60 seconds." >&2 + exit 1 + fi + fi + if [[ "${vif_enabled}" != "1" ]]; then exit 0 fi diff --git a/README.md b/README.md index edac409..13a6601 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,16 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `envs/canary/.env.db`: canary PostgreSQL settings - `envs/production/compose.yml`: production Swarm overrides - `envs/production/.env.db`: production PostgreSQL settings -- `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the Vif, Makepad, Vestiaire, and Runtrace Keycloak databases +- `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the existing Vif, Makepad, Vestiaire, and Runtrace Keycloak databases - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database +- `bootstrap/brio-staging-app.sql`: idempotent SQL bootstrap for the Brio staging application database +- `bootstrap/keycloak-brio-staging.sql`: targeted idempotent bootstrap for Brio's Keycloak database - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets +- `scripts/run-brio-encrypted-backup.sh`: streaming CMS-encrypted backup for one allowlisted Brio database +- `scripts/verify-brio-encrypted-restore.sh`: destructive two-database Brio restore verification ## Networks @@ -30,17 +34,22 @@ Production also joins the VIF-specific external overlay network: - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` +Canary additionally joins Brio's staging-only, application-owned network: + +- `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` (`makepad_brio_staging_db`) + The manual deploy workflow sources these Compose variables from environment secrets with this mapping: - `${MAKEPAD_POSTGRES_DB_NETWORK}` <- `DEPLOY_CATWLK_DB_NETWORK` - `${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK}` <- `DEPLOY_LE_PETIT_COIN_DB_NETWORK` - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK` production only +- `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` <- `DEPLOY_BRIO_STAGING_DB_NETWORK` canary only -Every database network must be an attachable Swarm overlay created with `--opt encrypted`. The deploy workflow creates new networks with encryption and fails closed when an existing network is not encrypted. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and `--opt encrypted`, then redeploy PostgreSQL and the dependent stacks. +Every database network must be an attachable Swarm overlay created with `--opt encrypted`; Brio's dedicated network must additionally be `--internal`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. Application network topology is owned by the consuming application repositories. New Keycloak instances keep their own DB-facing Docker networks in the Keycloak repository and connect to this PostgreSQL server through the configured DB endpoint. -When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. Canary does not attach the VIF network. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead. The production override publishes PostgreSQL port 5432 in host mode so certificate-verified clients on the Keycloak and application VMs retain that endpoint while PostgreSQL remains pinned to the database node. +When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. Brio staging attaches only through `makepad_brio_staging_db` and verifies the alias `makepad-postgres-brio-staging`. Canary does not attach the VIF network. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead. The production override publishes PostgreSQL port 5432 in host mode so certificate-verified clients on the Keycloak and application VMs retain that endpoint while PostgreSQL remains pinned to the database node. ## Node Labels @@ -60,10 +69,16 @@ policy with `compose.host.yml` after provisioning the certificate, key, CA, password files, backup directory, and committed HBA policy: ```bash +: "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set to the DB certificate SAN hostname}" docker compose --env-file envs/production/.env.db -f compose.host.yml config docker compose --env-file envs/production/.env.db -f compose.host.yml up -d --pull always --remove-orphans --wait ``` +The standalone DB-VM deployment additionally requires +`MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST` to be exported as the exact DB +VM hostname present in the PostgreSQL server certificate SAN. The encrypted +identity backup refuses any connection mode other than `verify-full`. + The host deployment preserves the existing host-network endpoint used by Keycloak while requiring TLS and SCRAM for `runtrace` and `keycloak_runtrace`. Other databases keep their existing SCRAM transport policy. @@ -79,6 +94,12 @@ Required environment secrets: - `DEPLOY_CATWLK_DB_NETWORK` - `DEPLOY_LE_PETIT_COIN_DB_NETWORK` +Canary additionally requires: + +- `DEPLOY_BRIO_STAGING_DB_NETWORK` set exactly to `makepad_brio_staging_db`; + the workflow rejects alternate names so Brio and PostgreSQL cannot drift onto + disconnected look-alike networks + Production additionally requires: - `DEPLOY_VIF_DB_NETWORK` @@ -90,16 +111,28 @@ Production can override the VIF database and role names with `DEPLOY_VIF_DB_NAME Before the first deployment, provision the PostgreSQL superuser password as a non-empty root-owned file on the database node. The production default path is `/etc/makepad/secrets/postgres-superuser-password`; canary uses `/etc/makepad/secrets/postgres-canary-superuser-password`. Keep the file outside the repository, set mode `0600`, and override `MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH` only when the host secret manager materializes it elsewhere. PostgreSQL receives the value through `POSTGRES_PASSWORD_FILE`, and deployment helpers mount the same file read-only instead of placing the password in command arguments or tracked environment files. -Provision a private-CA-issued PostgreSQL server certificate before deployment. Its SANs must include every hostname clients verify, including `makepad-postgres` and the DB VM hostname used by Keycloak. Keep the unencrypted private key outside git and create versioned Swarm objects on the database manager: +Provision a private-CA-issued PostgreSQL server certificate before deployment. Its SANs must include every hostname clients verify, including `makepad-postgres`, `makepad-postgres-brio-staging`, and the DB VM hostname used by Keycloak. Keep the unencrypted private key outside git and create versioned Swarm objects on the database manager. Canary intentionally requires new `v2` objects so the older certificate cannot be reused without the Brio alias: ```sh docker config create makepad_postgres_tls_cert_v1 /secure/path/server.crt docker secret create makepad_postgres_tls_key_v1 /secure/path/server.key +docker config create makepad_postgres_canary_tls_cert_v2 /secure/path/canary-server.crt +docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-server.key ``` -The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace` and `keycloak_runtrace` and requires SCRAM authentication over TLS for both; unrelated shared databases retain their current SCRAM transport policy during migration. +The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging` and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. -The workflow deploys only the PostgreSQL stack. It validates the password file before deployment. If one of the configured database networks does not exist yet, it is created as an encrypted overlay on the manager before deployment. +The workflow deploys only the PostgreSQL stack. Before deployment it validates +the password and CA files, certificate chain, seven-day expiry margin, and—for +canary—the exact `makepad-postgres-brio-staging` SAN. After the stack update it +performs a real `sslmode=verify-full` query over Brio's isolated network using +that alias; a certificate/key mismatch prevents PostgreSQL from becoming ready +and causes the canary service update to roll back. If one of the configured +database networks does not exist yet, it is created as an encrypted overlay on +the manager before deployment. After `docker stack deploy`, the workflow waits +until PostgreSQL and the environment's Brio backup service are running the exact +pinned image and the Swarm update has completed; only then does it run the TLS +database probe. ## Runtrace Backup And Restore @@ -127,11 +160,105 @@ scripts/verify-runtrace-restore.sh /var/lib/makepad/postgres-backups/runtrace/.dump.cms`, checksummed metadata, and a health marker +are atomically published. Connections require `sslmode=verify-full`, backups +run every six hours, and timestamp directories are retained for exactly 35 +days. + +Create the X.509 encryption recipient and its private key in an offline recovery +environment. Install only the public recipient certificate on both database +hosts at `/etc/makepad/tls/backups/brio-recipient.crt`; the recovery private key +must never be copied to a database host, backup container, repository, CI +secret, or ordinary application secret store. Provision the service inputs: + +```bash +# Canary application database host. +sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/brio-staging +sudo install -o 999 -g 999 -m 0400 /secure/path/brio-staging-backup-role-password \ + /etc/makepad/secrets/postgres-brio-app-backup-password + +# Production identity database host. +sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/keycloak-brio-staging +sudo install -o 999 -g 999 -m 0400 /secure/path/keycloak-brio-staging-backup-role-password \ + /etc/makepad/secrets/postgres-brio-identity-backup-password + +# Public encryption material only; retain the matching private key offline. +sudo install -d -o root -g root -m 0755 /etc/makepad/tls/backups +sudo install -o root -g root -m 0444 /secure/path/brio-recipient.crt \ + /etc/makepad/tls/backups/brio-recipient.crt +``` + +The Swarm deploy preflight copies the tracked backup scripts, rejects symlinked +inputs, requires each backup directory to be owned by uid 999 with mode 0700, +requires each database credential to be owned by uid 999 with mode 0400, and +requires a root-owned, non-writable public recipient certificate that remains +valid for at least seven days and can create a CMS AES-256-GCM envelope. +Each mounted credential is the password for its database-specific backup role; +never place a PostgreSQL superuser or application-owner password in either +backup credential file. The backup command refuses any role other than the +expected `brio_staging_backup` or `keycloak_brio_staging_backup` identity. + +The host paths above are local staging artifacts, not disaster-recovery +storage. Replicate every completed timestamp directory and `last-success.json` +from each host to independently administered off-host storage without +decrypting it. Preserve the 35-day encrypted retention window there and alert +before either health marker exceeds two backup intervals. + +Restore verification requires the newest application and identity timestamp +directories, an external copy of the public certificate and private key, two +explicit non-production libpq services, and a mode-0700 temporary directory. +Both libpq service names must end in `_restore_test` as an additional guard +against selecting an ordinary runtime service. +The temporary directory should be a dedicated tmpfs because it is the only +place where the verifier writes plaintext. The verifier checks the envelopes +and checksums before decrypting, validates each custom archive, restores with a +single transaction and exit-on-error, and requires `schema_migrations` plus +`communities` for Brio and `realm` for Keycloak: + +```bash +export PGSERVICEFILE=/secure/restore/postgres-restore-services.conf +export BRIO_APP_RESTORE_SERVICE=brio_app_restore_test +export BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test +export BRIO_RESTORE_RECIPIENT_CERT=/offline-recovery/brio-recipient.crt +export BRIO_RESTORE_RECIPIENT_KEY=/offline-recovery/brio-recipient.key +export BRIO_RESTORE_TEMP_ROOT=/run/brio-restore-tmpfs +export BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets +scripts/verify-brio-encrypted-restore.sh \ + /off-host/brio-staging/ \ + /off-host/keycloak-brio-staging/ +``` + +Keep the service file and recovery private key non-symlinked and accessible only +to the restore operator. Record the source timestamps, artifact checksums, +recipient-certificate fingerprint, target names, duration, result, and operator. +Off-host replication and a successful recorded restore of both databases remain +external release gates; repository tests cannot attest that those operational +steps occurred. + ## Application Databases Create one database and one dedicated user per application. -Vif, Makepad, Vestiaire, and Runtrace Keycloak use these databases and roles: +Vif, Makepad, Vestiaire, Runtrace, and Brio staging Keycloak use these databases and roles: | Application | Database | Role | | --- | --- | --- | @@ -139,6 +266,7 @@ Vif, Makepad, Vestiaire, and Runtrace Keycloak use these databases and roles: | Makepad | `keycloak_makepad` | `keycloak_makepad_app` | | Vestiaire | `keycloak_vestiaire` | `keycloak_vestiaire_app` | | Runtrace Keycloak | `keycloak_runtrace` | `keycloak_runtrace_app` | +| Brio staging Keycloak | `keycloak_brio_staging` | `keycloak_brio_staging_app` | Runtrace application persistence uses: @@ -152,7 +280,15 @@ OpenPanel application persistence uses: | --- | --- | --- | | OpenPanel app | `openpanel` | `openpanel_app` | -Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must be a PostgreSQL superuser connection URI for the target server, usually using the `postgres` role, because the bootstrap creates roles, sets passwords, creates databases, and assigns database ownership. For example: `postgres://postgres@:5432/postgres?sslmode=disable`. +Brio staging application persistence and backups use: + +| Purpose | Database | Role | +| --- | --- | --- | +| Brio staging app | `brio_staging` | `brio_staging_app` | +| Brio staging read-only backup | `brio_staging` | `brio_staging_backup` | +| Brio Keycloak read-only backup | `keycloak_brio_staging` | `keycloak_brio_staging_backup` | + +Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must be a PostgreSQL superuser connection URI for the target server, usually using the `postgres` role, because the bootstrap creates roles, sets passwords, creates databases, and assigns database ownership. Run it on the database host over a Unix-domain socket, for example: `postgresql:///postgres?host=%2Fvar%2Frun%2Fpostgresql&user=postgres`. If remote administration is unavoidable, use the certificate-SAN hostname with `sslmode=verify-full`, the issuing CA, and a protected libpq password source. Both targeted Brio bootstraps inspect their own session and refuse a remote plaintext administrator session before changing any role or password. ```bash : "${POSTGRES_ADMIN_URL:?set POSTGRES_ADMIN_URL to a PostgreSQL superuser connection URI}" @@ -162,6 +298,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must : "${KEYCLOAK_RUNTRACE_DB_PASSWORD:?set KEYCLOAK_RUNTRACE_DB_PASSWORD to a generated password}" : "${RUNTRACE_DB_PASSWORD:?set RUNTRACE_DB_PASSWORD to a generated password}" : "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}" +: "${BRIO_STAGING_DB_PASSWORD:?set BRIO_STAGING_DB_PASSWORD to a generated password}" +: "${BRIO_STAGING_BACKUP_DB_PASSWORD:?set BRIO_STAGING_BACKUP_DB_PASSWORD to a distinct generated password}" psql "$POSTGRES_ADMIN_URL" \ -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ @@ -179,11 +317,28 @@ psql "$POSTGRES_ADMIN_URL" \ -v keycloak_runtrace_app_password="$KEYCLOAK_RUNTRACE_DB_PASSWORD" \ -f bootstrap/keycloak-runtrace-app.sql +: "${KEYCLOAK_BRIO_STAGING_DB_PASSWORD:?set KEYCLOAK_BRIO_STAGING_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD:?set KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD to a distinct generated password}" +psql "$POSTGRES_ADMIN_URL" \ + -v keycloak_brio_staging_app_password="$KEYCLOAK_BRIO_STAGING_DB_PASSWORD" \ + -v keycloak_brio_staging_backup_password="$KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD" \ + -f bootstrap/keycloak-brio-staging.sql + psql "$POSTGRES_ADMIN_URL" \ -v openpanel_app_password="$OPENPANEL_DB_PASSWORD" \ -f bootstrap/openpanel-app.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v brio_staging_app_password="$BRIO_STAGING_DB_PASSWORD" \ + -v brio_staging_backup_password="$BRIO_STAGING_BACKUP_DB_PASSWORD" \ + -f bootstrap/brio-staging-app.sql ``` +The backup roles have no ownership or write privileges, default to read-only +transactions, and receive only schema usage plus `SELECT` on current and future +tables and sequences in their own database. Keep all four Brio credentials +distinct. + The current production Keycloak environments connect with the DB VM host: ```text @@ -191,8 +346,10 @@ postgres://keycloak_vif_app:@:5432/keycloak_vif?sslmode=disa postgres://keycloak_makepad_app:@:5432/keycloak_makepad?sslmode=disable postgres://keycloak_vestiaire_app:@:5432/keycloak_vestiaire?sslmode=disable postgres://keycloak_runtrace_app:@:5432/keycloak_runtrace?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt +postgres://keycloak_brio_staging_app:@:5432/keycloak_brio_staging?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt postgres://runtrace_app:@:5432/runtrace?sslmode=verify-full&sslrootcert=/etc/runtrace/postgres/ca.crt postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable +postgres://brio_staging_app:@makepad-postgres-brio-staging:5432/brio_staging?sslmode=verify-full&sslrootcert=/etc/brio/postgres/ca.crt ``` Stacks deployed through this repository's shared overlay network should use the `makepad-postgres` alias instead: @@ -219,14 +376,31 @@ The production VIF app uses its app-specific overlay alias and deploy-time provi postgres://vif:@makepad-postgres-vif:5432/vif?sslmode=disable ``` +Brio staging uses only its isolated encrypted overlay and certificate-matching alias: + +```text +postgres://brio_staging_app:@makepad-postgres-brio-staging:5432/brio_staging?sslmode=verify-full&sslrootcert=/etc/brio/postgres/ca.crt +``` + If production overrides `DEPLOY_VIF_DB_NAME` or `DEPLOY_VIF_DB_USER`, use those values in the connection URI. ## Validation +Run the static deployment checks and the disposable PostgreSQL 16 bootstrap test: + +```sh +./scripts/validate-postgres-config.sh +./scripts/test-brio-bootstrap.sh +./scripts/test-brio-encrypted-backup.sh +./scripts/test-brio-encrypted-restore.sh +``` + Run the local static checks before opening a deployment PR: ```bash bash scripts/validate-postgres-config.sh bash scripts/test-runtrace-tls-policy.sh bash scripts/test-runtrace-backup.sh +bash scripts/test-brio-encrypted-backup.sh +bash scripts/test-brio-encrypted-restore.sh ``` diff --git a/bootstrap/brio-staging-app.sql b/bootstrap/brio-staging-app.sql new file mode 100644 index 0000000..84e011b --- /dev/null +++ b/bootstrap/brio-staging-app.sql @@ -0,0 +1,82 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. This creates the staging-only +-- Brio application role/database pair without embedding credentials. + +\if :{?brio_staging_app_password} +\else + \echo 'missing required psql variable: brio_staging_app_password' + SELECT 1 / 0; +\endif + +\if :{?brio_staging_backup_password} +\else + \echo 'missing required psql variable: brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'brio_staging_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS brio_staging_app_password_is_nonempty \gset +\if :brio_staging_app_password_is_nonempty +\else + \echo 'empty required psql variable: brio_staging_app_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'brio_staging_backup_password'), '') IS NULL THEN 'false' ELSE 'true' END AS brio_staging_backup_password_is_nonempty \gset +\if :brio_staging_backup_password_is_nonempty +\else + \echo 'empty required psql variable: brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE + WHEN inet_client_addr() IS NULL OR coalesce((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false) + THEN 'true' ELSE 'false' +END AS brio_staging_bootstrap_transport_is_secure \gset +\if :brio_staging_bootstrap_transport_is_secure +\else + \echo 'Brio staging bootstrap refuses a remote plaintext PostgreSQL session' + SELECT 1 / 0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('brio-staging-app-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'brio_staging_app') THEN + CREATE ROLE brio_staging_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'brio_staging_backup') THEN + CREATE ROLE brio_staging_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; +END; +$$; +ALTER ROLE brio_staging_app LOGIN PASSWORD :'brio_staging_app_password'; +ALTER ROLE brio_staging_app NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS; +ALTER ROLE brio_staging_backup LOGIN PASSWORD :'brio_staging_backup_password'; +ALTER ROLE brio_staging_backup NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2; +SELECT 'CREATE DATABASE brio_staging OWNER brio_staging_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'brio_staging') \gexec +SELECT 'ALTER DATABASE brio_staging OWNER TO brio_staging_app' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'brio_staging' + AND r.rolname <> 'brio_staging_app' +) \gexec +REVOKE ALL ON DATABASE brio_staging FROM PUBLIC; +REVOKE ALL ON DATABASE brio_staging FROM brio_staging_backup; +GRANT CONNECT ON DATABASE brio_staging TO brio_staging_app; +GRANT CONNECT ON DATABASE brio_staging TO brio_staging_backup; +ALTER ROLE brio_staging_backup IN DATABASE brio_staging SET default_transaction_read_only TO on; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('brio-staging-app-bootstrap')); + +\connect brio_staging +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO brio_staging_backup; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO brio_staging_backup; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE brio_staging_app IN SCHEMA public GRANT SELECT ON TABLES TO brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE brio_staging_app IN SCHEMA public GRANT SELECT ON SEQUENCES TO brio_staging_backup; diff --git a/bootstrap/keycloak-brio-staging.sql b/bootstrap/keycloak-brio-staging.sql new file mode 100644 index 0000000..6c7e3a6 --- /dev/null +++ b/bootstrap/keycloak-brio-staging.sql @@ -0,0 +1,83 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. This creates only the +-- staging-only Brio Keycloak role/database pair and never rotates credentials +-- for another Keycloak realm. + +\if :{?keycloak_brio_staging_app_password} +\else + \echo 'missing required psql variable: keycloak_brio_staging_app_password' + SELECT 1 / 0; +\endif + +\if :{?keycloak_brio_staging_backup_password} +\else + \echo 'missing required psql variable: keycloak_brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'keycloak_brio_staging_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_brio_staging_app_password_is_nonempty \gset +\if :keycloak_brio_staging_app_password_is_nonempty +\else + \echo 'empty required psql variable: keycloak_brio_staging_app_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'keycloak_brio_staging_backup_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_brio_staging_backup_password_is_nonempty \gset +\if :keycloak_brio_staging_backup_password_is_nonempty +\else + \echo 'empty required psql variable: keycloak_brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE + WHEN inet_client_addr() IS NULL OR coalesce((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false) + THEN 'true' ELSE 'false' +END AS keycloak_brio_staging_bootstrap_transport_is_secure \gset +\if :keycloak_brio_staging_bootstrap_transport_is_secure +\else + \echo 'Brio Keycloak bootstrap refuses a remote plaintext PostgreSQL session' + SELECT 1 / 0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('keycloak-brio-staging-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_brio_staging_app') THEN + CREATE ROLE keycloak_brio_staging_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_brio_staging_backup') THEN + CREATE ROLE keycloak_brio_staging_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; +END; +$$; +ALTER ROLE keycloak_brio_staging_app LOGIN PASSWORD :'keycloak_brio_staging_app_password'; +ALTER ROLE keycloak_brio_staging_app NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS; +ALTER ROLE keycloak_brio_staging_backup LOGIN PASSWORD :'keycloak_brio_staging_backup_password'; +ALTER ROLE keycloak_brio_staging_backup NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2; +SELECT 'CREATE DATABASE keycloak_brio_staging OWNER keycloak_brio_staging_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_brio_staging') \gexec +SELECT 'ALTER DATABASE keycloak_brio_staging OWNER TO keycloak_brio_staging_app' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'keycloak_brio_staging' + AND r.rolname <> 'keycloak_brio_staging_app' +) \gexec +REVOKE ALL ON DATABASE keycloak_brio_staging FROM PUBLIC; +REVOKE ALL ON DATABASE keycloak_brio_staging FROM keycloak_brio_staging_backup; +GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_app; +GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_backup; +ALTER ROLE keycloak_brio_staging_backup IN DATABASE keycloak_brio_staging SET default_transaction_read_only TO on; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('keycloak-brio-staging-bootstrap')); + +\connect keycloak_brio_staging +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO keycloak_brio_staging_backup; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO keycloak_brio_staging_backup; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO keycloak_brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT SELECT ON TABLES TO keycloak_brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT SELECT ON SEQUENCES TO keycloak_brio_staging_backup; diff --git a/compose.host.yml b/compose.host.yml index 6ab3ba7..0fcfaeb 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -81,3 +81,52 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + keycloak_brio_staging_backup: + image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} + user: "999:999" + network_mode: host + command: + - /usr/local/bin/run-brio-encrypted-backup-loop.sh + environment: + PGHOST: ${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST to the DB certificate SAN hostname} + PGPORT: "5432" + PGUSER: keycloak_brio_staging_backup + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password + BRIO_BACKUP_DATABASE: keycloak_brio_staging + BRIO_BACKUP_ROOT: /backups + BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt + BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} + BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} + BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH:-/var/lib/makepad/postgres-backups/keycloak-brio-staging}:/backups" + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-brio-identity-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:-/etc/makepad/tls/backups/brio-recipient.crt}:/etc/postgresql/brio-backup-recipient.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-brio-encrypted-backup.sh}:/usr/local/bin/run-brio-encrypted-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-brio-encrypted-backup-loop.sh}:/usr/local/bin/run-brio-encrypted-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/compose.yml b/compose.yml index 2cc8077..03ca83b 100644 --- a/compose.yml +++ b/compose.yml @@ -54,6 +54,10 @@ configs: file: ./scripts/run-runtrace-backup.sh runtrace_backup_loop_script: file: ./scripts/run-runtrace-backup-loop.sh + brio_encrypted_backup_script: + file: ./scripts/run-brio-encrypted-backup.sh + brio_encrypted_backup_loop_script: + file: ./scripts/run-brio-encrypted-backup-loop.sh secrets: postgres_tls_key: diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 55be5a8..251f826 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -3,6 +3,16 @@ local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject +hostnossl brio_staging all all reject +hostnossl keycloak_brio_staging all all reject hostssl runtrace all all scram-sha-256 hostssl keycloak_runtrace all all scram-sha-256 +hostssl brio_staging brio_staging_app all scram-sha-256 +hostssl brio_staging brio_staging_backup all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_backup all scram-sha-256 +host all brio_staging_app all reject +host all brio_staging_backup all reject +host all keycloak_brio_staging_app all reject +host all keycloak_brio_staging_backup all reject host all all all scram-sha-256 diff --git a/envs/canary/.env.db b/envs/canary/.env.db index 43296e7..545c4d9 100644 --- a/envs/canary/.env.db +++ b/envs/canary/.env.db @@ -1,9 +1,23 @@ # Canary PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +BRIO_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres-canary MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-canary-superuser-password -MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v1 -MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v1 +MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v2 +MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v2 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v2 +MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt +MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=/var/lib/makepad/postgres-backups/brio-staging +MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-app-backup-password +MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=/etc/makepad/tls/backups/brio-recipient.crt +MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 +MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G +MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 +MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION=128M +MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE=20m +MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES=5 diff --git a/envs/canary/compose.yml b/envs/canary/compose.yml index ffe261f..6d31ec4 100644 --- a/envs/canary/compose.yml +++ b/envs/canary/compose.yml @@ -18,6 +18,7 @@ services: update_config: parallelism: 1 order: stop-first + failure_action: rollback rollback_config: parallelism: 1 order: stop-first @@ -28,6 +29,76 @@ services: reservations: cpus: "${MAKEPAD_POSTGRES_CPU_RESERVATION:-0.25}" memory: ${MAKEPAD_POSTGRES_MEMORY_RESERVATION:-256M} + networks: + brio_staging: + aliases: + - makepad-postgres-brio-staging + + brio_staging_backup: + image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} + user: "999:999" + command: + - /usr/local/bin/run-brio-encrypted-backup-loop.sh + environment: + PGHOST: makepad-postgres-brio-staging + PGPORT: "5432" + PGUSER: brio_staging_backup + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password + BRIO_BACKUP_DATABASE: brio_staging + BRIO_BACKUP_ROOT: /backups + BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt + BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} + BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} + BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + networks: + - brio_staging + volumes: + - "${MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH:?set MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH}:/backups" + - "${MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_CA_CERT_HOST_PATH}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH}:/etc/postgresql/brio-backup-recipient.crt:ro" + configs: + - source: brio_encrypted_backup_script + target: /usr/local/bin/run-brio-encrypted-backup.sh + mode: 0555 + - source: brio_encrypted_backup_loop_script + target: /usr/local/bin/run-brio-encrypted-backup-loop.sh + mode: 0555 + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + deploy: + replicas: 1 + placement: + constraints: + - node.labels.infra.makepad.postgres == true + restart_policy: + condition: on-failure + delay: 30s + resources: + limits: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT:-1.0}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT:-1G} + reservations: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} networks: db: @@ -36,3 +107,6 @@ networks: le_petit_coin: external: true name: ${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK} + brio_staging: + external: true + name: ${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK:?set MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK} diff --git a/envs/production/.env.db b/envs/production/.env.db index fbe71c5..a37d94a 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -1,18 +1,27 @@ # Production PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +BRIO_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-superuser-password MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_tls_cert_v1 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v1 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v2 MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=/var/lib/makepad/postgres-backups/runtrace MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH=/var/lib/makepad/postgres-backups/keycloak-brio-staging +MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-identity-backup-password +MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=/etc/makepad/tls/backups/brio-recipient.crt +MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS=35 +# Standalone compose.host.yml additionally requires MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST +# to be exported as the DB VM hostname present in the PostgreSQL server certificate SAN. MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 diff --git a/envs/production/compose.yml b/envs/production/compose.yml index 76cdbe4..3a6011d 100644 --- a/envs/production/compose.yml +++ b/envs/production/compose.yml @@ -107,6 +107,72 @@ services: cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} + keycloak_brio_staging_backup: + image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} + user: "999:999" + command: + - /usr/local/bin/run-brio-encrypted-backup-loop.sh + environment: + PGHOST: makepad-postgres + PGPORT: "5432" + PGUSER: keycloak_brio_staging_backup + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password + BRIO_BACKUP_DATABASE: keycloak_brio_staging + BRIO_BACKUP_ROOT: /backups + BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt + BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} + BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} + BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + networks: + - db + volumes: + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH}:/backups" + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_CA_CERT_HOST_PATH}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH}:/etc/postgresql/brio-backup-recipient.crt:ro" + configs: + - source: brio_encrypted_backup_script + target: /usr/local/bin/run-brio-encrypted-backup.sh + mode: 0555 + - source: brio_encrypted_backup_loop_script + target: /usr/local/bin/run-brio-encrypted-backup-loop.sh + mode: 0555 + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + deploy: + replicas: 1 + placement: + constraints: + - node.labels.infra.makepad.postgres == true + restart_policy: + condition: on-failure + delay: 30s + resources: + limits: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT:-1.0}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT:-1G} + reservations: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} + networks: db: external: true diff --git a/scripts/run-brio-encrypted-backup-loop.sh b/scripts/run-brio-encrypted-backup-loop.sh new file mode 100755 index 0000000..ec924f1 --- /dev/null +++ b/scripts/run-brio-encrypted-backup-loop.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +interval_seconds=${BRIO_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${BRIO_BACKUP_RETRY_SECONDS:-300} +backup_root=${BRIO_BACKUP_ROOT:-/backups} + +validate_timing_value() { + case "$2" in + ''|*[!0-9]*) + echo "$1 must be a positive integer." >&2 + exit 1 + ;; + esac +} +validate_timing_value BRIO_BACKUP_INTERVAL_SECONDS "${interval_seconds}" +validate_timing_value BRIO_BACKUP_RETRY_SECONDS "${retry_seconds}" +if [ "${interval_seconds}" -lt 300 ] || [ "${retry_seconds}" -lt 30 ]; then + echo "Brio backup interval must be at least 300 seconds and retry delay at least 30 seconds." >&2 + exit 1 +fi + +if [ "${1:-}" = "healthcheck" ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then + : + else + modified=$(stat -f %m "${status_file}") + fi + max_age=$((interval_seconds * 2 + retry_seconds)) + [ $((now - modified)) -le "${max_age}" ] + exit +fi + +while :; do + if /usr/local/bin/run-brio-encrypted-backup.sh; then + sleep "${interval_seconds}" + else + echo "Encrypted Brio PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-brio-encrypted-backup.sh b/scripts/run-brio-encrypted-backup.sh new file mode 100755 index 0000000..c443708 --- /dev/null +++ b/scripts/run-brio-encrypted-backup.sh @@ -0,0 +1,233 @@ +#!/bin/sh +set -eu + +umask 077 + +database=${BRIO_BACKUP_DATABASE:?BRIO_BACKUP_DATABASE must be brio_staging or keycloak_brio_staging} +backup_root=${BRIO_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_BACKUP_PASSWORD_FILE:-/run/secrets/postgres_backup_password} +recipient_cert=${BRIO_BACKUP_RECIPIENT_CERT:-/etc/postgresql/brio-backup-recipient.crt} +retention_days=${BRIO_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:?PGHOST is required} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:?PGUSER must identify the database-specific Brio backup role} + +case "${database}" in + brio_staging) expected_pg_user=brio_staging_backup ;; + keycloak_brio_staging) expected_pg_user=keycloak_brio_staging_backup ;; + *) + echo "BRIO_BACKUP_DATABASE must be brio_staging or keycloak_brio_staging." >&2 + exit 1 + ;; +esac +if [ "${pg_user}" != "${expected_pg_user}" ]; then + echo "PGUSER must be ${expected_pg_user} when backing up ${database}." >&2 + exit 1 +fi +case "${retention_days}" in + ''|*[!0-9]*) + echo "BRIO_BACKUP_RETENTION_DAYS must be a positive integer." >&2 + exit 1 + ;; +esac +if [ "${retention_days}" -ne 35 ]; then + echo "Brio backup retention must remain exactly 35 days." >&2 + exit 1 +fi +case "${pg_port}" in + ''|*[!0-9]*) + echo "PGPORT must be a positive integer." >&2 + exit 1 + ;; +esac +if [ "${pg_port}" -lt 1 ] || [ "${pg_port}" -gt 65535 ]; then + echo "PGPORT must be between 1 and 65535." >&2 + exit 1 +fi +case "${pg_host}" in + ''|*[!A-Za-z0-9._-]*) + echo "PGHOST contains unsupported characters." >&2 + exit 1 + ;; +esac +case "${pg_user}" in + ''|*[!A-Za-z0-9._-]*) + echo "PGUSER contains unsupported characters." >&2 + exit 1 + ;; +esac +case "${backup_root}" in + ''|/) + echo "BRIO_BACKUP_ROOT must identify a dedicated backup directory." >&2 + exit 1 + ;; +esac + +for command_name in pg_dump openssl sha256sum mkfifo; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "Missing required backup command: ${command_name}" >&2 + exit 1 + fi +done +if [ ! -d "${backup_root}" ] || [ -L "${backup_root}" ]; then + echo "BRIO_BACKUP_ROOT must be a pre-provisioned non-symlink directory." >&2 + exit 1 +fi +chmod 0700 "${backup_root}" +if [ ! -s "${password_file}" ] || [ -L "${password_file}" ]; then + echo "PostgreSQL backup credential must be a non-empty, non-symlink file." >&2 + exit 1 +fi +if [ ! -s "${recipient_cert}" ] || [ -L "${recipient_cert}" ]; then + echo "Brio backup recipient certificate must be a non-empty, non-symlink file." >&2 + exit 1 +fi +if ! openssl x509 -in "${recipient_cert}" -noout -checkend 604800 >/dev/null 2>&1; then + echo "Brio backup recipient certificate is invalid or expires in less than seven days." >&2 + exit 1 +fi + +lock_dir=/tmp/.brio-backup-${database}.lock +if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another Brio PostgreSQL backup is already running for ${database}." >&2 + exit 1 +fi + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +encrypted_name=${database}.dump.cms +encrypted_path=${partial_dir}/${encrypted_name} +pipe_path=/tmp/brio-backup-${database}-${timestamp}-$$.pipe +pgpass=/tmp/brio-backup-${database}-${timestamp}-$$.pgpass +encrypt_pid= + +cleanup() { + if [ -n "${encrypt_pid}" ]; then + kill "${encrypt_pid}" 2>/dev/null || true + wait "${encrypt_pid}" 2>/dev/null || true + fi + if [ -p "${pipe_path}" ] || [ -f "${pipe_path}" ]; then + rm -f "${pipe_path}" + fi + if [ -f "${pgpass}" ]; then + rm -f "${pgpass}" + fi + if [ -d "${partial_dir}" ]; then + find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${partial_dir}" 2>/dev/null || true + fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 1' HUP INT TERM + +if [ -e "${final_dir}" ] || [ -L "${final_dir}" ] || [ -e "${partial_dir}" ] || [ -L "${partial_dir}" ]; then + echo "Backup destination already exists for ${timestamp}." >&2 + exit 1 +fi + +password=$(tr -d '\r\n' < "${password_file}") +if [ -z "${password}" ]; then + echo "PostgreSQL backup credential file contains no usable value." >&2 + exit 1 +fi +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 0600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" +if [ "${PGSSLMODE}" != "verify-full" ]; then + echo "Brio backups require PGSSLMODE=verify-full." >&2 + exit 1 +fi + +mkdir "${partial_dir}" +mkfifo -m 0600 "${pipe_path}" +openssl cms \ + -encrypt \ + -binary \ + -stream \ + -outform DER \ + -aes-256-gcm \ + -recip "${recipient_cert}" \ + -in "${pipe_path}" \ + -out "${encrypted_path}" & +encrypt_pid=$! + +if pg_dump \ + --host="${pg_host}" \ + --port="${pg_port}" \ + --username="${pg_user}" \ + --dbname="${database}" \ + --format=custom \ + --compress=9 \ + --no-owner \ + --no-acl \ + > "${pipe_path}"; then + dump_status=0 +else + dump_status=$? +fi + +if wait "${encrypt_pid}"; then + encrypt_status=0 +else + encrypt_status=$? +fi +encrypt_pid= +rm -f "${pipe_path}" + +if [ "${dump_status}" -ne 0 ] || [ "${encrypt_status}" -ne 0 ]; then + echo "Encrypted Brio backup failed before publication for ${database}." >&2 + exit 1 +fi +if [ ! -s "${encrypted_path}" ]; then + echo "Encrypted Brio backup output is empty." >&2 + exit 1 +fi +if ! openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null 2>&1; then + echo "Encrypted Brio backup is not a valid CMS envelope." >&2 + exit 1 +fi + +recipient_fingerprint=$(openssl x509 -in "${recipient_cert}" -noout -fingerprint -sha256 | cut -d= -f2- | tr -d ':') +case "${recipient_fingerprint}" in + ''|*[!A-Fa-f0-9]*) + echo "Unable to derive the backup recipient certificate fingerprint." >&2 + exit 1 + ;; +esac +cat > "${partial_dir}/metadata.json" < SHA256SUMS +) +chmod 0600 "${partial_dir}"/* +mv "${partial_dir}" "${final_dir}" + +if [ -e "${backup_root}/latest" ] && [ ! -L "${backup_root}/latest" ]; then + echo "Backup latest marker exists but is not a symlink." >&2 + exit 1 +fi +ln -sfn "${timestamp}" "${backup_root}/latest" + +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","database":"%s","encrypted":true}\n' \ + "${timestamp}" "${timestamp}" "${database}" > "${status_tmp}" +chmod 0600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +retention_find_days=$((retention_days - 1)) +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_find_days}" -exec sh -c ' + for directory do + find "$directory" -mindepth 1 -delete + rmdir "$directory" + done +' sh {} + + +echo "Encrypted Brio PostgreSQL backup completed for ${database}: ${final_dir}" diff --git a/scripts/test-brio-bootstrap.sh b/scripts/test-brio-bootstrap.sh new file mode 100755 index 0000000..0af51eb --- /dev/null +++ b/scripts/test-brio-bootstrap.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +postgres_image=$(awk -F= '$1 == "POSTGRES_IMAGE" {print $2}' "${repo_root}/envs/canary/.env.db") +container_name="brio-postgres-bootstrap-${RANDOM}-$$" +postgres_password='brio-bootstrap-integration-only' +brio_app_password='brio-app-integration-only' +brio_backup_password='brio-app-backup-integration-only' +keycloak_app_password='brio-keycloak-integration-only' +keycloak_backup_password='brio-keycloak-backup-integration-only' + +cleanup() { + docker rm -f "${container_name}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run -d --name "${container_name}" \ + -e POSTGRES_PASSWORD="${postgres_password}" \ + -v "${repo_root}/bootstrap:/bootstrap:ro" \ + "${postgres_image}" >/dev/null + +for _ in $(seq 1 100); do + if docker exec "${container_name}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then + break + fi + sleep 0.1 +done +docker exec "${container_name}" pg_isready -U postgres -d postgres >/dev/null + +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null 2>&1; then + echo "Brio application bootstrap unexpectedly accepted a missing backup-role password." >&2 + exit 1 +fi +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password='' \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null 2>&1; then + echo "Brio Keycloak bootstrap unexpectedly accepted an empty backup-role password." >&2 + exit 1 +fi + +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -v brio_staging_backup_password="${brio_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null 2>&1; then + echo "Brio application bootstrap unexpectedly accepted a remote plaintext administrator session." >&2 + exit 1 +fi +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password="${keycloak_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null 2>&1; then + echo "Brio Keycloak bootstrap unexpectedly accepted a remote plaintext administrator session." >&2 + exit 1 +fi + +run_keycloak_brio_bootstrap() { + docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password="${keycloak_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null +} + +run_brio_bootstrap() { + docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -v brio_staging_backup_password="${brio_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null +} + +# Both bootstraps are explicitly idempotent. +run_keycloak_brio_bootstrap +run_brio_bootstrap +run_keycloak_brio_bootstrap +run_brio_bootstrap + +role_contract=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_roles WHERE rolname IN ('brio_staging_app','brio_staging_backup','keycloak_brio_staging_app','keycloak_brio_staging_backup') AND rolcanlogin AND NOT rolsuper AND NOT rolcreatedb AND NOT rolcreaterole AND NOT rolreplication AND NOT rolbypassrls") +[[ "${role_contract}" == "4" ]] + +owner_contract=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_database d JOIN pg_roles r ON r.oid=d.datdba WHERE (d.datname='brio_staging' AND r.rolname='brio_staging_app') OR (d.datname='keycloak_brio_staging' AND r.rolname='keycloak_brio_staging_app')") +[[ "${owner_contract}" == "2" ]] + +public_connect=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_database d, LATERAL aclexplode(coalesce(d.datacl, acldefault('d', d.datdba))) a WHERE d.datname IN ('brio_staging','keycloak_brio_staging') AND a.grantee=0 AND a.privilege_type='CONNECT'") +[[ "${public_connect}" == "0" ]] + +assert_backup_role() { + local database=$1 + local app_user=$2 + local app_password=$3 + local backup_user=$4 + local backup_password=$5 + local other_database=$6 + + docker exec -e PGPASSWORD="${app_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U "${app_user}" -d "${database}" \ + -c "CREATE TABLE IF NOT EXISTS public.backup_role_probe (marker text NOT NULL)" \ + -c "TRUNCATE public.backup_role_probe" \ + -c "INSERT INTO public.backup_role_probe(marker) VALUES ('readable')" >/dev/null + + read_only=$(docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -At -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "SHOW default_transaction_read_only") + [[ "${read_only}" == "on" ]] + + readable=$(docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -At -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "SELECT marker FROM public.backup_role_probe") + [[ "${readable}" == "readable" ]] + + if docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "INSERT INTO public.backup_role_probe(marker) VALUES ('forbidden')" >/dev/null 2>&1; then + echo "Backup role ${backup_user} unexpectedly wrote to ${database}." >&2 + exit 1 + fi + + docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + pg_dump -h 127.0.0.1 -U "${backup_user}" -d "${database}" --format=custom >/dev/null + + if docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -h 127.0.0.1 -U "${backup_user}" -d "${other_database}" -c "SELECT 1" >/dev/null 2>&1; then + echo "Backup role ${backup_user} unexpectedly connected to ${other_database}." >&2 + exit 1 + fi +} + +assert_backup_role \ + brio_staging brio_staging_app "${brio_app_password}" \ + brio_staging_backup "${brio_backup_password}" keycloak_brio_staging +assert_backup_role \ + keycloak_brio_staging keycloak_brio_staging_app "${keycloak_app_password}" \ + keycloak_brio_staging_backup "${keycloak_backup_password}" brio_staging + +echo "Brio PostgreSQL app and read-only backup role, database, privilege, and idempotency tests passed" diff --git a/scripts/test-brio-encrypted-backup.sh b/scripts/test-brio-encrypted-backup.sh new file mode 100755 index 0000000..c805826 --- /dev/null +++ b/scripts/test-brio-encrypted-backup.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups/app" "${work_dir}/backups/failure" +chmod 0700 "${work_dir}/backups/app" "${work_dir}/backups/failure" +mkdir "${work_dir}/backups/app/20200101T000000Z" "${work_dir}/backups/app/20990101T000000Z" +touch -t 202001010000 "${work_dir}/backups/app/20200101T000000Z" +printf 'validation-password\n' > "${work_dir}/password" +chmod 0600 "${work_dir}/password" + +openssl req \ + -x509 \ + -newkey rsa:2048 \ + -nodes \ + -days 30 \ + -subj /CN=brio-backup-contract \ + -keyout "${work_dir}/recipient.key" \ + -out "${work_dir}/recipient.crt" >/dev/null 2>&1 +chmod 0600 "${work_dir}/recipient.key" + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +database= +for argument in "$@"; do + case "${argument}" in + --dbname=*) database=${argument#--dbname=} ;; + --file=*) + echo "Brio backup must stream pg_dump and never write a plaintext file." >&2 + exit 91 + ;; + esac +done +[ "${database}" = "brio_staging" ] || [ "${database}" = "keycloak_brio_staging" ] +printf 'contract plaintext for %s\n' "${database}" +if [ "${FAKE_PG_DUMP_FAIL:-0}" = "1" ]; then + exit 42 +fi +EOF +chmod 0700 "${work_dir}/bin/pg_dump" + +PATH="${work_dir}/bin:${PATH}" \ +BRIO_BACKUP_DATABASE=brio_staging \ +BRIO_BACKUP_ROOT="${work_dir}/backups/app" \ +BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ +BRIO_BACKUP_RETENTION_DAYS=35 \ +POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ +PGHOST=makepad-postgres-brio-staging \ +PGPORT=5432 \ +PGUSER=brio_staging_backup \ +PGSSLMODE=verify-full \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ +sh "${repo_root}/scripts/run-brio-encrypted-backup.sh" + +latest=$(readlink "${work_dir}/backups/app/latest") +backup_dir=${work_dir}/backups/app/${latest} +for expected in brio_staging.dump.cms SHA256SUMS metadata.json; do + test -s "${backup_dir}/${expected}" +done +test ! -e "${backup_dir}/brio_staging.dump" +if grep -aFq 'contract plaintext' "${backup_dir}/brio_staging.dump.cms"; then + echo "Encrypted artifact exposed plaintext backup content." >&2 + exit 1 +fi +( + cd "${backup_dir}" + sha256sum --check --strict SHA256SUMS >/dev/null +) +openssl cms \ + -decrypt \ + -binary \ + -inform DER \ + -recip "${work_dir}/recipient.crt" \ + -inkey "${work_dir}/recipient.key" \ + -in "${backup_dir}/brio_staging.dump.cms" \ + -out "${work_dir}/decrypted.dump" +grep -Fq 'contract plaintext for brio_staging' "${work_dir}/decrypted.dump" +test -s "${work_dir}/backups/app/last-success.json" +test ! -e "${work_dir}/backups/app/20200101T000000Z" +test -d "${work_dir}/backups/app/20990101T000000Z" + +BRIO_BACKUP_ROOT="${work_dir}/backups/app" \ +BRIO_BACKUP_INTERVAL_SECONDS=300 \ +BRIO_BACKUP_RETRY_SECONDS=30 \ +sh "${repo_root}/scripts/run-brio-encrypted-backup-loop.sh" healthcheck + +if PATH="${work_dir}/bin:${PATH}" \ + FAKE_PG_DUMP_FAIL=1 \ + BRIO_BACKUP_DATABASE=keycloak_brio_staging \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_BACKUP_RETENTION_DAYS=35 \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + PGPORT=5432 \ + PGUSER=keycloak_brio_staging_backup \ + PGSSLMODE=verify-full \ + PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "A failing pg_dump unexpectedly published a Brio backup." >&2 + exit 1 +fi +if find "${work_dir}/backups/failure" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' | grep -q .; then + echo "A failing pg_dump left a published timestamp directory." >&2 + exit 1 +fi +if find "${work_dir}/backups/failure" -type f -name '*.dump' -o -name '*.dump.cms' | grep -q .; then + echo "A failing pg_dump left a backup artifact." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE=brio_staging \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres-brio-staging \ + PGUSER=postgres \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "The PostgreSQL superuser unexpectedly passed Brio backup validation." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE=unexpected_database \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "An unallowlisted database unexpectedly passed backup validation." >&2 + exit 1 +fi + +echo "Brio encrypted PostgreSQL backup contract passed." diff --git a/scripts/test-brio-encrypted-restore.sh b/scripts/test-brio-encrypted-restore.sh new file mode 100755 index 0000000..e220cee --- /dev/null +++ b/scripts/test-brio-encrypted-restore.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p \ + "${work_dir}/bin" \ + "${work_dir}/backups/app" \ + "${work_dir}/backups/keycloak" \ + "${work_dir}/restore-tmp" +chmod 0700 "${work_dir}/backups/app" "${work_dir}/backups/keycloak" "${work_dir}/restore-tmp" +printf 'validation-password\n' > "${work_dir}/password" +printf '[brio_app_restore_test]\nhost=nonproduction.invalid\n\n[brio_keycloak_restore_test]\nhost=nonproduction.invalid\n' > "${work_dir}/pg_service.conf" +chmod 0600 "${work_dir}/password" "${work_dir}/pg_service.conf" + +openssl req \ + -x509 \ + -newkey rsa:2048 \ + -nodes \ + -days 30 \ + -subj /CN=brio-restore-contract \ + -keyout "${work_dir}/recipient.key" \ + -out "${work_dir}/recipient.crt" >/dev/null 2>&1 +chmod 0600 "${work_dir}/recipient.key" + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +database= +for argument in "$@"; do + case "${argument}" in + --dbname=*) database=${argument#--dbname=} ;; + esac +done +printf 'fake custom dump for %s\n' "${database}" +EOF + +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +if [ "${1:-}" = "--list" ]; then + grep -Fq 'fake custom dump for ' "$2" + exit 0 +fi +joined=" $* " +last_argument= +for argument in "$@"; do + last_argument=${argument} +done +for expected in '--clean' '--if-exists' '--no-owner' '--no-acl' '--exit-on-error' '--single-transaction'; do + case "${joined}" in + *" ${expected} "*) ;; + *) echo "missing restore flag: ${expected}" >&2; exit 1 ;; + esac +done +case "${joined}" in + *' --dbname=service=brio_app_restore_test '*) grep -Fq 'fake custom dump for brio_staging' "${last_argument}" ;; + *' --dbname=service=brio_keycloak_restore_test '*) grep -Fq 'fake custom dump for keycloak_brio_staging' "${last_argument}" ;; + *) echo "unexpected restore service" >&2; exit 1 ;; +esac +printf '%s\n' "$*" >> "${RESTORE_LOG}" +EOF + +cat > "${work_dir}/bin/psql" <<'EOF' +#!/bin/sh +set -eu +case " $* " in + *' service=brio_app_restore_test '*) database=brio_app_restore_test ;; + *' service=brio_keycloak_restore_test '*) database=brio_keycloak_restore_test ;; + *) echo "unexpected psql service" >&2; exit 1 ;; +esac +case " $* " in + *'SELECT current_database();'*) printf '%s\n' "${FAKE_RESTORE_DATABASE:-${database}}" ;; + *) printf 't\n' ;; +esac +EOF +chmod 0700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" "${work_dir}/bin/psql" + +create_bundle() { + local database=$1 + local root=$2 + local backup_user + case "${database}" in + brio_staging) backup_user=brio_staging_backup ;; + keycloak_brio_staging) backup_user=keycloak_brio_staging_backup ;; + *) return 1 ;; + esac + PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE="${database}" \ + BRIO_BACKUP_ROOT="${root}" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_BACKUP_RETENTION_DAYS=35 \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + PGPORT=5432 \ + PGUSER="${backup_user}" \ + PGSSLMODE=verify-full \ + PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh" >/dev/null +} + +create_bundle brio_staging "${work_dir}/backups/app" +create_bundle keycloak_brio_staging "${work_dir}/backups/keycloak" +app_backup=${work_dir}/backups/app/$(readlink "${work_dir}/backups/app/latest") +keycloak_backup=${work_dir}/backups/keycloak/$(readlink "${work_dir}/backups/keycloak/latest") + +if PATH="${work_dir}/bin:${PATH}" \ + PGSERVICEFILE="${work_dir}/pg_service.conf" \ + BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ + BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ + BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ + BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ + BRIO_RESTORE_CONFIRM=wrong-confirmation \ + bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}"; then + echo "Restore unexpectedly accepted an invalid destructive confirmation." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + FAKE_RESTORE_DATABASE=brio_staging \ + PGSERVICEFILE="${work_dir}/pg_service.conf" \ + BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ + BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ + BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ + BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ + BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets \ + bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}"; then + echo "Restore unexpectedly accepted a production-shaped database target." >&2 + exit 1 +fi + +export RESTORE_LOG=${work_dir}/restore.log +PATH="${work_dir}/bin:${PATH}" \ +PGSERVICEFILE="${work_dir}/pg_service.conf" \ +BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ +BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ +BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ +BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ +BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ +BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets \ +bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}" + +test "$(wc -l < "${RESTORE_LOG}" | tr -d ' ')" = "2" +test -z "$(find "${work_dir}/restore-tmp" -mindepth 1 -print -quit)" + +echo "Brio encrypted PostgreSQL restore contract passed." diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 5f7d504..672c208 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -60,6 +60,8 @@ sql = read_required_text(repo_root / "bootstrap/keycloak-new-instances.sql", "SQ runtrace_sql = read_required_text(repo_root / "bootstrap/runtrace-app.sql", "Runtrace app SQL bootstrap") keycloak_runtrace_sql = read_required_text(repo_root / "bootstrap/keycloak-runtrace-app.sql", "targeted Runtrace Keycloak SQL bootstrap") openpanel_sql = read_required_text(repo_root / "bootstrap/openpanel-app.sql", "OpenPanel app SQL bootstrap") +brio_sql = read_required_text(repo_root / "bootstrap/brio-staging-app.sql", "Brio staging app SQL bootstrap") +keycloak_brio_sql = read_required_text(repo_root / "bootstrap/keycloak-brio-staging.sql", "targeted Brio Keycloak SQL bootstrap") readme = read_required_text(repo_root / "README.md", "README") base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") host_compose = read_required_text(repo_root / "compose.host.yml", "host Compose file") @@ -68,11 +70,22 @@ runtrace_backup = read_required_text(repo_root / "scripts/run-runtrace-backup.sh runtrace_backup_loop = read_required_text(repo_root / "scripts/run-runtrace-backup-loop.sh", "Runtrace backup loop") runtrace_restore = read_required_text(repo_root / "scripts/verify-runtrace-restore.sh", "Runtrace restore verifier") runtrace_backup_test = read_required_text(repo_root / "scripts/test-runtrace-backup.sh", "Runtrace backup contract test") +brio_backup_path = repo_root / "scripts/run-brio-encrypted-backup.sh" +brio_backup_loop_path = repo_root / "scripts/run-brio-encrypted-backup-loop.sh" +brio_restore_path = repo_root / "scripts/verify-brio-encrypted-restore.sh" +brio_backup_test_path = repo_root / "scripts/test-brio-encrypted-backup.sh" +brio_restore_test_path = repo_root / "scripts/test-brio-encrypted-restore.sh" +brio_backup = read_required_text(brio_backup_path, "Brio encrypted backup script") +brio_backup_loop = read_required_text(brio_backup_loop_path, "Brio encrypted backup loop") +brio_restore = read_required_text(brio_restore_path, "Brio encrypted restore verifier") +brio_backup_test = read_required_text(brio_backup_test_path, "Brio encrypted backup contract test") +brio_restore_test = read_required_text(brio_restore_test_path, "Brio encrypted restore contract test") canary_compose = read_required_text(repo_root / "envs/canary/compose.yml", "canary Compose override") production_compose = read_required_text(repo_root / "envs/production/compose.yml", "production Compose override") canary_env = read_required_text(repo_root / "envs/canary/.env.db", "canary database environment") production_env = read_required_text(repo_root / "envs/production/.env.db", "production database environment") manual_deploy = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") +ci_workflow = read_required_text(repo_root / ".github/workflows/ci.yml", "CI workflow") normalized_readme = re.sub(r"\s+", " ", readme) require("docker network create" not in sql, "SQL bootstrap must not manage Docker networks.") @@ -130,6 +143,8 @@ require("name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK}" in production_compose, "Produ for required in ("target: 5432", "published: 5432", "protocol: tcp", "mode: host"): require(required in production_compose, f"Production Compose must publish PostgreSQL for DB VM clients: {required}") require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy workflow must reject root SSH users.") +require("DEPLOY_BRIO_STAGING_DB_NETWORK must be makepad_brio_staging_db" in manual_deploy, "Manual deploy must reject a non-canonical Brio database network secret.") +require("Brio deployment bundle must use makepad_brio_staging_db" in manual_deploy, "Remote deploy must revalidate the canonical Brio database network.") require("postgres:16-alpine@sha256:" in base_compose, "Base Compose must pin PostgreSQL to an immutable digest.") require("pg_isready" in base_compose, "Base Compose must define a PostgreSQL healthcheck.") for required in ( @@ -157,6 +172,45 @@ for required in ( for database in ("runtrace", "keycloak_runtrace"): require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") require(re.search(rf"^hostssl\s+{database}\s+all\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must require TLS and SCRAM for {database}.") +for database, roles in ( + ("brio_staging", ("brio_staging_app", "brio_staging_backup")), + ("keycloak_brio_staging", ("keycloak_brio_staging_app", "keycloak_brio_staging_backup")), +): + require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") + for role in roles: + require(re.search(rf"^hostssl\s+{database}\s+{role}\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must allow TLS access to {database} for {role}.") + require(re.search(rf"^host\s+all\s+{role}\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject {role} from every non-target database.") +require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose Brio's certificate-matching database alias.") +require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach Brio's isolated database network.") +require("ensure_internal_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate Brio's internal encrypted database network.") +for content, role, database in ( + (brio_sql, "brio_staging_app", "brio_staging"), + (keycloak_brio_sql, "keycloak_brio_staging_app", "keycloak_brio_staging"), +): + require("NOBYPASSRLS" in content, f"{role} bootstrap must strip elevated role capabilities.") + require(f"REVOKE ALL ON DATABASE {database} FROM PUBLIC" in content, f"{database} must revoke default public database access.") + +for content, app_role, backup_role, database, password_variable in ( + (brio_sql, "brio_staging_app", "brio_staging_backup", "brio_staging", "brio_staging_backup_password"), + (keycloak_brio_sql, "keycloak_brio_staging_app", "keycloak_brio_staging_backup", "keycloak_brio_staging", "keycloak_brio_staging_backup_password"), +): + for required in ( + f"CREATE ROLE {backup_role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION", + f"ALTER ROLE {backup_role} LOGIN PASSWORD :'{password_variable}'", + f"ALTER ROLE {backup_role} NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2", + f"GRANT CONNECT ON DATABASE {database} TO {backup_role}", + f"ALTER ROLE {backup_role} IN DATABASE {database} SET default_transaction_read_only TO on", + f"GRANT USAGE ON SCHEMA public TO {backup_role}", + f"GRANT SELECT ON ALL TABLES IN SCHEMA public TO {backup_role}", + f"GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO {backup_role}", + f"ALTER DEFAULT PRIVILEGES FOR ROLE {app_role} IN SCHEMA public GRANT SELECT ON TABLES TO {backup_role}", + f"ALTER DEFAULT PRIVILEGES FOR ROLE {app_role} IN SCHEMA public GRANT SELECT ON SEQUENCES TO {backup_role}", + ): + require(required in content, f"{database} backup bootstrap is missing: {required}") + require(f"NULLIF(btrim(:'{password_variable}'), '')" in content, f"{database} backup bootstrap must reject empty passwords.") + require("inet_client_addr() IS NULL" in content, f"{database} bootstrap must allow a local Unix-domain socket.") + require("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()" in content, f"{database} bootstrap must reject remote plaintext administrator sessions.") + require(r"\quit 1" not in content and "SELECT 1 / 0;" in content, f"{database} bootstrap failure branches must terminate psql with a non-zero status.") for label, content in (("canary", canary_env), ("production", production_env)): require("POSTGRES_PASSWORD=" not in content, f"{label} database environment must not contain POSTGRES_PASSWORD.") require("POSTGRES_IMAGE=postgres:16-alpine@sha256:" in content, f"{label} database environment must pin POSTGRES_IMAGE.") @@ -237,6 +291,11 @@ require("DEPLOY_VIF_DB_NETWORK production environment secret" in manual_deploy, require('if [[ "${deploy_env}" == "production" ]]; then' in manual_deploy, "Manual deploy workflow must gate VIF setup to production.") require('if [[ "${vif_enabled}" != "1" ]]; then' in manual_deploy, "Manual deploy workflow must skip VIF provisioning outside production.") require("postgres_ready=0" in manual_deploy, "Manual deploy workflow must track Postgres readiness.") +require("wait_for_service_convergence" in manual_deploy, "Manual deploy must wait for exact-image Swarm task convergence.") +require("docker service ps --no-trunc --filter desired-state=running" in manual_deploy, "Manual deploy must inspect running task images rather than stale replica counts.") +require('wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}"' in manual_deploy, "Manual deploy must converge PostgreSQL before probing it.") +require('wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}"' in manual_deploy, "Canary deploy must converge the Brio application backup task.") +require('wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}"' in manual_deploy, "Production deploy must converge the Brio identity backup task.") require("Postgres did not become reachable via makepad-postgres-vif" in manual_deploy, "Manual deploy workflow must fail clearly when VIF readiness times out.") require( not re.search(r"\S\\gexec", manual_deploy), @@ -316,4 +375,154 @@ require("NULLIF(btrim(:'openpanel_app_password'), '')" in openpanel_sql, "OpenPa require("openpanel_app" in normalized_readme, "README must document the OpenPanel app role.") require("postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable" in readme, "README must document the OpenPanel DB VM host connection URI.") require("${OPENPANEL_DB_PASSWORD:?" in readme, "README bootstrap command must fail fast for OPENPANEL_DB_PASSWORD.") + +for expected in ( + "brio_staging_app", + "brio_staging", + "brio_staging_app_password", + "pg_advisory_lock", + "pg_advisory_unlock", + "CREATE DATABASE brio_staging OWNER brio_staging_app", + "ALTER DATABASE brio_staging OWNER TO brio_staging_app", +): + require(expected in brio_sql, f"Brio staging bootstrap is missing {expected}.") +require("PostgreSQL superuser connection" in brio_sql, "Brio staging bootstrap must document its superuser requirement.") +require("NULLIF(btrim(:'brio_staging_app_password'), '')" in brio_sql, "Brio staging bootstrap must reject empty passwords.") +require("NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION" in brio_sql, "Brio staging role must be least privilege.") +require("brio_staging_app" in normalized_readme, "README must document the Brio staging app role.") +require("keycloak_brio_staging_app" in normalized_readme, "README must document the Brio staging Keycloak role.") +require("${BRIO_STAGING_DB_PASSWORD:?" in readme, "README must fail fast for BRIO_STAGING_DB_PASSWORD.") +require("${KEYCLOAK_BRIO_STAGING_DB_PASSWORD:?" in readme, "README must fail fast for KEYCLOAK_BRIO_STAGING_DB_PASSWORD.") +require("${BRIO_STAGING_BACKUP_DB_PASSWORD:?" in readme, "README must fail fast for BRIO_STAGING_BACKUP_DB_PASSWORD.") +require("${KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD:?" in readme, "README must fail fast for KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD.") +require("refuse a remote plaintext administrator session" in normalized_readme, "README must document the Brio bootstrap transport guard.") +require("bootstrap/keycloak-brio-staging.sql" in readme, "README must document the targeted Brio Keycloak bootstrap.") +require("keycloak_brio_staging_app_password" not in sql, "The shared Keycloak bootstrap must not rotate the Brio staging credential.") +require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach the isolated Brio staging DB network.") +require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose the certificate-matching Brio DB alias.") +backup_image = "postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825" +require(f"BRIO_BACKUP_IMAGE={backup_image}" in canary_env, "Canary must pin the exact Brio backup image.") +require(f"BRIO_BACKUP_IMAGE={backup_image}" in production_env, "Production must pin the exact Brio backup image.") +for config_name in ("brio_encrypted_backup_script", "brio_encrypted_backup_loop_script"): + require(config_name in base_compose, f"Base Compose is missing Brio backup config {config_name}.") + +require(" brio_staging_backup:" in canary_compose, "Canary Compose must run the Brio application backup service.") +canary_backup_service = canary_compose.split(" brio_staging_backup:", 1)[1].split("\nnetworks:", 1)[0] +for required in ( + "BRIO_BACKUP_DATABASE: brio_staging", + "PGHOST: makepad-postgres-brio-staging", + "PGUSER: brio_staging_backup", + "PGSSLMODE: verify-full", + "BRIO_BACKUP_RETENTION_DAYS", + "BRIO_BACKUP_RECIPIENT_CERT", + "user: \"999:999\"", + "read_only: true", + "no-new-privileges:true", + "- brio_staging", +): + require(required in canary_backup_service, f"Canary Brio backup service is missing: {required}") +require("- db" not in canary_backup_service, "Canary Brio backup must attach only to Brio's isolated database network.") + +require(" keycloak_brio_staging_backup:" in production_compose, "Production Compose must run the Brio identity backup service.") +production_backup_service = production_compose.split(" keycloak_brio_staging_backup:", 1)[1].split("\nnetworks:", 1)[0] +for required in ( + "BRIO_BACKUP_DATABASE: keycloak_brio_staging", + "PGHOST: makepad-postgres", + "PGUSER: keycloak_brio_staging_backup", + "PGSSLMODE: verify-full", + "BRIO_BACKUP_RETENTION_DAYS", + "BRIO_BACKUP_RECIPIENT_CERT", + "user: \"999:999\"", + "read_only: true", + "no-new-privileges:true", + "- db", +): + require(required in production_backup_service, f"Production Brio identity backup service is missing: {required}") +require("BRIO_RESTORE_RECIPIENT_KEY" not in canary_compose + production_compose + host_compose, "Backup services must never mount the Brio recovery private key.") +for required in ( + "keycloak_brio_staging_backup:", + "BRIO_BACKUP_DATABASE: keycloak_brio_staging", + "MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST", + "PGUSER: keycloak_brio_staging_backup", + "PGSSLMODE: verify-full", +): + require(required in host_compose, f"Standalone DB-VM Compose is missing Brio identity backup control: {required}") + +for path in (brio_backup_path, brio_backup_loop_path, brio_restore_path, brio_backup_test_path, brio_restore_test_path): + require(os.access(path, os.X_OK), f"Brio backup/restore script must be executable: {path}") +for required in ( + "brio_staging) expected_pg_user=brio_staging_backup", + "keycloak_brio_staging) expected_pg_user=keycloak_brio_staging_backup", + "BRIO_BACKUP_RETENTION_DAYS:-35", + "Brio backup retention must remain exactly 35 days", + "retention_find_days=$((retention_days - 1))", + "mkfifo", + "pg_dump", + "openssl cms", + "-aes-256-gcm", + ".dump.cms", + "PGSSLMODE=verify-full", + "PGUSER must identify the database-specific Brio backup role", + "mv \"${partial_dir}\" \"${final_dir}\"", + "sha256sum \"${encrypted_name}\" metadata.json", +): + require(required in brio_backup, f"Brio encrypted backup script is missing: {required}") +require("--file=" not in brio_backup, "Brio backup must stream pg_dump instead of writing a plaintext dump file.") +require("BRIO_RESTORE_RECIPIENT_KEY" not in brio_backup, "Brio backup service must not receive the recovery private key.") +require("PGUSER=postgres" not in canary_backup_service + production_backup_service, "Brio backup services must never run as the PostgreSQL superuser.") +require("healthcheck" in brio_backup_loop and "interval_seconds * 2" in brio_backup_loop, "Brio backup health check must enforce freshness.") +for required in ( + "replace-nonproduction-brio-restore-targets", + "BRIO_APP_RESTORE_SERVICE", + "BRIO_KEYCLOAK_RESTORE_SERVICE", + "BRIO_RESTORE_RECIPIENT_CERT", + "BRIO_RESTORE_RECIPIENT_KEY", + "BRIO_RESTORE_TEMP_ROOT", + "*_restore_test", + "SELECT current_database();", + "openssl cms", + "-decrypt", + "--single-transaction", + "--exit-on-error", + "public.schema_migrations", + "public.communities", + "public.realm", +): + require(required in brio_restore, f"Brio encrypted restore verifier is missing: {required}") +require("run-brio-encrypted-backup.sh" in brio_backup_test, "Brio backup contract test must execute the real backup script.") +require("verify-brio-encrypted-restore.sh" in brio_restore_test, "Brio restore contract test must execute the real restore verifier.") +for required in ("test-brio-bootstrap.sh", "test-brio-encrypted-backup.sh", "test-brio-encrypted-restore.sh"): + require(required in ci_workflow, f"CI must run the Brio PostgreSQL contract: {required}") +for required in ( + "independently administered off-host storage", + "successful recorded restore of both databases remain external release gates", + "private key must never be copied to a database host", + "scripts/test-brio-encrypted-backup.sh", + "scripts/test-brio-encrypted-restore.sh", +): + require(required in normalized_readme, f"README is missing Brio backup/restore guidance: {required}") + +for required in ( + "-checkhost makepad-postgres-brio-staging", + "-checkend 604800", + "openssl verify -purpose sslserver", + "PGSSLMODE=verify-full", + "-h makepad-postgres-brio-staging", + "cp scripts/run-brio-encrypted-backup.sh", + "cp scripts/run-brio-encrypted-backup-loop.sh", + "brio_backup_directory_mode", + "brio_backup_password_mode", + "brio_backup_recipient_mode", + "uid 999 with mode 0700", + "uid 999 with mode 0400", + "openssl cms -encrypt", +): + require(required in manual_deploy, f"Manual deploy is missing Brio certificate/connection preflight marker: {required}") +for policy in ( + "hostnossl brio_staging", + "hostnossl keycloak_brio_staging", + "hostssl brio_staging", + "hostssl keycloak_brio_staging", +): + require(policy in runtrace_hba, f"PostgreSQL HBA policy is missing {policy}.") PY diff --git a/scripts/verify-brio-encrypted-restore.sh b/scripts/verify-brio-encrypted-restore.sh new file mode 100755 index 0000000..96f8fb9 --- /dev/null +++ b/scripts/verify-brio-encrypted-restore.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +app_backup_dir=$1 +keycloak_backup_dir=$2 +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a protected libpq service file}" +: "${BRIO_APP_RESTORE_SERVICE:?BRIO_APP_RESTORE_SERVICE must name an explicit non-production database}" +: "${BRIO_KEYCLOAK_RESTORE_SERVICE:?BRIO_KEYCLOAK_RESTORE_SERVICE must name an explicit non-production database}" +: "${BRIO_RESTORE_RECIPIENT_CERT:?BRIO_RESTORE_RECIPIENT_CERT must identify the external recipient certificate}" +: "${BRIO_RESTORE_RECIPIENT_KEY:?BRIO_RESTORE_RECIPIENT_KEY must identify the external recipient private key}" +: "${BRIO_RESTORE_TEMP_ROOT:?BRIO_RESTORE_TEMP_ROOT must identify a mode-0700 temporary storage directory}" +: "${BRIO_RESTORE_CONFIRM:?set BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets}" + +if [[ "${BRIO_RESTORE_CONFIRM}" != "replace-nonproduction-brio-restore-targets" ]]; then + echo "Brio restore verification requires explicit non-production replacement confirmation." >&2 + exit 1 +fi +if [[ "${BRIO_APP_RESTORE_SERVICE}" == "${BRIO_KEYCLOAK_RESTORE_SERVICE}" ]]; then + echo "Brio application and Keycloak restore services must be different." >&2 + exit 1 +fi +for service_name in "${BRIO_APP_RESTORE_SERVICE}" "${BRIO_KEYCLOAK_RESTORE_SERVICE}"; do + if [[ ! "${service_name}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "Restore service names contain unsupported characters." >&2 + exit 1 + fi + if [[ "${service_name}" != *_restore_test ]]; then + echo "Restore service names must end in _restore_test." >&2 + exit 1 + fi +done +for command_name in openssl sha256sum pg_restore psql mktemp; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "Missing required restore command: ${command_name}" >&2 + exit 1 + fi +done + +validate_protected_file() { + local path=$1 + local label=$2 + if [[ ! -s "${path}" || -L "${path}" ]]; then + echo "${label} must be a non-empty, non-symlink file." >&2 + exit 1 + fi + local mode + mode=$(stat -c '%a' "${path}" 2>/dev/null || stat -f '%Lp' "${path}") + if (( (8#${mode} & 8#022) != 0 )); then + echo "${label} must not be group- or world-writable." >&2 + exit 1 + fi +} + +validate_protected_file "${PGSERVICEFILE}" "PGSERVICEFILE" +validate_protected_file "${BRIO_RESTORE_RECIPIENT_CERT}" "Brio restore recipient certificate" +validate_protected_file "${BRIO_RESTORE_RECIPIENT_KEY}" "Brio restore recipient private key" +service_file_mode=$(stat -c '%a' "${PGSERVICEFILE}" 2>/dev/null || stat -f '%Lp' "${PGSERVICEFILE}") +if (( (8#${service_file_mode} & 8#077) != 0 )); then + echo "PGSERVICEFILE must not be accessible to group or other users." >&2 + exit 1 +fi +key_mode=$(stat -c '%a' "${BRIO_RESTORE_RECIPIENT_KEY}" 2>/dev/null || stat -f '%Lp' "${BRIO_RESTORE_RECIPIENT_KEY}") +if (( (8#${key_mode} & 8#077) != 0 )); then + echo "Brio restore recipient private key must not be accessible to group or other users." >&2 + exit 1 +fi +if [[ ! -d "${BRIO_RESTORE_TEMP_ROOT}" || -L "${BRIO_RESTORE_TEMP_ROOT}" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must be a pre-provisioned non-symlink directory." >&2 + exit 1 +fi +temp_mode=$(stat -c '%a' "${BRIO_RESTORE_TEMP_ROOT}" 2>/dev/null || stat -f '%Lp' "${BRIO_RESTORE_TEMP_ROOT}") +if [[ "${temp_mode}" != "700" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must have mode 0700." >&2 + exit 1 +fi +temp_owner=$(stat -c '%u' "${BRIO_RESTORE_TEMP_ROOT}" 2>/dev/null || stat -f '%u' "${BRIO_RESTORE_TEMP_ROOT}") +if [[ "${temp_owner}" != "$(id -u)" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must be owned by the restore operator." >&2 + exit 1 +fi + +cert_public_key=$(openssl x509 -in "${BRIO_RESTORE_RECIPIENT_CERT}" -pubkey -noout \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | sha256sum | awk '{print $1}') +private_public_key=$(openssl pkey -in "${BRIO_RESTORE_RECIPIENT_KEY}" -pubout -outform DER 2>/dev/null \ + | sha256sum | awk '{print $1}') +if [[ -z "${cert_public_key}" || "${cert_public_key}" != "${private_public_key}" ]]; then + echo "Brio restore recipient certificate and private key do not match." >&2 + exit 1 +fi + +validate_bundle() { + local backup_dir=$1 + local database=$2 + local encrypted_name=${database}.dump.cms + if [[ ! -d "${backup_dir}" || -L "${backup_dir}" ]]; then + echo "Backup directory must be a regular directory and not a symlink: ${backup_dir}" >&2 + exit 1 + fi + for required in "${encrypted_name}" SHA256SUMS metadata.json; do + if [[ ! -s "${backup_dir}/${required}" || -L "${backup_dir}/${required}" ]]; then + echo "Encrypted backup artifact is missing, empty, or a symlink: ${required}" >&2 + exit 1 + fi + done + ( + cd "${backup_dir}" + sha256sum --check --strict SHA256SUMS + ) + if ! grep -Fq "\"database\":\"${database}\"" "${backup_dir}/metadata.json" \ + || ! grep -Fq '"envelope":"openssl-cms-der"' "${backup_dir}/metadata.json" \ + || ! grep -Fq '"cipher":"aes-256-gcm"' "${backup_dir}/metadata.json"; then + echo "Encrypted backup metadata does not match ${database}." >&2 + exit 1 + fi + if ! openssl cms -cmsout -inform DER -in "${backup_dir}/${encrypted_name}" -noout >/dev/null 2>&1; then + echo "Encrypted backup is not a valid CMS envelope for ${database}." >&2 + exit 1 + fi +} + +validate_bundle "${app_backup_dir}" brio_staging +validate_bundle "${keycloak_backup_dir}" keycloak_brio_staging + +restore_dir=$(mktemp -d "${BRIO_RESTORE_TEMP_ROOT%/}/brio-restore.XXXXXX") +chmod 0700 "${restore_dir}" +cleanup() { + find "${restore_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${restore_dir}" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 1' HUP INT TERM + +decrypt_bundle() { + local backup_dir=$1 + local database=$2 + local output=$3 + openssl cms \ + -decrypt \ + -binary \ + -inform DER \ + -recip "${BRIO_RESTORE_RECIPIENT_CERT}" \ + -inkey "${BRIO_RESTORE_RECIPIENT_KEY}" \ + -in "${backup_dir}/${database}.dump.cms" \ + -out "${output}" + chmod 0600 "${output}" + pg_restore --list "${output}" >/dev/null +} + +app_dump=${restore_dir}/brio_staging.dump +keycloak_dump=${restore_dir}/keycloak_brio_staging.dump +decrypt_bundle "${app_backup_dir}" brio_staging "${app_dump}" +decrypt_bundle "${keycloak_backup_dir}" keycloak_brio_staging "${keycloak_dump}" + +restore_dump() { + local service=$1 + local dump=$2 + local database + database=$(psql "service=${service}" -v ON_ERROR_STOP=1 -Atc "SELECT current_database();") + if [[ "${database}" != *_restore_test ]]; then + echo "Restore service ${service} resolved to a database without the _restore_test suffix." >&2 + exit 1 + fi + pg_restore \ + --dbname="service=${service}" \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl \ + --exit-on-error \ + --single-transaction \ + "${dump}" +} + +restore_dump "${BRIO_APP_RESTORE_SERVICE}" "${app_dump}" +restore_dump "${BRIO_KEYCLOAK_RESTORE_SERVICE}" "${keycloak_dump}" + +app_state=$(psql "service=${BRIO_APP_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.schema_migrations') IS NOT NULL AND to_regclass('public.communities') IS NOT NULL;") +keycloak_state=$(psql "service=${BRIO_KEYCLOAK_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.realm') IS NOT NULL;") +if [[ "${app_state}" != "t" || "${keycloak_state}" != "t" ]]; then + echo "Restored databases are missing Brio or Keycloak durable state tables." >&2 + exit 1 +fi + +echo "Brio application and Keycloak encrypted restore verification completed against non-production targets." From f439f935469cedbb53a9b48c6d765f17de9a8e07 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 16:52:35 +0200 Subject: [PATCH 02/20] ci: harden PostgreSQL deployment validation --- .github/workflows/ci.yml | 1 + .github/workflows/manual-deploy.yml | 349 +--------------------------- README.md | 3 +- scripts/deploy-postgres-stack.sh | 346 +++++++++++++++++++++++++++ scripts/test-brio-bootstrap.sh | 25 +- scripts/validate-postgres-config.sh | 13 +- 6 files changed, 392 insertions(+), 345 deletions(-) create mode 100755 scripts/deploy-postgres-stack.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0583c16..41c58b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: shellcheck scripts/run-brio-encrypted-backup.sh scripts/run-brio-encrypted-backup-loop.sh + scripts/deploy-postgres-stack.sh scripts/verify-brio-encrypted-restore.sh scripts/test-brio-bootstrap.sh scripts/test-brio-encrypted-backup.sh diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 4280f1a..b1cdf6a 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -73,6 +73,7 @@ jobs: cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh" cp scripts/run-brio-encrypted-backup.sh "${bundle_root}/scripts/run-brio-encrypted-backup.sh" cp scripts/run-brio-encrypted-backup-loop.sh "${bundle_root}/scripts/run-brio-encrypted-backup-loop.sh" + cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <&2 - exit 1 - fi - postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") - if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then - echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 - exit 1 - fi - for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do - if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then - echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 - exit 1 - fi - done - if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then - echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 - exit 1 - fi - brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") - brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") - if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then - echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 - exit 1 - fi - if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then - echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 - exit 1 - fi - brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") - brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") - if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then - echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 - exit 1 - fi - if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then - echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 - exit 1 - fi - brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") - brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") - if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then - echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 - exit 1 - fi - if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ - || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then - echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 - exit 1 - fi - server_certificate=$(mktemp) - cleanup_server_certificate() { rm -f "${server_certificate}"; } - trap cleanup_server_certificate EXIT - docker config inspect "${postgres_tls_cert_config}" --format '{{printf "%s" .Spec.Data}}' > "${server_certificate}" - if ! openssl x509 -in "${server_certificate}" -noout -checkend 604800 >/dev/null; then - echo "PostgreSQL TLS certificate is invalid or expires in less than seven days." >&2 - exit 1 - fi - if ! openssl verify -purpose sslserver -CAfile "${postgres_ca_cert_file}" -untrusted "${server_certificate}" "${server_certificate}" >/dev/null; then - echo "PostgreSQL TLS certificate does not chain to the configured CA." >&2 - exit 1 - fi - if [[ "${deploy_env}" == "canary" ]] && ! openssl x509 -in "${server_certificate}" -noout -checkhost makepad-postgres-brio-staging >/dev/null; then - echo "Canary PostgreSQL TLS certificate does not cover makepad-postgres-brio-staging." >&2 - exit 1 - fi - if [[ "${deploy_env}" == "production" ]]; then - if [[ ! -d "${runtrace_backup_path}" || -L "${runtrace_backup_path}" ]]; then - echo "Runtrace backup path must be a pre-provisioned non-symlink directory: ${runtrace_backup_path}" >&2 - exit 1 - fi - backup_directory_mode=$(stat -c '%a' "${runtrace_backup_path}") - backup_directory_uid=$(stat -c '%u' "${runtrace_backup_path}") - if [[ "${backup_directory_mode}" != "700" || "${backup_directory_uid}" != "70" ]]; then - echo "Runtrace backup path must be owned by uid 70 with mode 0700: ${runtrace_backup_path}" >&2 - exit 1 - fi - if [[ ! -s "${runtrace_backup_password_file}" || -L "${runtrace_backup_password_file}" ]]; then - echo "Runtrace backup credential must be a non-empty, non-symlink file: ${runtrace_backup_password_file}" >&2 - exit 1 - fi - backup_password_mode=$(stat -c '%a' "${runtrace_backup_password_file}") - backup_password_uid=$(stat -c '%u' "${runtrace_backup_password_file}") - if [[ "${backup_password_mode}" != "400" || "${backup_password_uid}" != "70" ]]; then - echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2 - exit 1 - fi - fi - hba_path="${remote_dir}/config/runtrace-pg_hba.conf" - hba_sha256=$(sha256sum "${hba_path}" | awk '{print $1}') - if docker config inspect "${postgres_runtrace_hba_config}" >/dev/null 2>&1; then - deployed_hba_sha256=$(docker config inspect "${postgres_runtrace_hba_config}" --format '{{index .Spec.Labels "content-sha256"}}') - if [[ "${deployed_hba_sha256}" != "${hba_sha256}" ]]; then - echo "PostgreSQL HBA config ${postgres_runtrace_hba_config} does not match the repository policy. Create a new versioned config name and update MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG." >&2 - exit 1 - fi - else - docker config create --label "content-sha256=${hba_sha256}" "${postgres_runtrace_hba_config}" "${hba_path}" >/dev/null - fi - if [[ "${vif_enabled}" == "1" ]]; then - : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" - : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" - : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" - fi - if [[ "${brio_staging_enabled}" == "1" ]]; then - : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" - if [[ "${brio_staging_db_network}" != "makepad_brio_staging_db" ]]; then - echo "Brio deployment bundle must use makepad_brio_staging_db." >&2 - exit 1 - fi - fi - - ensure_encrypted_overlay_network() { - local network_name=$1 - if docker network inspect "${network_name}" >/dev/null 2>&1; then - local driver scope encrypted - driver=$(docker network inspect "${network_name}" --format '{{.Driver}}') - scope=$(docker network inspect "${network_name}" --format '{{.Scope}}') - encrypted=$(docker network inspect "${network_name}" --format '{{index .Options "encrypted"}}') - if [[ "${driver}" != "overlay" || "${scope}" != "swarm" || "${encrypted}" != "true" ]]; then - echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted, then rerun this deployment." >&2 - exit 1 - fi - return - fi - docker network create --driver overlay --attachable --opt encrypted "${network_name}" >/dev/null - } - - ensure_internal_encrypted_overlay_network() { - local network_name=$1 - if ! docker network inspect "${network_name}" >/dev/null 2>&1; then - docker network create --driver overlay --attachable --internal --opt encrypted "${network_name}" >/dev/null - fi - local details - details=$(docker network inspect "${network_name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') - if [[ "${details}" != "overlay swarm true true true" ]]; then - echo "Brio database network ${network_name} must be an internal, encrypted, attachable Swarm overlay; got ${details}." >&2 - exit 1 - fi - } + scp "${scp_opts[@]}" "${bundle_root}/scripts/deploy-postgres-stack.sh" "${remote_target}:${REMOTE_DIR}/scripts/deploy-postgres-stack.sh" - ensure_encrypted_overlay_network "${db_network}" - ensure_encrypted_overlay_network "${le_petit_coin_db_network}" - if [[ "${vif_enabled}" == "1" ]]; then - ensure_encrypted_overlay_network "${vif_db_network}" - export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" - fi - if [[ "${brio_staging_enabled}" == "1" ]]; then - ensure_internal_encrypted_overlay_network "${brio_staging_db_network}" - export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_staging_db_network}" - fi - - export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" - export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" - docker compose \ - --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ - --env-file "${env_deploy}" \ - -f "${remote_dir}/compose.yml" \ - -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ - config > "${remote_dir}/stack.yml" - - docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" - - wait_for_service_convergence() { - local service_name=$1 - local expected_image=$2 - local update_state desired running_snapshot running_count wrong_image - for attempt in $(seq 1 60); do - if ! docker service inspect "${service_name}" >/dev/null 2>&1; then - sleep 2 - continue - fi - update_state=$(docker service inspect "${service_name}" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{else}}none{{end}}') - case "${update_state}" in - paused|rollback_started|rollback_paused|rollback_completed) - echo "Service ${service_name} update did not complete successfully: ${update_state}." >&2 - docker service ps --no-trunc "${service_name}" >&2 - return 1 - ;; - updating) - sleep 2 - continue - ;; - esac - desired=$(docker service inspect "${service_name}" --format '{{.Spec.Mode.Replicated.Replicas}}') - running_snapshot=$(docker service ps --no-trunc --filter desired-state=running --format '{{.Image}} {{.CurrentState}}' "${service_name}") - running_count=$(printf '%s\n' "${running_snapshot}" | awk '$2 == "Running" {count++} END {print count + 0}') - wrong_image=$(printf '%s\n' "${running_snapshot}" | awk -v expected="${expected_image}" '$2 == "Running" && $1 != expected {print $1; exit}') - if [[ "${running_count}" == "${desired}" && -z "${wrong_image}" && ( "${update_state}" == "completed" || "${update_state}" == "none" ) ]]; then - return 0 - fi - sleep 2 - done - echo "Service ${service_name} did not converge to ${expected_image}." >&2 - docker service ps --no-trunc "${service_name}" >&2 || true - return 1 - } - - wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}" - if [[ "${deploy_env}" == "canary" ]]; then - wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}" - else - wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}" - fi - - if [[ "${brio_staging_enabled}" == "1" ]]; then - brio_tls_ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --network "${brio_staging_db_network}" \ - -e PGSSLMODE=verify-full \ - -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ - -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - -v "${postgres_ca_cert_file}:/etc/postgresql/ca.crt:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-brio-staging -U "${postgres_root_user}" -d postgres -Atc "select 1" >/dev/null 2>&1; then - brio_tls_ready=1 - break - fi - sleep 2 - done - if [[ "${brio_tls_ready}" != "1" ]]; then - echo "PostgreSQL did not pass sslmode=verify-full using makepad-postgres-brio-staging within 60 seconds." >&2 - exit 1 - fi - fi - - if [[ "${vif_enabled}" != "1" ]]; then - exit 0 - fi - - postgres_ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --network "${vif_db_network}" \ - -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres -c "select 1" >/dev/null 2>&1; then - postgres_ready=1 - break - fi - sleep 2 - done - if [[ "${postgres_ready}" != "1" ]]; then - echo "Postgres did not become reachable via makepad-postgres-vif on ${vif_db_network} after 60 seconds." >&2 - exit 1 - fi - - docker run --rm --network "${vif_db_network}" \ - -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres \ - -v ON_ERROR_STOP=1 \ - -v vif_db="${vif_db_name}" \ - -v vif_user="${vif_db_user}" \ - -v vif_password="${vif_db_password}" <<'SQL' - SELECT format('CREATE ROLE %I LOGIN', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec - SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec - SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec - SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') - WHERE EXISTS ( - SELECT 1 - FROM pg_database d - JOIN pg_roles r ON r.oid = d.datdba - WHERE d.datname = :'vif_db' - AND r.rolname <> :'vif_user' - ) \gexec - SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec - SQL - EOF + printf -v remote_script_q %q "${REMOTE_DIR}/scripts/deploy-postgres-stack.sh" + printf -v remote_dir_q %q "${REMOTE_DIR}" + printf -v stack_name_q %q "${STACK_NAME}" + printf -v deploy_env_q %q "${{ inputs.environment }}" + # Values are intentionally expanded locally and shell-escaped with %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_dir_q} ${stack_name_q} ${deploy_env_q}" diff --git a/README.md b/README.md index 13a6601..7284ce4 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets - `scripts/run-brio-encrypted-backup.sh`: streaming CMS-encrypted backup for one allowlisted Brio database - `scripts/verify-brio-encrypted-restore.sh`: destructive two-database Brio restore verification +- `scripts/deploy-postgres-stack.sh`: checked-in remote Swarm preflight, deployment, convergence, and database-provisioning entrypoint ## Networks @@ -122,7 +123,7 @@ docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-serv The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging` and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. -The workflow deploys only the PostgreSQL stack. Before deployment it validates +The workflow copies the checked-in remote deployment entrypoint with the deployment bundle and deploys only the PostgreSQL stack. Before deployment it validates the password and CA files, certificate chain, seven-day expiry margin, and—for canary—the exact `makepad-postgres-brio-staging` SAN. After the stack update it performs a real `sslmode=verify-full` query over Brio's isolated network using diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh new file mode 100755 index 0000000..abe70f9 --- /dev/null +++ b/scripts/deploy-postgres-stack.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 3)); then + echo "Usage: deploy-postgres-stack.sh " >&2 + exit 2 +fi + +remote_dir=$1 +stack_name=$2 +deploy_env=$3 +env_deploy="${remote_dir}/envs/${deploy_env}/.env.deploy" +db_env="${remote_dir}/envs/${deploy_env}/.env.db" +db_network=$(grep '^MAKEPAD_POSTGRES_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) +le_petit_coin_db_network=$(grep '^MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) +postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) +brio_backup_image=$(grep '^BRIO_BACKUP_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_root_user=$(grep '^POSTGRES_USER=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_root_password_file=$(grep '^MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_tls_cert_config=$(grep '^MAKEPAD_POSTGRES_TLS_CERT_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_tls_key_secret=$(grep '^MAKEPAD_POSTGRES_TLS_KEY_SECRET=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_runtrace_hba_config=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) +runtrace_backup_path=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +runtrace_backup_password_file=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_ca_cert_file=$(grep '^MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +brio_backup_recipient_cert=$(grep '^MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +vif_enabled=0 +brio_staging_enabled=0 +if [[ "${deploy_env}" == "canary" ]]; then + brio_staging_enabled=1 + brio_staging_db_network=$(grep '^MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + brio_backup_path=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_backup_password_file=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +fi +if [[ "${deploy_env}" == "production" ]]; then + vif_enabled=1 + vif_db_network=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_name=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NAME=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_user=$(grep '^MAKEPAD_POSTGRES_VIF_DB_USER=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_password=$(grep '^MAKEPAD_POSTGRES_VIF_DB_PASSWORD=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + brio_backup_path=$(grep '^MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_backup_password_file=$(grep '^MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +fi +: "${db_network:?MAKEPAD_POSTGRES_DB_NETWORK is missing or empty in ${env_deploy}}" +: "${le_petit_coin_db_network:?MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK is missing or empty in ${env_deploy}}" +: "${postgres_image:?POSTGRES_IMAGE is missing or empty in ${db_env}}" +: "${brio_backup_image:?BRIO_BACKUP_IMAGE is missing or empty in ${db_env}}" +: "${postgres_root_user:?POSTGRES_USER is missing or empty in ${db_env}}" +: "${postgres_root_password_file:?MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" +: "${postgres_tls_cert_config:?MAKEPAD_POSTGRES_TLS_CERT_CONFIG is missing or empty in ${db_env}}" +: "${postgres_tls_key_secret:?MAKEPAD_POSTGRES_TLS_KEY_SECRET is missing or empty in ${db_env}}" +: "${postgres_runtrace_hba_config:?MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG is missing or empty in ${db_env}}" +: "${postgres_ca_cert_file:?MAKEPAD_POSTGRES_CA_CERT_HOST_PATH is missing or empty in ${db_env}}" +: "${brio_backup_recipient_cert:?MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH is missing or empty in ${db_env}}" +: "${brio_backup_path:?Brio backup directory is missing or empty in ${db_env}}" +: "${brio_backup_password_file:?Brio backup password-file path is missing or empty in ${db_env}}" +if [[ "${deploy_env}" == "production" ]]; then + : "${runtrace_backup_path:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH is missing or empty in ${db_env}}" + : "${runtrace_backup_password_file:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" +fi +if [[ ! -s "${postgres_root_password_file}" ]]; then + echo "PostgreSQL superuser password file is missing or empty: ${postgres_root_password_file}" >&2 + exit 1 +fi +if ! docker config inspect "${postgres_tls_cert_config}" >/dev/null 2>&1; then + echo "PostgreSQL TLS certificate config does not exist: ${postgres_tls_cert_config}" >&2 + exit 1 +fi +if ! docker secret inspect "${postgres_tls_key_secret}" >/dev/null 2>&1; then + echo "PostgreSQL TLS private-key secret does not exist: ${postgres_tls_key_secret}" >&2 + exit 1 +fi +command -v openssl >/dev/null 2>&1 || { + echo "openssl is required for PostgreSQL certificate preflight." >&2 + exit 1 +} +if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then + echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2 + exit 1 +fi +postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") +if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then + echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 + exit 1 +fi +for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then + echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 + exit 1 + fi +done +if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then + echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 + exit 1 +fi +brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") +brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") +if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then + echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 + exit 1 +fi +if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then + echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 + exit 1 +fi +brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") +brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") +if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then + echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 + exit 1 +fi +if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then + echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 + exit 1 +fi +brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") +brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") +if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then + echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 + exit 1 +fi +if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then + echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 + exit 1 +fi +server_certificate=$(mktemp) +cleanup_server_certificate() { rm -f "${server_certificate}"; } +trap cleanup_server_certificate EXIT +docker config inspect "${postgres_tls_cert_config}" --format '{{printf "%s" .Spec.Data}}' > "${server_certificate}" +if ! openssl x509 -in "${server_certificate}" -noout -checkend 604800 >/dev/null; then + echo "PostgreSQL TLS certificate is invalid or expires in less than seven days." >&2 + exit 1 +fi +if ! openssl verify -purpose sslserver -CAfile "${postgres_ca_cert_file}" -untrusted "${server_certificate}" "${server_certificate}" >/dev/null; then + echo "PostgreSQL TLS certificate does not chain to the configured CA." >&2 + exit 1 +fi +if [[ "${deploy_env}" == "canary" ]] && ! openssl x509 -in "${server_certificate}" -noout -checkhost makepad-postgres-brio-staging >/dev/null; then + echo "Canary PostgreSQL TLS certificate does not cover makepad-postgres-brio-staging." >&2 + exit 1 +fi +if [[ "${deploy_env}" == "production" ]]; then + if [[ ! -d "${runtrace_backup_path}" || -L "${runtrace_backup_path}" ]]; then + echo "Runtrace backup path must be a pre-provisioned non-symlink directory: ${runtrace_backup_path}" >&2 + exit 1 + fi + backup_directory_mode=$(stat -c '%a' "${runtrace_backup_path}") + backup_directory_uid=$(stat -c '%u' "${runtrace_backup_path}") + if [[ "${backup_directory_mode}" != "700" || "${backup_directory_uid}" != "70" ]]; then + echo "Runtrace backup path must be owned by uid 70 with mode 0700: ${runtrace_backup_path}" >&2 + exit 1 + fi + if [[ ! -s "${runtrace_backup_password_file}" || -L "${runtrace_backup_password_file}" ]]; then + echo "Runtrace backup credential must be a non-empty, non-symlink file: ${runtrace_backup_password_file}" >&2 + exit 1 + fi + backup_password_mode=$(stat -c '%a' "${runtrace_backup_password_file}") + backup_password_uid=$(stat -c '%u' "${runtrace_backup_password_file}") + if [[ "${backup_password_mode}" != "400" || "${backup_password_uid}" != "70" ]]; then + echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2 + exit 1 + fi +fi +hba_path="${remote_dir}/config/runtrace-pg_hba.conf" +hba_sha256=$(sha256sum "${hba_path}" | awk '{print $1}') +if docker config inspect "${postgres_runtrace_hba_config}" >/dev/null 2>&1; then + deployed_hba_sha256=$(docker config inspect "${postgres_runtrace_hba_config}" --format '{{index .Spec.Labels "content-sha256"}}') + if [[ "${deployed_hba_sha256}" != "${hba_sha256}" ]]; then + echo "PostgreSQL HBA config ${postgres_runtrace_hba_config} does not match the repository policy. Create a new versioned config name and update MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG." >&2 + exit 1 + fi +else + docker config create --label "content-sha256=${hba_sha256}" "${postgres_runtrace_hba_config}" "${hba_path}" >/dev/null +fi +if [[ "${vif_enabled}" == "1" ]]; then + : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" + : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" + : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" + : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" +fi +if [[ "${brio_staging_enabled}" == "1" ]]; then + : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" + if [[ "${brio_staging_db_network}" != "makepad_brio_staging_db" ]]; then + echo "Brio deployment bundle must use makepad_brio_staging_db." >&2 + exit 1 + fi +fi + +ensure_encrypted_overlay_network() { + local network_name=$1 + if docker network inspect "${network_name}" >/dev/null 2>&1; then + local driver scope encrypted + driver=$(docker network inspect "${network_name}" --format '{{.Driver}}') + scope=$(docker network inspect "${network_name}" --format '{{.Scope}}') + encrypted=$(docker network inspect "${network_name}" --format '{{index .Options "encrypted"}}') + if [[ "${driver}" != "overlay" || "${scope}" != "swarm" || "${encrypted}" != "true" ]]; then + echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted, then rerun this deployment." >&2 + exit 1 + fi + return + fi + docker network create --driver overlay --attachable --opt encrypted "${network_name}" >/dev/null +} + +ensure_internal_encrypted_overlay_network() { + local network_name=$1 + if ! docker network inspect "${network_name}" >/dev/null 2>&1; then + docker network create --driver overlay --attachable --internal --opt encrypted "${network_name}" >/dev/null + fi + local details + details=$(docker network inspect "${network_name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') + if [[ "${details}" != "overlay swarm true true true" ]]; then + echo "Brio database network ${network_name} must be an internal, encrypted, attachable Swarm overlay; got ${details}." >&2 + exit 1 + fi +} + +ensure_encrypted_overlay_network "${db_network}" +ensure_encrypted_overlay_network "${le_petit_coin_db_network}" +if [[ "${vif_enabled}" == "1" ]]; then + ensure_encrypted_overlay_network "${vif_db_network}" + export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" +fi +if [[ "${brio_staging_enabled}" == "1" ]]; then + ensure_internal_encrypted_overlay_network "${brio_staging_db_network}" + export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_staging_db_network}" +fi + +export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" +export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" +docker compose \ + --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ + --env-file "${env_deploy}" \ + -f "${remote_dir}/compose.yml" \ + -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ + config > "${remote_dir}/stack.yml" + +docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" + +wait_for_service_convergence() { + local service_name=$1 + local expected_image=$2 + local update_state desired running_snapshot running_count wrong_image + for _ in $(seq 1 60); do + if ! docker service inspect "${service_name}" >/dev/null 2>&1; then + sleep 2 + continue + fi + update_state=$(docker service inspect "${service_name}" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{else}}none{{end}}') + case "${update_state}" in + paused|rollback_started|rollback_paused|rollback_completed) + echo "Service ${service_name} update did not complete successfully: ${update_state}." >&2 + docker service ps --no-trunc "${service_name}" >&2 + return 1 + ;; + updating) + sleep 2 + continue + ;; + esac + desired=$(docker service inspect "${service_name}" --format '{{.Spec.Mode.Replicated.Replicas}}') + running_snapshot=$(docker service ps --no-trunc --filter desired-state=running --format '{{.Image}} {{.CurrentState}}' "${service_name}") + running_count=$(printf '%s\n' "${running_snapshot}" | awk '$2 == "Running" {count++} END {print count + 0}') + wrong_image=$(printf '%s\n' "${running_snapshot}" | awk -v expected="${expected_image}" '$2 == "Running" && $1 != expected {print $1; exit}') + if [[ "${running_count}" == "${desired}" && -z "${wrong_image}" && ( "${update_state}" == "completed" || "${update_state}" == "none" ) ]]; then + return 0 + fi + sleep 2 + done + echo "Service ${service_name} did not converge to ${expected_image}." >&2 + docker service ps --no-trunc "${service_name}" >&2 || true + return 1 +} + +wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}" +if [[ "${deploy_env}" == "canary" ]]; then + wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}" +else + wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}" +fi + +if [[ "${brio_staging_enabled}" == "1" ]]; then + brio_tls_ready=0 + for _ in $(seq 1 30); do + if docker run --rm --network "${brio_staging_db_network}" \ + -e PGSSLMODE=verify-full \ + -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${postgres_ca_cert_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-brio-staging -U "${postgres_root_user}" -d postgres -Atc "select 1" >/dev/null 2>&1; then + brio_tls_ready=1 + break + fi + sleep 2 + done + if [[ "${brio_tls_ready}" != "1" ]]; then + echo "PostgreSQL did not pass sslmode=verify-full using makepad-postgres-brio-staging within 60 seconds." >&2 + exit 1 + fi +fi + +if [[ "${vif_enabled}" != "1" ]]; then + exit 0 +fi + +postgres_ready=0 +for _ in $(seq 1 30); do + if docker run --rm --network "${vif_db_network}" \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres -c "select 1" >/dev/null 2>&1; then + postgres_ready=1 + break + fi + sleep 2 +done +if [[ "${postgres_ready}" != "1" ]]; then + echo "Postgres did not become reachable via makepad-postgres-vif on ${vif_db_network} after 60 seconds." >&2 + exit 1 +fi + +docker run --rm --network "${vif_db_network}" \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres \ + -v ON_ERROR_STOP=1 \ + -v vif_db="${vif_db_name}" \ + -v vif_user="${vif_db_user}" \ + -v vif_password="${vif_db_password}" <<'SQL' +SELECT format('CREATE ROLE %I LOGIN', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec +SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec +SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = :'vif_db' + AND r.rolname <> :'vif_user' +) \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec +SQL diff --git a/scripts/test-brio-bootstrap.sh b/scripts/test-brio-bootstrap.sh index 0af51eb..83b79c9 100755 --- a/scripts/test-brio-bootstrap.sh +++ b/scripts/test-brio-bootstrap.sh @@ -11,7 +11,16 @@ keycloak_app_password='brio-keycloak-integration-only' keycloak_backup_password='brio-keycloak-backup-integration-only' cleanup() { + local status=$? + trap - EXIT + if ((status != 0)); then + echo "Brio bootstrap test failed; PostgreSQL container state and logs follow." >&2 + docker inspect "${container_name}" \ + --format 'status={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' >&2 2>/dev/null || true + docker logs --tail 200 "${container_name}" >&2 2>/dev/null || true + fi docker rm -f "${container_name}" >/dev/null 2>&1 || true + exit "${status}" } trap cleanup EXIT @@ -20,13 +29,23 @@ docker run -d --name "${container_name}" \ -v "${repo_root}/bootstrap:/bootstrap:ro" \ "${postgres_image}" >/dev/null -for _ in $(seq 1 100); do - if docker exec "${container_name}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then +postgres_ready=false +for _ in $(seq 1 300); do + if docker exec "${container_name}" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null 2>&1; then + postgres_ready=true break fi + if [[ "$(docker inspect "${container_name}" --format '{{.State.Running}}' 2>/dev/null || true)" != "true" ]]; then + echo "PostgreSQL container exited before TCP readiness." >&2 + exit 1 + fi sleep 0.1 done -docker exec "${container_name}" pg_isready -U postgres -d postgres >/dev/null +if [[ "${postgres_ready}" != "true" ]]; then + echo "PostgreSQL did not become ready on TCP loopback within 30 seconds." >&2 + exit 1 +fi +docker exec "${container_name}" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 672c208..1f042f9 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -84,7 +84,10 @@ canary_compose = read_required_text(repo_root / "envs/canary/compose.yml", "cana production_compose = read_required_text(repo_root / "envs/production/compose.yml", "production Compose override") canary_env = read_required_text(repo_root / "envs/canary/.env.db", "canary database environment") production_env = read_required_text(repo_root / "envs/production/.env.db", "production database environment") -manual_deploy = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") +manual_deploy_workflow = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") +remote_deploy_path = repo_root / "scripts/deploy-postgres-stack.sh" +remote_deploy = read_required_text(remote_deploy_path, "remote deploy script") +manual_deploy = manual_deploy_workflow + "\n" + remote_deploy ci_workflow = read_required_text(repo_root / ".github/workflows/ci.yml", "CI workflow") normalized_readme = re.sub(r"\s+", " ", readme) @@ -143,6 +146,14 @@ require("name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK}" in production_compose, "Produ for required in ("target: 5432", "published: 5432", "protocol: tcp", "mode: host"): require(required in production_compose, f"Production Compose must publish PostgreSQL for DB VM clients: {required}") require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy workflow must reject root SSH users.") +require(remote_deploy_path.stat().st_mode & 0o111, "Remote deploy script must be executable.") +for required in ( + 'cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh"', + 'scp "${scp_opts[@]}" "${bundle_root}/scripts/deploy-postgres-stack.sh"', + 'printf -v remote_script_q %q "${REMOTE_DIR}/scripts/deploy-postgres-stack.sh"', +): + require(required in manual_deploy_workflow, f"Manual deploy workflow must bundle and invoke the remote deploy script: {required}") +require("<<'EOF'" not in manual_deploy_workflow, "Manual deploy workflow must not embed the oversized remote deployment heredoc.") require("DEPLOY_BRIO_STAGING_DB_NETWORK must be makepad_brio_staging_db" in manual_deploy, "Manual deploy must reject a non-canonical Brio database network secret.") require("Brio deployment bundle must use makepad_brio_staging_db" in manual_deploy, "Remote deploy must revalidate the canonical Brio database network.") require("postgres:16-alpine@sha256:" in base_compose, "Base Compose must pin PostgreSQL to an immutable digest.") From 199abfccbf3bbcf8c77c7cfdf1584d6cead94b33 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 20:04:50 +0200 Subject: [PATCH 03/20] fix(postgres): preserve restricted shared HBA rules --- README.md | 12 ++++++-- config/runtrace-pg_hba.conf | 23 +++++++++++++-- scripts/validate-postgres-config.sh | 44 +++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7284ce4..ba25476 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,14 @@ VM hostname present in the PostgreSQL server certificate SAN. The encrypted identity backup refuses any connection mode other than `verify-full`. The host deployment preserves the existing host-network endpoint used by -Keycloak while requiring TLS and SCRAM for `runtrace` and -`keycloak_runtrace`. Other databases keep their existing SCRAM transport policy. +Keycloak while requiring TLS and SCRAM for `runtrace`, `keycloak_runtrace`, +`fresko_production`, `betacrew`, and `keycloak_betacrew`. Fresko's runtime, +schema-owner, and importer roles and the BetaCrew application role are limited +to the private WireGuard source `10.80.0.1/32`; the BetaCrew Keycloak role is +limited to `88.99.209.165/32`, with local maintenance access limited to +`127.0.0.1/32`. The committed HBA policy rejects every other source or plaintext +connection for those databases before reaching the shared fallback. Other +databases keep their existing SCRAM transport policy. Required environment secrets: @@ -121,7 +127,7 @@ docker config create makepad_postgres_canary_tls_cert_v2 /secure/path/canary-ser docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-server.key ``` -The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging` and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. +The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy preserves the source-restricted Fresko and BetaCrew rules described above, rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging`, and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. The workflow copies the checked-in remote deployment entrypoint with the deployment bundle and deploys only the PostgreSQL stack. Before deployment it validates the password and CA files, certificate chain, seven-day expiry margin, and—for diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 251f826..ff38f3d 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -3,10 +3,29 @@ local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject -hostnossl brio_staging all all reject -hostnossl keycloak_brio_staging all all reject hostssl runtrace all all scram-sha-256 hostssl keycloak_runtrace all all scram-sha-256 +# Fresko can reach PostgreSQL only over the Makepad private WireGuard route. +# The application, migration, and importer roles have separate passwords; any +# other source or TLS mode is explicitly rejected before the shared fallback. +hostnossl fresko_production all all reject +hostssl fresko_production fresko_runtime 10.80.0.1/32 scram-sha-256 +hostssl fresko_production fresko_schema_owner 10.80.0.1/32 scram-sha-256 +hostssl fresko_production fresko_importer 10.80.0.1/32 scram-sha-256 +hostssl fresko_production all all reject +# BetaCrew app and identity traffic require TLS and exact source roles. +hostnossl betacrew all all reject +hostnossl keycloak_betacrew all all reject +hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256 +hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256 +hostssl betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl betacrew all all reject +hostssl keycloak_betacrew all all reject +# Brio staging application and identity traffic require TLS and cannot access +# any other database through the shared fallback. +hostnossl brio_staging all all reject +hostnossl keycloak_brio_staging all all reject hostssl brio_staging brio_staging_app all scram-sha-256 hostssl brio_staging brio_staging_backup all scram-sha-256 hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 1f042f9..ff57df6 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -183,6 +183,50 @@ for required in ( for database in ("runtrace", "keycloak_runtrace"): require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") require(re.search(rf"^hostssl\s+{database}\s+all\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must require TLS and SCRAM for {database}.") +hba_records = [ + tuple(line.split()) + for line in runtrace_hba.splitlines() + if line.strip() and not line.lstrip().startswith("#") +] +fresko_betacrew_records = [ + record + for record in hba_records + if len(record) >= 2 and record[1] in {"fresko_production", "betacrew", "keycloak_betacrew"} +] +require( + fresko_betacrew_records + == [ + ("hostnossl", "fresko_production", "all", "all", "reject"), + ("hostssl", "fresko_production", "fresko_runtime", "10.80.0.1/32", "scram-sha-256"), + ("hostssl", "fresko_production", "fresko_schema_owner", "10.80.0.1/32", "scram-sha-256"), + ("hostssl", "fresko_production", "fresko_importer", "10.80.0.1/32", "scram-sha-256"), + ("hostssl", "fresko_production", "all", "all", "reject"), + ("hostnossl", "betacrew", "all", "all", "reject"), + ("hostnossl", "keycloak_betacrew", "all", "all", "reject"), + ("hostssl", "betacrew", "betacrew_app", "10.80.0.1/32", "scram-sha-256"), + ("hostssl", "keycloak_betacrew", "keycloak_betacrew_app", "88.99.209.165/32", "scram-sha-256"), + ("hostssl", "betacrew", "postgres", "127.0.0.1/32", "scram-sha-256"), + ("hostssl", "keycloak_betacrew", "postgres", "127.0.0.1/32", "scram-sha-256"), + ("hostssl", "betacrew", "all", "all", "reject"), + ("hostssl", "keycloak_betacrew", "all", "all", "reject"), + ], + "HBA must preserve the exact live Fresko and BetaCrew TLS, source, role, and rejection policy.", +) +for required in ( + "`fresko_production`", + "`betacrew`", + "`keycloak_betacrew`", + "`10.80.0.1/32`", + "`88.99.209.165/32`", + "`127.0.0.1/32`", +): + require(required in readme, f"README must document the preserved Fresko/BetaCrew HBA policy: {required}") +shared_fallback = ("host", "all", "all", "all", "scram-sha-256") +require(shared_fallback in hba_records, "HBA must retain the shared SCRAM fallback.") +require( + max(hba_records.index(record) for record in fresko_betacrew_records) < hba_records.index(shared_fallback), + "Every Fresko and BetaCrew restriction must precede the shared HBA fallback.", +) for database, roles in ( ("brio_staging", ("brio_staging_app", "brio_staging_backup")), ("keycloak_brio_staging", ("keycloak_brio_staging_app", "keycloak_brio_staging_backup")), From 83135489e287a7136134f726e799e4c472856d63 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 20:04:54 +0200 Subject: [PATCH 04/20] ci(postgres): isolate self-hosted deployment jobs --- .github/workflows/ci.yml | 3 +- .github/workflows/manual-deploy.yml | 53 ++++++++++++++++++++++------- scripts/validate-postgres-config.sh | 13 +++++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41c58b3..f92c381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,8 @@ permissions: jobs: validate: - runs-on: ubuntu-latest + name: policy-and-integration + runs-on: [self-hosted, linux, x64, makepad] steps: - uses: actions/checkout@v5 - name: Validate PostgreSQL deployment contract diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index b1cdf6a..9548792 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -13,7 +13,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64, makepad] environment: ${{ inputs.environment }} permissions: contents: read @@ -21,7 +21,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - - name: Configure SSH key + - name: Configure job-scoped SSH material shell: bash env: DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} @@ -30,12 +30,16 @@ jobs: set -euo pipefail : "${DEPLOY_SSH_PRIVATE_KEY:?set DEPLOY_SSH_PRIVATE_KEY}" : "${DEPLOY_SSH_KNOWN_HOSTS:?set DEPLOY_SSH_KNOWN_HOSTS}" - mkdir -p "${HOME}/.ssh" - chmod 700 "${HOME}/.ssh" - printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" > "${HOME}/.ssh/id_ed25519" - chmod 600 "${HOME}/.ssh/id_ed25519" - printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${HOME}/.ssh/known_hosts" - chmod 600 "${HOME}/.ssh/known_hosts" + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${ssh_directory}" ]]; then + echo "Refusing to reuse an existing job-scoped SSH directory: ${ssh_directory}" >&2 + exit 1 + fi + mkdir -m 0700 "${ssh_directory}" + printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" > "${ssh_directory}/id_ed25519" + chmod 0600 "${ssh_directory}/id_ed25519" + printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${ssh_directory}/known_hosts" + chmod 0600 "${ssh_directory}/known_hosts" - name: Prepare deployment bundle shell: bash @@ -65,7 +69,11 @@ jobs: DEPLOY_VIF_DB_NAME="${DEPLOY_VIF_DB_NAME:-vif}" DEPLOY_VIF_DB_USER="${DEPLOY_VIF_DB_USER:-vif}" fi - bundle_root="${RUNNER_TEMP}/bundle" + bundle_root="${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${bundle_root}" ]]; then + echo "Refusing to reuse an existing job-scoped deployment bundle: ${bundle_root}" >&2 + exit 1 + fi mkdir -p "${bundle_root}/config" "${bundle_root}/scripts" "${bundle_root}/envs/${{ inputs.environment }}" cp compose.yml "${bundle_root}/compose.yml" cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" @@ -109,10 +117,17 @@ jobs: echo "DEPLOY_SSH_USER must not be root." >&2 exit 1 fi - bundle_root="${RUNNER_TEMP}/bundle" + bundle_root="${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + private_key_file="${ssh_directory}/id_ed25519" + known_hosts_file="${ssh_directory}/known_hosts" + [[ -r "${private_key_file}" && -r "${known_hosts_file}" ]] || { + echo "Job-scoped SSH material is missing or unreadable." >&2 + exit 1 + } remote_port=${REMOTE_PORT:-22} - ssh_opts=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="${HOME}/.ssh/known_hosts" -p "${remote_port}") - scp_opts=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="${HOME}/.ssh/known_hosts" -P "${remote_port}") + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -p "${remote_port}") + scp_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -P "${remote_port}") remote_target="${REMOTE_USER}@${REMOTE_HOST}" ssh "${ssh_opts[@]}" "${remote_target}" mkdir -p "${REMOTE_DIR}/config" "${REMOTE_DIR}/scripts" "${REMOTE_DIR}/envs/${{ inputs.environment }}" @@ -135,3 +150,17 @@ jobs: # Values are intentionally expanded locally and shell-escaped with %q. # shellcheck disable=SC2029 ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_dir_q} ${stack_name_q} ${deploy_env_q}" + + - name: Remove job-scoped deployment material + if: always() + shell: bash + run: | + set -euo pipefail + for path in \ + "${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do + case "${path}" in + "${RUNNER_TEMP}"/postgres-deploy-*) rm -rf -- "${path}" ;; + *) echo "Refusing to remove unexpected cleanup path: ${path}" >&2; exit 1 ;; + esac + done diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index ff57df6..d58affe 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -573,6 +573,19 @@ for required in ( "openssl cms -encrypt", ): require(required in manual_deploy, f"Manual deploy is missing Brio certificate/connection preflight marker: {required}") +for required in ( + "postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + 'UserKnownHostsFile=${known_hosts_file}', + "-F /dev/null", + "GlobalKnownHostsFile=/dev/null", + "IdentitiesOnly=yes", + "Remove job-scoped deployment material", + "if: always()", +): + require(required in manual_deploy_workflow, f"Self-hosted deploy workflow is missing job-scoped cleanup control: {required}") +for forbidden in ('${HOME}/.ssh', "$HOME/.ssh", "~/.ssh", "add-ssh-host-key-action"): + require(forbidden not in manual_deploy_workflow, f"Self-hosted deploy workflow must not persist SSH state via {forbidden}.") for policy in ( "hostnossl brio_staging", "hostnossl keycloak_brio_staging", From a4c30cfae3c41ad6d6ec43b2a5adc3bf05655390 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 20:18:43 +0200 Subject: [PATCH 05/20] ci(postgres): disable persisted checkout credentials --- .github/workflows/ci.yml | 2 ++ .github/workflows/manual-deploy.yml | 2 ++ scripts/validate-postgres-config.sh | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f92c381..501a44a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,8 @@ jobs: runs-on: [self-hosted, linux, x64, makepad] steps: - uses: actions/checkout@v5 + with: + persist-credentials: false - name: Validate PostgreSQL deployment contract run: ./scripts/validate-postgres-config.sh - name: Check deployment shell scripts diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 9548792..e8d59d8 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -20,6 +20,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v5 + with: + persist-credentials: false - name: Configure job-scoped SSH material shell: bash diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index d58affe..91ab3f7 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -586,6 +586,16 @@ for required in ( require(required in manual_deploy_workflow, f"Self-hosted deploy workflow is missing job-scoped cleanup control: {required}") for forbidden in ('${HOME}/.ssh', "$HOME/.ssh", "~/.ssh", "add-ssh-host-key-action"): require(forbidden not in manual_deploy_workflow, f"Self-hosted deploy workflow must not persist SSH state via {forbidden}.") +for workflow_name, workflow_text in ( + ("CI", ci_workflow), + ("manual deploy", manual_deploy_workflow), +): + checkout_count = workflow_text.count("uses: actions/checkout@v5") + require(checkout_count > 0, f"{workflow_name} workflow must check out the repository.") + require( + workflow_text.count("persist-credentials: false") == checkout_count, + f"Every self-hosted checkout in the {workflow_name} workflow must disable persisted Git credentials.", + ) for policy in ( "hostnossl brio_staging", "hostnossl keycloak_brio_staging", From 26d6572c392d8ed7135d24ba303bd3ecdfdf3db2 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 20:18:47 +0200 Subject: [PATCH 06/20] fix(postgres): set encrypted overlay option explicitly --- README.md | 2 +- scripts/deploy-postgres-stack.sh | 6 +++--- scripts/validate-postgres-config.sh | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ba25476..cd5dee1 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The manual deploy workflow sources these Compose variables from environment secr - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK` production only - `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` <- `DEPLOY_BRIO_STAGING_DB_NETWORK` canary only -Every database network must be an attachable Swarm overlay created with `--opt encrypted`; Brio's dedicated network must additionally be `--internal`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. +Every database network must be an attachable Swarm overlay created with `--opt encrypted=true`; Brio's dedicated network must additionally be `--internal`. The explicit value matters: Docker records a valueless `--opt encrypted` as an empty option rather than the required `true`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. Application network topology is owned by the consuming application repositories. New Keycloak instances keep their own DB-facing Docker networks in the Keycloak repository and connect to this PostgreSQL server through the configured DB endpoint. diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh index abe70f9..4cfc7ff 100755 --- a/scripts/deploy-postgres-stack.sh +++ b/scripts/deploy-postgres-stack.sh @@ -195,18 +195,18 @@ ensure_encrypted_overlay_network() { scope=$(docker network inspect "${network_name}" --format '{{.Scope}}') encrypted=$(docker network inspect "${network_name}" --format '{{index .Options "encrypted"}}') if [[ "${driver}" != "overlay" || "${scope}" != "swarm" || "${encrypted}" != "true" ]]; then - echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted, then rerun this deployment." >&2 + echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted=true, then rerun this deployment." >&2 exit 1 fi return fi - docker network create --driver overlay --attachable --opt encrypted "${network_name}" >/dev/null + docker network create --driver overlay --attachable --opt encrypted=true "${network_name}" >/dev/null } ensure_internal_encrypted_overlay_network() { local network_name=$1 if ! docker network inspect "${network_name}" >/dev/null 2>&1; then - docker network create --driver overlay --attachable --internal --opt encrypted "${network_name}" >/dev/null + docker network create --driver overlay --attachable --internal --opt encrypted=true "${network_name}" >/dev/null fi local details details=$(docker network inspect "${network_name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 91ab3f7..5cecb9a 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -278,7 +278,8 @@ for label, content in (("canary", canary_compose), ("production", production_com require("/run/secrets/postgres_superuser_password:ro" in content, f"{label} Compose must mount the superuser password file read-only.") require("resources:" in content and "limits:" in content and "reservations:" in content, f"{label} Compose must set resource limits and reservations.") require("ensure_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate encrypted database overlay networks.") -require("--opt encrypted" in manual_deploy, "Manual deploy must create database overlay networks with encryption.") +require(len(re.findall(r"docker network create[^\n]+--opt encrypted=true", manual_deploy)) == 2, "Manual deploy must explicitly create both database overlay network variants with encrypted=true.") +require(re.search(r"--opt\s+encrypted(?:\s|$)", manual_deploy) is None, "Manual deploy must not use Docker's valueless encrypted option.") require("postgres_root_password_file" in manual_deploy, "Manual deploy must load the PostgreSQL superuser password from the host file.") require('docker config inspect "${postgres_tls_cert_config}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS certificate config.") require('docker secret inspect "${postgres_tls_key_secret}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS private-key secret.") @@ -287,7 +288,7 @@ require('deployed_hba_sha256' in manual_deploy, "Manual deploy must reject Runtr require("config/runtrace-pg_hba.conf" in manual_deploy, "Manual deploy must include the Runtrace HBA policy.") require("-e PGPASSWORD=" not in manual_deploy, "Manual deploy must not expose the PostgreSQL superuser password as a container environment argument.") require("POSTGRES_PASSWORD_FILE" in normalized_readme, "README must document file-based PostgreSQL bootstrap credentials.") -require("--opt encrypted" in normalized_readme, "README must document encrypted database overlays.") +require("--opt encrypted=true" in normalized_readme, "README must document the explicit encrypted=true database overlay option.") require("sslmode=verify-full" in normalized_readme, "README must document certificate-verified Runtrace database connections.") require("docker secret create makepad_postgres_tls_key_v1" in normalized_readme, "README must document private-key secret provisioning.") require("bash scripts/test-runtrace-tls-policy.sh" in normalized_readme, "README must document the container-level Runtrace TLS policy test.") From e51af43f5aee333f5dbba8158fb2705bd57b4c79 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Fri, 4 Sep 2026 20:40:36 +0200 Subject: [PATCH 07/20] ci(postgres): reject untrusted fork jobs --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 501a44a..0ac3a6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ permissions: jobs: validate: name: policy-and-integration + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: [self-hosted, linux, x64, makepad] steps: - uses: actions/checkout@v5 From 109421da341fdd5b2bcfb42efb066d0d9d7b7d0d Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 04:27:03 +0200 Subject: [PATCH 08/20] fix(deploy): make Brio database rollout crash-safe --- .github/workflows/deploy-brio-identity-db.yml | 227 +++++ .github/workflows/manual-deploy.yml | 198 +++- bootstrap/vif-app.sql | 36 + compose.host.yml | 3 +- config/runtrace-pg_hba.conf | 2 +- envs/canary/.env.db | 2 +- envs/production/.env.db | 11 +- envs/production/compose.yml | 66 -- scripts/brio-db-transaction.sh | 243 +++++ scripts/deploy-brio-canary-postgres.sh | 831 ++++++++++++++++ scripts/deploy-brio-identity-db-host.sh | 806 ++++++++++++++++ scripts/deploy-postgres-stack.sh | 186 ++-- scripts/ensure-brio-tmp-cleaner.sh | 122 +++ .../brio-deployment-failure-fixture.sh | 886 ++++++++++++++++++ scripts/test-brio-db-transaction.sh | 129 +++ scripts/test-brio-deploy-guards.sh | 42 + scripts/test-brio-deployment-contracts.sh | 201 ++++ scripts/test-brio-deployment-failures.sh | 13 + 18 files changed, 3829 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/deploy-brio-identity-db.yml create mode 100644 bootstrap/vif-app.sql create mode 100755 scripts/brio-db-transaction.sh create mode 100755 scripts/deploy-brio-canary-postgres.sh create mode 100755 scripts/deploy-brio-identity-db-host.sh create mode 100755 scripts/ensure-brio-tmp-cleaner.sh create mode 100755 scripts/fixtures/brio-deployment-failure-fixture.sh create mode 100755 scripts/test-brio-db-transaction.sh create mode 100755 scripts/test-brio-deploy-guards.sh create mode 100755 scripts/test-brio-deployment-contracts.sh create mode 100755 scripts/test-brio-deployment-failures.sh diff --git a/.github/workflows/deploy-brio-identity-db.yml b/.github/workflows/deploy-brio-identity-db.yml new file mode 100644 index 0000000..fe82228 --- /dev/null +++ b/.github/workflows/deploy-brio-identity-db.yml @@ -0,0 +1,227 @@ +name: Deploy Brio Identity Database + +on: + workflow_dispatch: + inputs: + restart_confirmation: + description: Type restart-standalone-postgres-for-brio-staging + required: true + type: string + backup_restore_confirmed: + description: Confirm a current encrypted backup was restore-tested before this DB-VM change + required: true + type: boolean + default: false + +concurrency: + group: postgres-standalone-db-vm + cancel-in-progress: false + +permissions: + contents: read + +jobs: + deploy: + runs-on: + group: Postgres Deploy + labels: [self-hosted, linux, x64, makepad, makepad-postgres-deploy] + environment: staging-brio-identity-db + timeout-minutes: 60 + steps: + - name: Enforce protected standalone deployment gate + shell: bash + env: + RESTART_CONFIRMATION: ${{ inputs.restart_confirmation }} + BACKUP_RESTORE_CONFIRMED: ${{ inputs.backup_restore_confirmed }} + run: | + set -euo pipefail + [[ "${GITHUB_REF}" == "refs/heads/main" ]] || { echo "Standalone DB-VM deployment is allowed only from main." >&2; exit 1; } + [[ "${RESTART_CONFIRMATION}" == "restart-standalone-postgres-for-brio-staging" ]] || { echo "The exact restart acknowledgement is required." >&2; exit 1; } + [[ "${BACKUP_RESTORE_CONFIRMED}" == "true" ]] || { echo "A current successful encrypted restore test is required." >&2; exit 1; } + + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + + - name: Configure job-scoped SSH material + shell: bash + env: + SSH_PRIVATE_KEY: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS }} + run: | + set -euo pipefail + : "${SSH_PRIVATE_KEY:?set BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY}" + : "${SSH_KNOWN_HOSTS:?set BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS}" + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${ssh_dir}" ]] || { echo "Refusing to reuse a job-scoped SSH directory." >&2; exit 1; } + install -d -m 0700 "${ssh_dir}" + umask 077 + printf '%s\n' "${SSH_PRIVATE_KEY}" > "${ssh_dir}/id_ed25519" + printf '%s\n' "${SSH_KNOWN_HOSTS}" > "${ssh_dir}/known_hosts" + chmod 0600 "${ssh_dir}"/* + + - name: Prepare standalone DB-VM bundle and secrets + shell: bash + env: + KEYCLOAK_APP_PASSWORD: ${{ secrets.KEYCLOAK_BRIO_STAGING_DB_PASSWORD }} + KEYCLOAK_BACKUP_PASSWORD: ${{ secrets.KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD }} + BACKUP_RECIPIENT_CERT: ${{ secrets.BRIO_BACKUP_RECIPIENT_CERT_PEM }} + run: | + set -euo pipefail + : "${KEYCLOAK_APP_PASSWORD:?set KEYCLOAK_BRIO_STAGING_DB_PASSWORD}" + : "${KEYCLOAK_BACKUP_PASSWORD:?set KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD}" + : "${BACKUP_RECIPIENT_CERT:?set BRIO_BACKUP_RECIPIENT_CERT_PEM}" + bundle_dir="${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${bundle_dir}" && ! -e "${runtime_dir}" ]] || { echo "Refusing to reuse job-scoped deployment paths." >&2; exit 1; } + install -d -m 0700 "${bundle_dir}/bootstrap" "${bundle_dir}/config" "${bundle_dir}/envs/production" "${bundle_dir}/scripts" "${runtime_dir}" + cp compose.host.yml "${bundle_dir}/compose.host.yml" + cp envs/production/.env.db "${bundle_dir}/envs/production/.env.db" + cp config/runtrace-pg_hba.conf "${bundle_dir}/config/runtrace-pg_hba.conf" + cp bootstrap/keycloak-brio-staging.sql "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" + cp scripts/run-runtrace-backup.sh scripts/run-runtrace-backup-loop.sh "${bundle_dir}/scripts/" + cp scripts/run-brio-encrypted-backup.sh scripts/run-brio-encrypted-backup-loop.sh "${bundle_dir}/scripts/" + cp scripts/deploy-brio-identity-db-host.sh "${bundle_dir}/scripts/deploy-brio-identity-db-host.sh" + cp scripts/brio-db-transaction.sh "${bundle_dir}/scripts/brio-db-transaction.sh" + cp scripts/ensure-brio-tmp-cleaner.sh "${bundle_dir}/scripts/ensure-brio-tmp-cleaner.sh" + umask 077 + printf '%s' "${KEYCLOAK_APP_PASSWORD}" > "${runtime_dir}/keycloak-brio-staging-app-password" + printf '%s' "${KEYCLOAK_BACKUP_PASSWORD}" > "${runtime_dir}/keycloak-brio-staging-backup-password" + printf '%s' "${BACKUP_RECIPIENT_CERT}" > "${runtime_dir}/brio-backup-recipient-cert.pem" + chmod 0600 "${runtime_dir}"/* + + - name: Deploy only to the standalone database VM + shell: bash + env: + REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + DB_HOSTNAME: ${{ vars.BRIO_IDENTITY_DB_HOSTNAME }} + KEYCLOAK_DB_SOURCE_CIDR: ${{ vars.BRIO_KEYCLOAK_DB_SOURCE_CIDR }} + run: | + set -euo pipefail + : "${REMOTE_HOST:?}" "${REMOTE_USER:?}" "${DB_HOSTNAME:?set BRIO_IDENTITY_DB_HOSTNAME}" "${KEYCLOAK_DB_SOURCE_CIDR:?set BRIO_KEYCLOAK_DB_SOURCE_CIDR}" + [[ "${REMOTE_USER}" != "root" ]] || { echo "The standalone DB deploy SSH user must not be root." >&2; exit 1; } + [[ "${DB_HOSTNAME}" == "65.21.134.125" ]] || { echo "BRIO_IDENTITY_DB_HOSTNAME must be the reviewed standalone DB IP 65.21.134.125." >&2; exit 1; } + [[ "${KEYCLOAK_DB_SOURCE_CIDR}" == "88.99.209.165/32" ]] || { echo "BRIO_KEYCLOAK_DB_SOURCE_CIDR must be the reviewed Keycloak egress 88.99.209.165/32." >&2; exit 1; } + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + bundle_dir="${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_bundle="/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime="/tmp/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -p "${remote_port}") + scp_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -P "${remote_port}") + target="${REMOTE_USER}@${REMOTE_HOST}" + printf -v remote_bundle_q %q "${remote_bundle}" + printf -v remote_runtime_q %q "${remote_runtime}" + # Both remote paths are fixed, run-scoped, and escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "if [ -e ${remote_bundle_q} ] || [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote identity deployment path.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + scp "${scp_opts[@]}" -r "${bundle_dir}/." "${target}:${remote_bundle}/" + printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" + printf -v db_env_q %q "${remote_bundle}/envs/production/.env.db" + # Start the host TTL guard before transferring any secret material. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "chmod 0755 ${cleaner_q} && ${cleaner_q} ${db_env_q} && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}/"* "${target}:${remote_runtime}/" + # All remote paths are fixed and escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "chmod 0755 ${remote_bundle_q}/scripts/deploy-brio-identity-db-host.sh ${remote_bundle_q}/scripts/brio-db-transaction.sh ${remote_bundle_q}/scripts/run-runtrace-backup.sh ${remote_bundle_q}/scripts/run-runtrace-backup-loop.sh ${remote_bundle_q}/scripts/run-brio-encrypted-backup.sh ${remote_bundle_q}/scripts/run-brio-encrypted-backup-loop.sh && chmod 0600 ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem" + printf -v deploy_script_q %q "${remote_bundle}/scripts/deploy-brio-identity-db-host.sh" + printf -v db_hostname_q %q "${DB_HOSTNAME}" + printf -v cidr_q %q "${KEYCLOAK_DB_SOURCE_CIDR}" + # Validated values are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" \ + "BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes ${deploy_script_q} ${remote_bundle_q} ${remote_runtime_q} ${db_hostname_q} ${cidr_q}" + + - name: Create canonical standalone deployment evidence + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" && "${GITHUB_REF}" == "refs/heads/main" ]] || { + echo "Deployment evidence is produced only by Makepad-fr/postgres main." >&2 + exit 1 + } + [[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ && "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ \ + && "${GITHUB_SHA}" =~ ^[0-9a-f]{40}$ ]] || { echo "Invalid immutable deployment identity." >&2; exit 1; } + evidence_dir="${RUNNER_TEMP}/postgres-identity-evidence-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${evidence_dir}" && ! -L "${evidence_dir}" ]] || { echo "Refusing to reuse deployment evidence output." >&2; exit 1; } + install -d -m 0700 "${evidence_dir}" + umask 077 + python3 - "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT}" "${GITHUB_SHA}" \ + > "${evidence_dir}/brio-db-deployment-evidence.json" <<'PY' + import json, sys + print(json.dumps({ + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": int(sys.argv[1]), + "postgres_run_attempt": int(sys.argv[2]), + "postgres_head_sha": sys.argv[3], + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, sort_keys=True, separators=(",", ":"))) + PY + chmod 0600 "${evidence_dir}/brio-db-deployment-evidence.json" + + - name: Publish immutable standalone deployment evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/postgres-identity-evidence-${{ github.run_id }}-${{ github.run_attempt }}/brio-db-deployment-evidence.json + if-no-files-found: error + retention-days: 35 + + - name: Remove remote job-scoped identity secrets + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + run: | + set -euo pipefail + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_dir}/id_ed25519" && -r "${ssh_dir}/known_hosts" ]] || exit 0 + remote_runtime="/tmp/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_bundle="/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -p "${remote_port}") + printf -v remote_runtime_q %q "${remote_runtime}" + printf -v remote_bundle_q %q "${remote_bundle}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${remote_runtime_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_runtime_q} ]; then rm -f ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem; if [ -f ${remote_runtime_q}/RECOVERY_REQUIRED ] && [ ! -L ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${remote_runtime_q} -depth -delete; fi; elif [ -e ${remote_runtime_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + + - name: Remove local job-scoped deployment material + if: always() + shell: bash + run: | + set -euo pipefail + for cleanup_target in \ + "${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-identity-evidence-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do + case "${cleanup_target}" in + "${RUNNER_TEMP}"/postgres-identity-*|"${RUNNER_TEMP}"/postgres-brio-identity-runtime-*) + if [[ -L "${cleanup_target}" ]]; then + echo "Refusing cleanup of a symlink: ${cleanup_target}" >&2 + exit 1 + elif [[ -d "${cleanup_target}" ]]; then + find "${cleanup_target}" -mindepth 1 -delete + rmdir "${cleanup_target}" + elif [[ -e "${cleanup_target}" ]]; then + echo "Expected a cleanup directory: ${cleanup_target}" >&2 + exit 1 + fi + ;; + *) echo "Refusing unexpected cleanup path: ${cleanup_target}" >&2; exit 1 ;; + esac + done diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index e8d59d8..52845a1 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -11,15 +11,28 @@ on: - canary - production +concurrency: + group: postgres-shared-swarm-target + cancel-in-progress: false + jobs: deploy: - runs-on: [self-hosted, linux, x64, makepad] + runs-on: + group: Postgres Deploy + labels: [self-hosted, linux, x64, makepad, makepad-postgres-deploy] environment: ${{ inputs.environment }} + timeout-minutes: 30 permissions: contents: read steps: + - name: Enforce reviewed deployment ref + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REF}" == "refs/heads/main" ]] || { echo "PostgreSQL deployment is allowed only from main." >&2; exit 1; } + - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: persist-credentials: false @@ -43,6 +56,57 @@ jobs: printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${ssh_directory}/known_hosts" chmod 0600 "${ssh_directory}/known_hosts" + - name: Materialize job-scoped Brio canary inputs + if: inputs.environment == 'canary' + shell: bash + env: + BRIO_BACKUP_RECIPIENT_CERT_PEM: ${{ secrets.BRIO_BACKUP_RECIPIENT_CERT_PEM }} + BRIO_STAGING_BACKUP_DB_PASSWORD: ${{ secrets.BRIO_STAGING_BACKUP_DB_PASSWORD }} + BRIO_STAGING_DB_PASSWORD: ${{ secrets.BRIO_STAGING_DB_PASSWORD }} + POSTGRES_CANARY_SUPERUSER_PASSWORD: ${{ secrets.POSTGRES_CANARY_SUPERUSER_PASSWORD }} + POSTGRES_CA_PEM: ${{ secrets.POSTGRES_CA_PEM }} + POSTGRES_SERVER_CERT_PEM: ${{ secrets.POSTGRES_SERVER_CERT_PEM }} + POSTGRES_SERVER_KEY_PEM: ${{ secrets.POSTGRES_SERVER_KEY_PEM }} + run: | + set -euo pipefail + : "${BRIO_BACKUP_RECIPIENT_CERT_PEM:?set BRIO_BACKUP_RECIPIENT_CERT_PEM}" + : "${BRIO_STAGING_BACKUP_DB_PASSWORD:?set BRIO_STAGING_BACKUP_DB_PASSWORD}" + : "${BRIO_STAGING_DB_PASSWORD:?set BRIO_STAGING_DB_PASSWORD}" + : "${POSTGRES_CANARY_SUPERUSER_PASSWORD:?set POSTGRES_CANARY_SUPERUSER_PASSWORD}" + : "${POSTGRES_CA_PEM:?set POSTGRES_CA_PEM}" + : "${POSTGRES_SERVER_CERT_PEM:?set POSTGRES_SERVER_CERT_PEM}" + : "${POSTGRES_SERVER_KEY_PEM:?set POSTGRES_SERVER_KEY_PEM}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${runtime_dir}" ]]; then + echo "Refusing to reuse an existing job-scoped Brio runtime directory." >&2 + exit 1 + fi + install -d -m 0700 "${runtime_dir}" + umask 077 + printf '%s' "${POSTGRES_CANARY_SUPERUSER_PASSWORD}" > "${runtime_dir}/postgres-superuser-password" + printf '%s' "${BRIO_STAGING_DB_PASSWORD}" > "${runtime_dir}/brio-staging-app-password" + printf '%s' "${BRIO_STAGING_BACKUP_DB_PASSWORD}" > "${runtime_dir}/brio-staging-backup-password" + printf '%s' "${POSTGRES_CA_PEM}" > "${runtime_dir}/postgres-ca.pem" + printf '%s' "${POSTGRES_SERVER_CERT_PEM}" > "${runtime_dir}/postgres-server-cert.pem" + printf '%s' "${POSTGRES_SERVER_KEY_PEM}" > "${runtime_dir}/postgres-server-key.pem" + printf '%s' "${BRIO_BACKUP_RECIPIENT_CERT_PEM}" > "${runtime_dir}/brio-backup-recipient-cert.pem" + chmod 0600 "${runtime_dir}"/* + + - name: Materialize job-scoped VIF credential + if: inputs.environment == 'production' + shell: bash + env: + DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} + run: | + set -euo pipefail + : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${runtime_dir}" ]] || { echo "Refusing to reuse a job-scoped VIF runtime directory." >&2; exit 1; } + install -d -m 0700 "${runtime_dir}" + umask 077 + printf '%s' "${DEPLOY_VIF_DB_PASSWORD}" > "${runtime_dir}/vif-db-password" + chmod 0600 "${runtime_dir}/vif-db-password" + - name: Prepare deployment bundle shell: bash env: @@ -51,10 +115,10 @@ jobs: DEPLOY_VIF_DB_NETWORK: ${{ secrets.DEPLOY_VIF_DB_NETWORK }} DEPLOY_VIF_DB_NAME: ${{ secrets.DEPLOY_VIF_DB_NAME }} DEPLOY_VIF_DB_USER: ${{ secrets.DEPLOY_VIF_DB_USER }} - DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} DEPLOY_BRIO_STAGING_DB_NETWORK: ${{ secrets.DEPLOY_BRIO_STAGING_DB_NETWORK }} run: | set -euo pipefail + umask 077 deploy_env="${{ inputs.environment }}" : "${DEPLOY_CATWLK_DB_NETWORK:?set DEPLOY_CATWLK_DB_NETWORK environment secret}" : "${DEPLOY_LE_PETIT_COIN_DB_NETWORK:?set DEPLOY_LE_PETIT_COIN_DB_NETWORK environment secret}" @@ -67,7 +131,6 @@ jobs: fi if [[ "${deploy_env}" == "production" ]]; then : "${DEPLOY_VIF_DB_NETWORK:?set DEPLOY_VIF_DB_NETWORK production environment secret}" - : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" DEPLOY_VIF_DB_NAME="${DEPLOY_VIF_DB_NAME:-vif}" DEPLOY_VIF_DB_USER="${DEPLOY_VIF_DB_USER:-vif}" fi @@ -76,7 +139,7 @@ jobs: echo "Refusing to reuse an existing job-scoped deployment bundle: ${bundle_root}" >&2 exit 1 fi - mkdir -p "${bundle_root}/config" "${bundle_root}/scripts" "${bundle_root}/envs/${{ inputs.environment }}" + mkdir -p "${bundle_root}/bootstrap" "${bundle_root}/config" "${bundle_root}/scripts" "${bundle_root}/envs/${{ inputs.environment }}" cp compose.yml "${bundle_root}/compose.yml" cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh" @@ -84,6 +147,11 @@ jobs: cp scripts/run-brio-encrypted-backup.sh "${bundle_root}/scripts/run-brio-encrypted-backup.sh" cp scripts/run-brio-encrypted-backup-loop.sh "${bundle_root}/scripts/run-brio-encrypted-backup-loop.sh" cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh" + cp scripts/deploy-brio-canary-postgres.sh "${bundle_root}/scripts/deploy-brio-canary-postgres.sh" + cp scripts/brio-db-transaction.sh "${bundle_root}/scripts/brio-db-transaction.sh" + cp scripts/ensure-brio-tmp-cleaner.sh "${bundle_root}/scripts/ensure-brio-tmp-cleaner.sh" + cp bootstrap/brio-staging-app.sql "${bundle_root}/bootstrap/brio-staging-app.sql" + cp bootstrap/vif-app.sql "${bundle_root}/bootstrap/vif-app.sql" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <&2 exit 1 fi + if [[ ! "${REMOTE_DIR}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+$ ]] \ + || [[ "${REMOTE_DIR}" == *"/../"* || "${REMOTE_DIR}" == *"/.." || "${REMOTE_DIR}" == *"/./"* || "${REMOTE_DIR}" == *"/." || "${REMOTE_DIR}" == *"//"* ]]; then + echo "DEPLOY_REMOTE_DIR must be a normalized application path below /srv or /opt." >&2 + exit 1 + fi bundle_root="${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" private_key_file="${ssh_directory}/id_ed25519" @@ -131,38 +203,110 @@ jobs: ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -p "${remote_port}") scp_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -P "${remote_port}") remote_target="${REMOTE_USER}@${REMOTE_HOST}" + remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_bundle_q %q "${remote_bundle}" + printf -v remote_parent_q %q "${REMOTE_DIR}/.deploy" + # The normalized, job-scoped paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "install -d -m 0755 ${remote_parent_q} && if [ -e ${remote_bundle_q} ]; then echo 'Refusing existing remote deployment bundle.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + scp "${scp_opts[@]}" -r "${bundle_root}/." "${remote_target}:${remote_bundle}/" + # Install the host-side expiry guard before any job credential is transferred. + printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" + printf -v db_env_q %q "${remote_bundle}/envs/${{ inputs.environment }}/.env.db" + # The job-scoped paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0755 ${cleaner_q} && ${cleaner_q} ${db_env_q}" - ssh "${ssh_opts[@]}" "${remote_target}" mkdir -p "${REMOTE_DIR}/config" "${REMOTE_DIR}/scripts" "${REMOTE_DIR}/envs/${{ inputs.environment }}" - scp "${scp_opts[@]}" "${bundle_root}/compose.yml" "${remote_target}:${REMOTE_DIR}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/config/runtrace-pg_hba.conf" "${remote_target}:${REMOTE_DIR}/config/runtrace-pg_hba.conf" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup.sh" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup-loop.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup-loop.sh" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-brio-encrypted-backup.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-brio-encrypted-backup.sh" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-brio-encrypted-backup-loop.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-brio-encrypted-backup-loop.sh" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.db" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.deploy" - - scp "${scp_opts[@]}" "${bundle_root}/scripts/deploy-postgres-stack.sh" "${remote_target}:${REMOTE_DIR}/scripts/deploy-postgres-stack.sh" - - printf -v remote_script_q %q "${REMOTE_DIR}/scripts/deploy-postgres-stack.sh" - printf -v remote_dir_q %q "${REMOTE_DIR}" + remote_script="${remote_bundle}/scripts/deploy-postgres-stack.sh" + if [[ "${{ inputs.environment }}" == "canary" ]]; then + runtime_dir="${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime_dir="/tmp/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_runtime_q %q "${remote_runtime_dir}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "if [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote runtime directory.' >&2; exit 1; fi && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}"/* "${remote_target}:${remote_runtime_dir}/" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0755 ${remote_bundle_q}/scripts/brio-db-transaction.sh && chmod 0600 ${remote_runtime_q}/postgres-superuser-password ${remote_runtime_q}/brio-staging-app-password ${remote_runtime_q}/brio-staging-backup-password ${remote_runtime_q}/postgres-ca.pem ${remote_runtime_q}/postgres-server-cert.pem ${remote_runtime_q}/postgres-server-key.pem ${remote_runtime_q}/brio-backup-recipient-cert.pem" + remote_script="${remote_bundle}/scripts/deploy-brio-canary-postgres.sh" + else + runtime_dir="${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime_dir="/tmp/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_runtime_q %q "${remote_runtime_dir}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "if [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote VIF runtime directory.' >&2; exit 1; fi && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}/vif-db-password" "${remote_target}:${remote_runtime_dir}/vif-db-password" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0600 ${remote_runtime_q}/vif-db-password" + fi + printf -v remote_script_q %q "${remote_script}" printf -v stack_name_q %q "${STACK_NAME}" printf -v deploy_env_q %q "${{ inputs.environment }}" # Values are intentionally expanded locally and shell-escaped with %q. # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_dir_q} ${stack_name_q} ${deploy_env_q}" + if [[ "${{ inputs.environment }}" == "canary" ]]; then + ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${remote_runtime_q}" + else + ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${deploy_env_q} ${remote_runtime_q}" + fi + + - name: Remove remote job-scoped deployment material + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} + REMOTE_DIR: ${{ secrets.DEPLOY_REMOTE_DIR }} + run: | + set -euo pipefail + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_directory}/id_ed25519" && -r "${ssh_directory}/known_hosts" ]] || exit 0 + if [[ ! "${REMOTE_DIR}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+$ ]] \ + || [[ "${REMOTE_DIR}" == *"/../"* || "${REMOTE_DIR}" == *"/.." || "${REMOTE_DIR}" == *"/./"* || "${REMOTE_DIR}" == *"/." || "${REMOTE_DIR}" == *"//"* ]]; then + echo "Refusing cleanup with invalid DEPLOY_REMOTE_DIR." >&2 + exit 1 + fi + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_directory}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_directory}/id_ed25519" -p "${remote_port}") + if [[ "${{ inputs.environment }}" == "canary" ]]; then + runtime_dir="/tmp/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + else + runtime_dir="/tmp/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + fi + remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v runtime_dir_q %q "${runtime_dir}" + printf -v remote_bundle_q %q "${remote_bundle}" + # The fixed job paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${runtime_dir_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${runtime_dir_q} ]; then rm -f ${runtime_dir_q}/postgres-superuser-password ${runtime_dir_q}/brio-staging-app-password ${runtime_dir_q}/brio-staging-backup-password ${runtime_dir_q}/postgres-ca.pem ${runtime_dir_q}/postgres-server-cert.pem ${runtime_dir_q}/postgres-server-key.pem ${runtime_dir_q}/brio-backup-recipient-cert.pem ${runtime_dir_q}/vif-db-password; if [ -f ${runtime_dir_q}/RECOVERY_REQUIRED ] && [ ! -L ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${runtime_dir_q} -depth -delete; fi; elif [ -e ${runtime_dir_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" - name: Remove job-scoped deployment material if: always() shell: bash run: | set -euo pipefail - for path in \ + for cleanup_target in \ "${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ - "${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do - case "${path}" in - "${RUNNER_TEMP}"/postgres-deploy-*) rm -rf -- "${path}" ;; - *) echo "Refusing to remove unexpected cleanup path: ${path}" >&2; exit 1 ;; + "${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do + case "${cleanup_target}" in + "${RUNNER_TEMP}"/postgres-deploy-*|"${RUNNER_TEMP}"/postgres-brio-canary-runtime-*|"${RUNNER_TEMP}"/postgres-brio-vif-runtime-*) + if [[ -L "${cleanup_target}" ]]; then + echo "Refusing to remove cleanup symlink: ${cleanup_target}" >&2 + exit 1 + elif [[ -d "${cleanup_target}" ]]; then + find "${cleanup_target}" -mindepth 1 -delete + rmdir "${cleanup_target}" + elif [[ -e "${cleanup_target}" ]]; then + echo "Expected a cleanup directory: ${cleanup_target}" >&2 + exit 1 + fi + ;; + *) echo "Refusing to remove unexpected cleanup path: ${cleanup_target}" >&2; exit 1 ;; esac done diff --git a/bootstrap/vif-app.sql b/bootstrap/vif-app.sql new file mode 100644 index 0000000..80a1caf --- /dev/null +++ b/bootstrap/vif-app.sql @@ -0,0 +1,36 @@ +\set ON_ERROR_STOP on + +\if :{?vif_db} +\else + \echo 'missing required psql variable: vif_db' + SELECT 1 / 0; +\endif + +\if :{?vif_user} +\else + \echo 'missing required psql variable: vif_user' + SELECT 1 / 0; +\endif + +\getenv vif_password VIF_PASSWORD +SELECT CASE WHEN NULLIF(btrim(:'vif_password'), '') IS NULL THEN 'false' ELSE 'true' END AS vif_password_is_nonempty \gset +\if :vif_password_is_nonempty +\else + \echo 'empty required environment variable: VIF_PASSWORD' + SELECT 1 / 0; +\endif + +SELECT format('CREATE ROLE %I LOGIN', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec +SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec +SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = :'vif_db' + AND r.rolname <> :'vif_user' +) \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec diff --git a/compose.host.yml b/compose.host.yml index 0fcfaeb..618b606 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -89,7 +89,8 @@ services: command: - /usr/local/bin/run-brio-encrypted-backup-loop.sh environment: - PGHOST: ${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST to the DB certificate SAN hostname} + PGHOST: ${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST to the DB certificate SAN host or IP} + PGHOSTADDR: 127.0.0.1 PGPORT: "5432" PGUSER: keycloak_brio_staging_backup PGSSLMODE: verify-full diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index ff38f3d..a6d5750 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -29,7 +29,7 @@ hostnossl keycloak_brio_staging all all reject hostssl brio_staging brio_staging_app all scram-sha-256 hostssl brio_staging brio_staging_backup all scram-sha-256 hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 -hostssl keycloak_brio_staging keycloak_brio_staging_backup all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_backup 127.0.0.1/32 scram-sha-256 host all brio_staging_app all reject host all brio_staging_backup all reject host all keycloak_brio_staging_app all reject diff --git a/envs/canary/.env.db b/envs/canary/.env.db index 545c4d9..c5b2127 100644 --- a/envs/canary/.env.db +++ b/envs/canary/.env.db @@ -7,7 +7,7 @@ MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres-canary MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-canary-superuser-password MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v2 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v2 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v2 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v3 MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=/var/lib/makepad/postgres-backups/brio-staging MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-app-backup-password diff --git a/envs/production/.env.db b/envs/production/.env.db index a37d94a..acf10b9 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -5,9 +5,12 @@ POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-superuser-password +MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH=/etc/makepad/tls/postgres/server.crt +MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH=/etc/makepad/secrets/postgres-server.key +MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH=/srv/makepad/postgres/config/runtrace-pg_hba.conf MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_tls_cert_v1 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v2 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v3 MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=/var/lib/makepad/postgres-backups/runtrace MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password @@ -17,11 +20,15 @@ MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETENTION_DAYS=35 MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH=/var/lib/makepad/postgres-backups/keycloak-brio-staging MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-identity-backup-password MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=/etc/makepad/tls/backups/brio-recipient.crt +MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-runtrace-backup.sh +MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-runtrace-backup-loop.sh +MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-brio-encrypted-backup.sh +MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-brio-encrypted-backup-loop.sh MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS=35 # Standalone compose.host.yml additionally requires MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST -# to be exported as the DB VM hostname present in the PostgreSQL server certificate SAN. +# to be exported as the DB VM host or IP present in the PostgreSQL server certificate SAN. MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 diff --git a/envs/production/compose.yml b/envs/production/compose.yml index 3a6011d..76cdbe4 100644 --- a/envs/production/compose.yml +++ b/envs/production/compose.yml @@ -107,72 +107,6 @@ services: cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} - keycloak_brio_staging_backup: - image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} - user: "999:999" - command: - - /usr/local/bin/run-brio-encrypted-backup-loop.sh - environment: - PGHOST: makepad-postgres - PGPORT: "5432" - PGUSER: keycloak_brio_staging_backup - PGSSLMODE: verify-full - PGSSLROOTCERT: /etc/postgresql/ca.crt - POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password - BRIO_BACKUP_DATABASE: keycloak_brio_staging - BRIO_BACKUP_ROOT: /backups - BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt - BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} - BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} - BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} - read_only: true - tmpfs: - - /tmp:mode=0700,uid=999,gid=999 - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - healthcheck: - test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] - interval: 5m - timeout: 10s - retries: 3 - start_period: 10m - networks: - - db - volumes: - - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH}:/backups" - - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH}:/run/secrets/postgres_backup_password:ro" - - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_CA_CERT_HOST_PATH}:/etc/postgresql/ca.crt:ro" - - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH}:/etc/postgresql/brio-backup-recipient.crt:ro" - configs: - - source: brio_encrypted_backup_script - target: /usr/local/bin/run-brio-encrypted-backup.sh - mode: 0555 - - source: brio_encrypted_backup_loop_script - target: /usr/local/bin/run-brio-encrypted-backup-loop.sh - mode: 0555 - logging: - driver: json-file - options: - max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" - max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" - deploy: - replicas: 1 - placement: - constraints: - - node.labels.infra.makepad.postgres == true - restart_policy: - condition: on-failure - delay: 30s - resources: - limits: - cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT:-1.0}" - memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT:-1G} - reservations: - cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" - memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} - networks: db: external: true diff --git a/scripts/brio-db-transaction.sh b/scripts/brio-db-transaction.sh new file mode 100755 index 0000000..bb6753d --- /dev/null +++ b/scripts/brio-db-transaction.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 3)); then + echo "Usage: brio-db-transaction.sh " >&2 + exit 2 +fi + +operation=$1 +scope=$2 +journal_dir=$3 + +case "${operation}" in prepare|restore|fingerprint) ;; *) echo "Unsupported database transaction operation." >&2; exit 2 ;; esac +case "${scope}" in + brio) + database=brio_staging + app_role=brio_staging_app + backup_role=brio_staging_backup + ;; + keycloak) + database=keycloak_brio_staging + app_role=keycloak_brio_staging_app + backup_role=keycloak_brio_staging_backup + ;; + *) echo "Unsupported Brio database transaction scope." >&2; exit 2 ;; +esac + +[[ "${journal_dir}" == /* && -d "${journal_dir}" && ! -L "${journal_dir}" ]] || { + echo "The database transaction journal must be a non-symlinked absolute directory." >&2 + exit 2 +} +: "${PGUSER:?PGUSER is required}" "${PGHOST:?PGHOST is required}" "${PGPASSWORD_FILE:?PGPASSWORD_FILE is required}" +[[ -s "${PGPASSWORD_FILE}" && ! -L "${PGPASSWORD_FILE}" ]] || { + echo "PGPASSWORD_FILE must be a non-empty regular file." >&2 + exit 2 +} +export PGPASSWORD +PGPASSWORD=$(cat "${PGPASSWORD_FILE}") +export PGAPPNAME=brio-db-transaction PSQL_HISTORY=/dev/null + +psql_base=(psql -X --no-password --set ON_ERROR_STOP=1 --quiet) + +database_exists() { + [[ $("${psql_base[@]}" --tuples-only --no-align --dbname postgres \ + --command "SELECT count(*) FROM pg_database WHERE datname = '${database}'") == 1 ]] +} + +emit_fingerprint() { + local db_exists=false role + database_exists && db_exists=true + printf 'scope\t%s\n' "${scope}" + for role in "${app_role}" "${backup_role}"; do + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'role', rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, + rolcanlogin, rolreplication, rolconnlimit, coalesce(rolvaliduntil::text, ''), + rolbypassrls, coalesce(rolpassword, '') + FROM pg_authid WHERE rolname = '${role}';" + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'global-setting', r.rolname, setting + FROM pg_db_role_setting s JOIN pg_roles r ON r.oid = s.setrole + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE s.setdatabase = 0 AND r.rolname = '${role}' ORDER BY setting;" + done + printf 'database-exists\t%s\n' "${db_exists}" + if [[ "${db_exists}" == true ]]; then + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'database', d.datname, r.rolname, d.datallowconn, d.datconnlimit, + coalesce(d.datacl::text, '') + FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = '${database}'; + SELECT 'setting', role_name, setting + FROM ( + SELECT r.rolname AS role_name, unnest(s.setconfig) AS setting + FROM pg_db_role_setting s + JOIN pg_roles r ON r.oid = s.setrole + JOIN pg_database d ON d.oid = s.setdatabase + WHERE d.datname = '${database}' AND r.rolname IN ('${app_role}', '${backup_role}') + ) q ORDER BY role_name, setting;" + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname "${database}" --command " + SELECT 'namespace', n.nspname, coalesce(n.nspacl::text, '') + FROM pg_namespace n WHERE n.nspname = 'public'; + SELECT 'relation', n.nspname, c.relname, c.relkind, coalesce(c.relacl::text, '') + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r','p','v','m','f','S') + ORDER BY n.nspname, c.relname, c.relkind; + SELECT 'default-acl', owner_name, schema_name, d.defaclobjtype, + grantee_name, x.privilege_type, x.is_grantable + FROM pg_default_acl d + JOIN pg_roles owner_role ON owner_role.oid = d.defaclrole + LEFT JOIN pg_namespace n ON n.oid = d.defaclnamespace + CROSS JOIN LATERAL aclexplode(d.defaclacl) x + LEFT JOIN pg_roles grantee_role ON grantee_role.oid = x.grantee + CROSS JOIN LATERAL (VALUES(owner_role.rolname, coalesce(n.nspname, ''), + CASE WHEN x.grantee = 0 THEN 'PUBLIC' ELSE grantee_role.rolname END)) + names(owner_name, schema_name, grantee_name) + WHERE owner_role.rolname = '${app_role}' AND coalesce(n.nspname, 'public') = 'public' + ORDER BY owner_name, schema_name, d.defaclobjtype, grantee_name, x.privilege_type, x.is_grantable;" + fi +} + +write_role_restore() { + local role=$1 output=$2 exists + exists=$("${psql_base[@]}" --tuples-only --no-align --dbname postgres \ + --command "SELECT count(*) FROM pg_authid WHERE rolname = '${role}'") + if [[ "${exists}" == 0 ]]; then + printf 'DROP ROLE IF EXISTS %s;\n' "${role}" >> "${output}" + return + fi + "${psql_base[@]}" --tuples-only --no-align --dbname postgres --command " + SELECT format( + 'ALTER ROLE %I WITH %s %s %s %s %s %s CONNECTION LIMIT %s %s PASSWORD %L VALID UNTIL %L;', + rolname, + CASE WHEN rolsuper THEN 'SUPERUSER' ELSE 'NOSUPERUSER' END, + CASE WHEN rolinherit THEN 'INHERIT' ELSE 'NOINHERIT' END, + CASE WHEN rolcreaterole THEN 'CREATEROLE' ELSE 'NOCREATEROLE' END, + CASE WHEN rolcreatedb THEN 'CREATEDB' ELSE 'NOCREATEDB' END, + CASE WHEN rolcanlogin THEN 'LOGIN' ELSE 'NOLOGIN' END, + CASE WHEN rolreplication THEN 'REPLICATION' ELSE 'NOREPLICATION' END, + rolconnlimit, + CASE WHEN rolbypassrls THEN 'BYPASSRLS' ELSE 'NOBYPASSRLS' END, + rolpassword, + coalesce(rolvaliduntil::text, 'infinity')) + FROM pg_authid WHERE rolname = '${role}'; + SELECT format('UPDATE pg_authid SET rolvaliduntil = NULL WHERE rolname = %L;', rolname) + FROM pg_authid WHERE rolname = '${role}' AND rolvaliduntil IS NULL; + SELECT format('ALTER ROLE %I RESET ALL;', rolname) + FROM pg_authid WHERE rolname = '${role}'; + SELECT format('ALTER ROLE %I SET %I TO %L;', r.rolname, + split_part(setting, '=', 1), substr(setting, strpos(setting, '=') + 1)) + FROM pg_db_role_setting s JOIN pg_roles r ON r.oid = s.setrole + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE s.setdatabase = 0 AND r.rolname = '${role}' ORDER BY setting;" >> "${output}" +} + +prepare_journal() { + local restore_tmp="${journal_dir}/.restore.sql.tmp" fingerprint_tmp="${journal_dir}/.prestate.fingerprint.tmp" + local restore_file="${journal_dir}/restore.sql" fingerprint_file="${journal_dir}/prestate.fingerprint" + for path in "${restore_file}" "${fingerprint_file}" "${restore_tmp}" "${fingerprint_tmp}"; do + [[ ! -e "${path}" && ! -L "${path}" ]] || { echo "Refusing to overwrite database journal material." >&2; exit 1; } + done + umask 077 + { + printf '%s\n' '\set ON_ERROR_STOP on' 'SET client_min_messages = warning;' + if database_exists; then + printf '\\connect %s\n' "${database}" + # These catalogs are the exact objects changed by the Brio bootstrap. The + # fixed OID/name predicates make the compensation fail closed on drift. + "${psql_base[@]}" --tuples-only --no-align --dbname "${database}" --command " + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_namespace WHERE oid = %s AND nspname = %L) THEN RAISE EXCEPTION ''namespace identity drift''; END IF; END \$do\$; UPDATE pg_namespace SET nspacl = %s WHERE oid = %s AND nspname = %L;', + oid, nspname, + CASE WHEN nspacl IS NULL THEN 'NULL' ELSE quote_literal(nspacl::text) || '::aclitem[]' END, + oid, nspname) + FROM pg_namespace WHERE nspname = 'public'; + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_class WHERE oid = %s AND relnamespace = %s AND relname = %L AND relkind = %L) THEN RAISE EXCEPTION ''relation identity drift''; END IF; END \$do\$; UPDATE pg_class SET relacl = %s WHERE oid = %s AND relnamespace = %s AND relname = %L AND relkind = %L;', + c.oid, c.relnamespace, c.relname, c.relkind, + CASE WHEN c.relacl IS NULL THEN 'NULL' ELSE quote_literal(c.relacl::text) || '::aclitem[]' END, + c.oid, c.relnamespace, c.relname, c.relkind) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r','p','v','m','f','S') + ORDER BY c.oid; + SELECT format('ALTER DEFAULT PRIVILEGES FOR ROLE %I IN SCHEMA public REVOKE ALL ON %s FROM %I;', + '${app_role}', CASE defaclobjtype WHEN 'r' THEN 'TABLES' ELSE 'SEQUENCES' END, '${backup_role}') + FROM (VALUES ('r'::\"char\"), ('S'::\"char\")) kinds(defaclobjtype); + SELECT format('ALTER DEFAULT PRIVILEGES FOR ROLE %I IN SCHEMA public GRANT %s ON %s TO %I%s;', + owner_role.rolname, string_agg(DISTINCT x.privilege_type, ', ' ORDER BY x.privilege_type), + CASE d.defaclobjtype WHEN 'r' THEN 'TABLES' ELSE 'SEQUENCES' END, + '${backup_role}', CASE WHEN bool_and(x.is_grantable) THEN ' WITH GRANT OPTION' ELSE '' END) + FROM pg_default_acl d + JOIN pg_roles owner_role ON owner_role.oid = d.defaclrole + JOIN pg_namespace n ON n.oid = d.defaclnamespace + CROSS JOIN LATERAL aclexplode(d.defaclacl) x + JOIN pg_roles grantee_role ON grantee_role.oid = x.grantee + WHERE owner_role.rolname = '${app_role}' AND n.nspname = 'public' + AND grantee_role.rolname = '${backup_role}' AND d.defaclobjtype IN ('r','S') + GROUP BY owner_role.rolname, d.defaclobjtype, x.is_grantable + ORDER BY d.defaclobjtype, x.is_grantable;" + printf '%s\n' '\connect postgres' + "${psql_base[@]}" --tuples-only --no-align --dbname postgres --command " + SELECT format('ALTER DATABASE %I OWNER TO %I;', d.datname, r.rolname) + FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = '${database}'; + SELECT format('ALTER DATABASE %I %s; ALTER DATABASE %I CONNECTION LIMIT %s;', + datname, CASE WHEN datallowconn THEN 'ALLOW_CONNECTIONS true' ELSE 'ALLOW_CONNECTIONS false' END, + datname, datconnlimit) + FROM pg_database WHERE datname = '${database}'; + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_database WHERE oid = %s AND datname = %L) THEN RAISE EXCEPTION ''database identity drift''; END IF; END \$do\$; UPDATE pg_database SET datacl = %s WHERE oid = %s AND datname = %L;', + oid, datname, + CASE WHEN datacl IS NULL THEN 'NULL' ELSE quote_literal(datacl::text) || '::aclitem[]' END, + oid, datname) + FROM pg_database WHERE datname = '${database}'; + SELECT format('ALTER ROLE %I IN DATABASE %I RESET ALL;', r.rolname, d.datname) + FROM pg_roles r CROSS JOIN pg_database d + WHERE r.rolname IN ('${app_role}', '${backup_role}') AND d.datname = '${database}' + ORDER BY r.rolname; + SELECT format('ALTER ROLE %I IN DATABASE %I SET %I TO %L;', r.rolname, d.datname, + split_part(setting, '=', 1), substr(setting, strpos(setting, '=') + 1)) + FROM pg_db_role_setting s + JOIN pg_roles r ON r.oid = s.setrole + JOIN pg_database d ON d.oid = s.setdatabase + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE r.rolname IN ('${app_role}', '${backup_role}') AND d.datname = '${database}' + ORDER BY r.rolname, setting;" + else + printf "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid();\n" "${database}" + printf 'DROP DATABASE IF EXISTS %s;\n' "${database}" + fi + } > "${restore_tmp}" + write_role_restore "${app_role}" "${restore_tmp}" + write_role_restore "${backup_role}" "${restore_tmp}" + emit_fingerprint > "${fingerprint_tmp}" + chmod 0600 "${restore_tmp}" "${fingerprint_tmp}" + mv -T "${restore_tmp}" "${restore_file}" + mv -T "${fingerprint_tmp}" "${fingerprint_file}" + sync -f "${journal_dir}" 2>/dev/null || sync +} + +restore_journal() { + [[ -s "${journal_dir}/restore.sql" && ! -L "${journal_dir}/restore.sql" \ + && -s "${journal_dir}/prestate.fingerprint" && ! -L "${journal_dir}/prestate.fingerprint" ]] || { + echo "The database transaction journal is incomplete." >&2 + exit 1 + } + "${psql_base[@]}" --dbname postgres --file "${journal_dir}/restore.sql" >/dev/null + local restored="${journal_dir}/.restored.fingerprint.tmp" + emit_fingerprint > "${restored}" + if ! cmp -s "${journal_dir}/prestate.fingerprint" "${restored}"; then + echo "Database compensation did not restore the exact prior Brio state." >&2 + # The fingerprint deliberately contains PostgreSQL SCRAM verifiers so the + # recovery check can prove exact credential restoration. Never emit either + # side of a mismatch: CI and remote deployment logs are not secret stores. + rm -f -- "${restored}" + exit 1 + fi + rm -f -- "${restored}" +} + +case "${operation}" in + prepare) prepare_journal ;; + restore) restore_journal ;; + fingerprint) emit_fingerprint ;; +esac diff --git a/scripts/deploy-brio-canary-postgres.sh b/scripts/deploy-brio-canary-postgres.sh new file mode 100755 index 0000000..e3326b2 --- /dev/null +++ b/scripts/deploy-brio-canary-postgres.sh @@ -0,0 +1,831 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 3)); then + echo "Usage: deploy-brio-canary-postgres.sh " >&2 + exit 2 +fi + +remote_dir=$1 +stack_name=$2 +runtime_dir=$3 +db_env="${remote_dir}/envs/canary/.env.db" +env_deploy="${remote_dir}/envs/canary/.env.deploy" +rollback_armed=0 +stack_mutated=0 +stack_preexisting=0 +db_mutated=0 +postgres_image= +validation_image= +declare -a missing_configs=() +declare -a missing_secrets=() +declare -a missing_networks=() +deployment_id=${runtime_dir##*/postgres-brio-canary-runtime-} +recovery_root=/var/lib/makepad/postgres-recovery/brio-canary +journal_dir="${recovery_root}/${deployment_id}" +failure_injection=${BRIO_DEPLOY_FAILURE_INJECTION:-} + +if [[ -n "${failure_injection}" ]]; then + [[ "${BRIO_DEPLOY_TEST_MODE:-}" == "isolated-container" && -f /.dockerenv ]] || { + echo "Failure injection is permitted only in the isolated deployment test container." >&2 + exit 2 + } + case "${failure_injection}" in + after-managed-file-promotion|term-after-managed-file-promotion|kill-after-managed-file-promotion|after-stack-deploy|after-bootstrap|kill-after-bootstrap|after-app-probe|after-plaintext-probe|after-nontarget-probe|after-backup-role-probe|after-backup-verification|rollback-restore) ;; + *) echo "Unsupported canary deployment failure injection." >&2; exit 2 ;; + esac +fi + +if [[ ! "${remote_dir}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+/\.deploy/postgres-[0-9]+-[0-9]+$ ]] \ + || [[ "${remote_dir}" == *"/../"* || "${remote_dir}" == *"/.." || "${remote_dir}" == *"/./"* || "${remote_dir}" == *"/." || "${remote_dir}" == *"//"* ]]; then + echo "remote-dir must be a unique /srv or /opt .deploy/postgres-- path." >&2 + exit 2 +fi +[[ "${runtime_dir}" =~ ^/tmp/postgres-brio-canary-runtime-[0-9]+-[0-9]+$ ]] || { echo "runtime-secret-dir must be a job-scoped /tmp/postgres-brio-canary-runtime-- path." >&2; exit 2; } +case "${stack_name}" in ''|*[!a-zA-Z0-9_-]*) echo "stack-name contains unsupported characters." >&2; exit 2 ;; esac + +cleanup_runtime() { + local name + for name in postgres-superuser-password brio-staging-app-password brio-staging-backup-password postgres-ca.pem postgres-server-cert.pem postgres-server-key.pem brio-backup-recipient-cert.pem; do + [[ ! -f "${runtime_dir}/${name}" || -L "${runtime_dir}/${name}" ]] || rm -f -- "${runtime_dir:?}/${name}" + done + if [[ -f "${runtime_dir}/RECOVERY_REQUIRED" && ! -L "${runtime_dir}/RECOVERY_REQUIRED" ]]; then + return + fi + if [[ -e "${runtime_dir}/RECOVERY_REQUIRED" || -L "${runtime_dir}/RECOVERY_REQUIRED" ]]; then + echo "Unsafe recovery marker type; preserving the canary runtime for operator inspection." >&2 + return 1 + fi + rm -f -- "${runtime_dir}/candidate-stack.yml" "${runtime_dir}/prior-services.list" + rm -f -- "${runtime_dir}/prior-service-spec-hashes.list" + if [[ -d "${runtime_dir}/prior-service-specs" && ! -L "${runtime_dir}/prior-service-specs" ]]; then + find "${runtime_dir}/prior-service-specs" -depth -delete + fi + rmdir -- "${runtime_dir}" 2>/dev/null || true +} + +handle_exit() { + local status=$? + trap - EXIT HUP INT TERM + if [[ "${rollback_armed}" == "1" ]]; then + if ! rollback_canary; then + journal_marker "${journal_dir}" RECOVERY_REQUIRED || true + { + printf 'deployment_id=%s\n' "${runtime_dir##*/postgres-brio-canary-runtime-}" + printf 'reason=%s\n' 'automatic-canary-rollback-failed' + } > "${runtime_dir}/RECOVERY_REQUIRED" + chmod 0600 "${runtime_dir}/RECOVERY_REQUIRED" + echo "Automatic canary rollback failed; root-protected snapshot retained in ${journal_dir}." >&2 + status=1 + fi + fi + cleanup_runtime || status=1 + exit "${status}" +} +trap handle_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +read_setting() { + local name=$1 file=$2 value + value=$(grep -E "^${name}=" "${file}" | tail -n 1 | cut -d= -f2-) + [[ -n "${value}" ]] || { echo "${name} is missing or empty in ${file}." >&2; exit 1; } + printf '%s' "${value}" +} + +run_db_transaction() { + local operation=$1 source_journal=${2:-${journal_dir}} helper + if [[ "${operation}" == prepare ]]; then helper="${remote_dir}/scripts/brio-db-transaction.sh"; else helper="${source_journal}/brio-db-transaction.sh"; fi + [[ -f "${helper}" && ! -L "${helper}" ]] || { echo "Canary database compensation helper is unavailable." >&2; return 1; } + # The Brio-only network is deliberately allowed to be absent on a first + # deployment. Journal preparation and recovery therefore use the already + # validated shared database network and its certificate SAN. + docker run --rm --network "${db_network}" \ + --mount "type=bind,src=${helper},dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + --mount "type=bind,src=${source_journal},dst=/journal" \ + --mount "type=bind,src=${superuser_host_file},dst=/run/secrets/postgres_superuser_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + -e "PGUSER=${postgres_user}" -e PGHOST=makepad-postgres \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e PGPASSWORD_FILE=/run/secrets/postgres_superuser_password \ + "${postgres_image}" /usr/local/bin/brio-db-transaction.sh "${operation}" brio /journal/database +} + +journal_marker() { + local source_journal=$1 marker=$2 + case "${marker}" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; + *) echo "Refusing unsupported canary journal marker." >&2; return 1 ;; + esac + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 1 ;; esac + [ -d /journal ] && [ ! -L /journal ] + tmp="/journal/.${MARKER}.tmp" + [ ! -e "$tmp" ] && [ ! -L "$tmp" ] + printf "%s\n" "$MARKER" > "$tmp" + chmod 0600 "$tmp" + mv -fT "$tmp" "/journal/$MARKER" + sync -f /journal 2>/dev/null || sync + ' +} + +remove_durable_journal() { + local source_journal=$1 + docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "JOURNAL=${source_journal}" "${validation_image}" sh -euc ' + case "$JOURNAL" in /var/lib/makepad/postgres-recovery/brio-canary/*) identifier=${JOURNAL##*/} ;; *) exit 1 ;; esac + case "$identifier" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$identifier" in *-*) ;; *) exit 1 ;; esac + target="/managed-var-lib${JOURNAL#/var/lib/makepad}" + [ -d "$target" ] && [ ! -L "$target" ] + find "$target" -depth -delete + sync -f /managed-var-lib/postgres-recovery/brio-canary 2>/dev/null || sync + ' +} + +journal_has_marker() { + local source_journal=$1 marker=$2 + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal,readonly" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 2 ;; esac + [ -f "/journal/$MARKER" ] && [ ! -L "/journal/$MARKER" ] + ' >/dev/null 2>&1 +} + +rollback_canary() { + local rollback_status=0 service_name previous_spec name expected_hash current_hash + echo "Canary deployment failed after the mutation boundary; restoring exact database, Swarm specs, and managed host files." >&2 + if [[ "${db_mutated}" == 1 ]] || journal_has_marker "${journal_dir}" DATABASE_MUTATION_ARMED; then + run_db_transaction restore "${journal_dir}" || rollback_status=1 + fi + if [[ "${failure_injection}" == rollback-restore ]]; then + rollback_status=1 + else + if [[ "${stack_mutated}" == 1 ]] || journal_has_marker "${journal_dir}" STACK_MUTATION_ARMED; then + while IFS= read -r service_name; do + if ! docker run --rm --mount "type=bind,src=${journal_dir},dst=/journal,readonly" \ + -e "SERVICE=${service_name}" "${validation_image}" sh -euc 'grep -Fxq "$SERVICE" /journal/swarm/prior-services.list'; then + docker service rm "${service_name}" >/dev/null || rollback_status=1 + fi + done < <(docker stack services "${stack_name}" --format '{{.Name}}' 2>/dev/null || true) + while IFS='|' read -r service_name expected_hash; do + docker service inspect "${service_name}" >/dev/null 2>&1 || { rollback_status=1; continue; } + current_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + if [[ "${current_hash}" != "${expected_hash}" ]]; then + previous_spec=$(docker service inspect "${service_name}" --format '{{if .PreviousSpec}}present{{else}}absent{{end}}') + if [[ "${previous_spec}" == present ]]; then + docker service rollback --detach=false "${service_name}" >/dev/null || rollback_status=1 + else + rollback_status=1 + fi + fi + current_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + [[ "${current_hash}" == "${expected_hash}" ]] || rollback_status=1 + done < <(docker run --rm --mount "type=bind,src=${journal_dir},dst=/journal,readonly" \ + "${validation_image}" cat /journal/swarm/prior-service-spec-hashes.list) + fi + docker run --rm \ + --mount type=bind,src=/etc,dst=/host/etc \ + --mount type=bind,src=/var/lib,dst=/host/var/lib \ + --mount "type=bind,src=${journal_dir}/rollback,dst=/rollback,readonly" \ + "${validation_image}" sh -euc ' + backup=/host/var/lib/makepad/postgres-backups/brio-staging + if [ -d "$backup" ] && [ ! -L "$backup" ]; then + for child in "$backup"/* "$backup"/.*; do + [ -e "$child" ] || [ -L "$child" ] || continue + name=${child##*/}; case "$name" in .|..|latest|last-success.json) continue ;; esac + if ! grep -Fxq "$name" /rollback/backup-entries.list; then + case "$name" in 20??????T??????Z|.20??????T??????Z.partial) ;; *) echo "Unsafe unexpected backup entry: $name" >&2; exit 1 ;; esac + [ ! -L "$child" ] || { echo "Unexpected backup entry is a symlink." >&2; exit 1; } + find "$child" -depth -delete + fi + done + rm -f -- "$backup/latest" "$backup/last-success.json" + if [ -f /rollback/prior-latest-target ]; then + target=$(cat /rollback/prior-latest-target) + case "$target" in 20??????T??????Z) ;; *) exit 1 ;; esac + ln -s "$target" "$backup/latest" + fi + if [ -f /rollback/prior-last-success.json ]; then cp -a /rollback/prior-last-success.json "$backup/last-success.json"; fi + fi + while IFS= read -r path; do + case "$path" in + etc/makepad/secrets/postgres-canary-superuser-password|etc/makepad/tls/postgres/ca.crt|etc/makepad/secrets/postgres-brio-app-backup-password|etc/makepad/tls/backups/brio-recipient.crt) rm -f -- "/host/$path" ;; + var/lib/makepad/postgres-backups/brio-staging) + if ! rmdir -- "/host/$path" 2>/dev/null; then echo "New backup directory is not empty." >&2; exit 1; fi ;; + *) echo "Unexpected absent rollback path." >&2; exit 1 ;; + esac + done < /rollback/absent.list + tar --numeric-owner -xpf /rollback/managed.tar -C /host + ' || rollback_status=1 + fi + for object_kind in secret config network; do + while IFS= read -r name; do + [[ -n "${name}" ]] || continue + if docker "${object_kind}" inspect "${name}" >/dev/null 2>&1; then + if [[ "${object_kind}" == network ]]; then + owner=$(docker network inspect "${name}" --format '{{index .Labels "makepad.brio.deployment-id"}}') + else + owner=$(docker "${object_kind}" inspect "${name}" --format '{{index .Spec.Labels "makepad.brio.deployment-id"}}') + fi + [[ "${owner}" == "${deployment_id}" ]] && docker "${object_kind}" rm "${name}" >/dev/null || rollback_status=1 + fi + done < <(docker "${object_kind}" ls --filter "label=makepad.brio.deployment-id=${deployment_id}" --format '{{.Name}}' 2>/dev/null || true) + done + if [[ "${rollback_status}" == 0 ]]; then journal_marker "${journal_dir}" ROLLED_BACK || rollback_status=1; fi + if [[ "${rollback_status}" == 0 ]]; then remove_durable_journal "${journal_dir}" || rollback_status=1; fi + return "${rollback_status}" +} + +recover_incomplete_journals() { + local current_id="${deployment_id}" current_journal="${journal_dir}" saved_injection="${failure_injection}" + local listing state identifier + listing=$(docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib,readonly \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-canary + [ ! -e "$root" ] && exit 0 + [ -d "$root" ] && [ ! -L "$root" ] + for candidate in "$root"/* "$root"/.*; do + [ -e "$candidate" ] || continue + name=${candidate##*/}; case "$name" in .|..) continue ;; .*.staging) echo "INCOMPLETE:$name"; continue ;; esac + case "$name" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "UNSAFE:$name"; continue ;; esac + case "$name" in *-*) ;; *) echo "UNSAFE:$name"; continue ;; esac + [ -d "$candidate" ] && [ ! -L "$candidate" ] || { echo "UNSAFE:$name"; continue; } + if [ -f "$candidate/COMMITTED" ] && [ ! -L "$candidate/COMMITTED" ]; then echo "COMMITTED:$name" + elif [ -f "$candidate/ROLLED_BACK" ] && [ ! -L "$candidate/ROLLED_BACK" ]; then echo "ROLLED_BACK:$name" + elif [ -f "$candidate/IN_PROGRESS" ] && [ ! -L "$candidate/IN_PROGRESS" ]; then echo "PENDING:$name" + else echo "UNSAFE:$name"; fi + done + ') + while IFS=: read -r state identifier; do + [[ -n "${state}" ]] || continue + case "${state}" in + COMMITTED|ROLLED_BACK) remove_durable_journal "${recovery_root}/${identifier}" ;; + PENDING) + deployment_id=${identifier} + journal_dir="${recovery_root}/${identifier}" + db_mutated=0; stack_mutated=0 + journal_has_marker "${journal_dir}" DATABASE_MUTATION_ARMED && db_mutated=1 + journal_has_marker "${journal_dir}" STACK_MUTATION_ARMED && stack_mutated=1 + failure_injection= + if ! rollback_canary; then + journal_marker "${journal_dir}" RECOVERY_REQUIRED || true + echo "Interrupted canary transaction could not be recovered." >&2 + return 1 + fi + ;; + INCOMPLETE|UNSAFE) echo "Unsafe or incomplete canary journal detected: ${identifier}" >&2; return 1 ;; + *) return 1 ;; + esac + done <<< "${listing}" + deployment_id=${current_id} + journal_dir=${current_journal} + failure_injection=${saved_injection} + db_mutated=0; stack_mutated=0 +} + +postgres_image=$(read_setting POSTGRES_IMAGE "${db_env}") +validation_image=$(read_setting BRIO_BACKUP_IMAGE "${db_env}") +postgres_user=$(read_setting POSTGRES_USER "${db_env}") +superuser_host_file=$(read_setting MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH "${db_env}") +ca_host_file=$(read_setting MAKEPAD_POSTGRES_CA_CERT_HOST_PATH "${db_env}") +backup_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH "${db_env}") +recipient_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH "${db_env}") +backup_host_dir=$(read_setting MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH "${db_env}") +tls_cert_config=$(read_setting MAKEPAD_POSTGRES_TLS_CERT_CONFIG "${db_env}") +tls_key_secret=$(read_setting MAKEPAD_POSTGRES_TLS_KEY_SECRET "${db_env}") +hba_config=$(read_setting MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG "${db_env}") +brio_network=$(read_setting MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK "${env_deploy}") +db_network=$(read_setting MAKEPAD_POSTGRES_DB_NETWORK "${env_deploy}") +le_petit_coin_network=$(read_setting MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK "${env_deploy}") + +if [[ "${brio_network}" != "makepad_brio_staging_db" ]]; then + echo "Canary provisioning requires makepad_brio_staging_db." >&2 + exit 1 +fi +[[ "${superuser_host_file}" == "/etc/makepad/secrets/postgres-canary-superuser-password" \ + && "${ca_host_file}" == "/etc/makepad/tls/postgres/ca.crt" \ + && "${backup_host_file}" == "/etc/makepad/secrets/postgres-brio-app-backup-password" \ + && "${recipient_host_file}" == "/etc/makepad/tls/backups/brio-recipient.crt" \ + && "${backup_host_dir}" == "/var/lib/makepad/postgres-backups/brio-staging" \ + && "${tls_cert_config}" == "makepad_postgres_canary_tls_cert_v2" \ + && "${tls_key_secret}" == "makepad_postgres_canary_tls_key_v2" \ + && "${hba_config}" == "makepad_postgres_canary_runtrace_hba_v3" ]] || { + echo "Canary environment does not match the exact reviewed managed-path and immutable-object contract." >&2 + exit 1 +} +for network_name in "${db_network}" "${le_petit_coin_network}" "${brio_network}"; do + [[ "${network_name}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$ ]] || { + echo "Canary network name is not normalized: ${network_name}" >&2 + exit 1 + } +done +[[ "${db_network}" != "${le_petit_coin_network}" && "${db_network}" != "${brio_network}" \ + && "${le_petit_coin_network}" != "${brio_network}" ]] || { + echo "Canary database networks must be distinct." >&2 + exit 1 +} + +assert_no_symlink_components() { + local path=$1 current='' component + local -a components=() + [[ "${path}" == /* && "${path}" != *"//"* && "${path}" != *"/./"* \ + && "${path}" != */. && "${path}" != *"/../"* && "${path}" != */.. ]] || { + echo "Managed path is not a normalized absolute path: ${path}" >&2 + return 1 + } + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + [[ -n "${component}" && "${component}" != "." && "${component}" != ".." ]] || { + echo "Managed path contains an unsafe component: ${path}" >&2 + return 1 + } + current="${current}/${component}" + [[ ! -L "${current}" ]] || { + echo "Managed path contains a symlink component: ${current}" >&2 + return 1 + } + done +} +for managed_path in "${superuser_host_file}" "${ca_host_file}" "${backup_host_file}" "${recipient_host_file}" "${backup_host_dir}" \ + /var/lib/makepad/postgres-recovery "${recovery_root}"; do + assert_no_symlink_components "${managed_path}" +done +for managed_parent in \ + /etc/makepad/secrets \ + /etc/makepad/tls/postgres \ + /etc/makepad/tls/backups \ + /var/lib/makepad/postgres-backups; do + [[ -d "${managed_parent}" && ! -L "${managed_parent}" ]] || { + echo "Required exact managed parent is missing, not a directory, or a symlink: ${managed_parent}" >&2 + exit 1 + } +done +for managed_file in "${superuser_host_file}" "${ca_host_file}" "${backup_host_file}" "${recipient_host_file}"; do + [[ ! -e "${managed_file}" || -f "${managed_file}" ]] || { + echo "Managed file destination has an unexpected type: ${managed_file}" >&2 + exit 1 + } +done +[[ ! -e "${backup_host_dir}" || -d "${backup_host_dir}" ]] || { + echo "Managed backup destination has an unexpected type: ${backup_host_dir}" >&2 + exit 1 +} + +require_runtime_file() { + local path="${runtime_dir}/$1" mode + if [[ ! -s "${path}" || -L "${path}" ]]; then + echo "Required job-scoped input is missing, empty, or a symlink: $1" >&2 + exit 1 + fi + mode=$(stat -c '%a' "${path}") + if [[ "${mode}" != "600" ]]; then + echo "Job-scoped input must have mode 0600: $1" >&2 + exit 1 + fi +} + +runtime_mode=$(stat -c '%a' "${runtime_dir}") +[[ ! -L "${runtime_dir}" && "${runtime_mode}" == "700" ]] || { + echo "Job-scoped runtime directory must be a non-symlink with mode 0700." >&2 + exit 1 +} +for input in \ + postgres-superuser-password \ + brio-staging-app-password \ + brio-staging-backup-password \ + postgres-ca.pem \ + postgres-server-cert.pem \ + postgres-server-key.pem \ + brio-backup-recipient-cert.pem; do + require_runtime_file "${input}" +done +for password_file in postgres-superuser-password brio-staging-app-password brio-staging-backup-password; do + if [[ $(awk 'END { print NR }' "${runtime_dir}/${password_file}") -ne 1 ]] || grep -q $'\r' "${runtime_dir}/${password_file}"; then + echo "Password input must contain one line and no carriage return: ${password_file}" >&2 + exit 1 + fi +done +if cmp -s "${runtime_dir}/postgres-superuser-password" "${runtime_dir}/brio-staging-app-password" \ + || cmp -s "${runtime_dir}/postgres-superuser-password" "${runtime_dir}/brio-staging-backup-password" \ + || cmp -s "${runtime_dir}/brio-staging-app-password" "${runtime_dir}/brio-staging-backup-password"; then + echo "PostgreSQL superuser, Brio application, and Brio backup credentials must all be distinct." >&2 + exit 1 +fi + +command -v openssl >/dev/null 2>&1 || { echo "openssl is required." >&2; exit 1; } +openssl x509 -in "${runtime_dir}/postgres-ca.pem" -noout -checkend 604800 >/dev/null +openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -noout -checkend 604800 >/dev/null +openssl verify -purpose sslserver -CAfile "${runtime_dir}/postgres-ca.pem" "${runtime_dir}/postgres-server-cert.pem" >/dev/null +openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -noout -checkhost makepad-postgres-brio-staging >/dev/null +cert_key_hash=$(openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -pubkey -noout | openssl pkey -pubin -outform DER | sha256sum | cut -d' ' -f1) +private_key_hash=$(openssl pkey -in "${runtime_dir}/postgres-server-key.pem" -pubout -outform DER | sha256sum | cut -d' ' -f1) +[[ "${cert_key_hash}" == "${private_key_hash}" ]] || { echo "PostgreSQL TLS certificate and private key do not match." >&2; exit 1; } +if grep -q -- 'PRIVATE KEY' "${runtime_dir}/brio-backup-recipient-cert.pem" \ + || ! openssl x509 -in "${runtime_dir}/brio-backup-recipient-cert.pem" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${runtime_dir}/brio-backup-recipient-cert.pem" -out /dev/null; then + echo "Brio backup recipient must be a valid public encryption certificate with at least seven days remaining." >&2 + exit 1 +fi + +prevalidate_swarm_config() { + local name=$1 source=$2 kind=$3 digest deployed_digest actual_file + digest=$(sha256sum "${source}" | cut -d' ' -f1) + if docker config inspect "${name}" >/dev/null 2>&1; then + deployed_digest=$(docker config inspect "${name}" --format '{{index .Spec.Labels "content-sha256"}}') + [[ "${deployed_digest}" == "${digest}" ]] || { + echo "${kind} config ${name} has different content; create a new versioned name." >&2 + return 1 + } + actual_file=$(mktemp) + docker config inspect "${name}" --format '{{printf "%s" .Spec.Data}}' > "${actual_file}" + if [[ $(sha256sum "${actual_file}" | cut -d' ' -f1) != "${digest}" ]]; then + rm -f "${actual_file}" + echo "${kind} config ${name} content does not match its label." >&2 + return 1 + fi + rm -f "${actual_file}" + else + missing_configs+=("${name}|${source}|${digest}") + fi +} + +prevalidate_swarm_secret() { + local name=$1 source=$2 digest deployed_digest + digest=$(sha256sum "${source}" | cut -d' ' -f1) + if docker secret inspect "${name}" >/dev/null 2>&1; then + deployed_digest=$(docker secret inspect "${name}" --format '{{index .Spec.Labels "content-sha256"}}') + [[ "${deployed_digest}" == "${digest}" ]] || { + echo "TLS private-key secret ${name} cannot be replaced in place; create a new versioned name." >&2 + return 1 + } + else + missing_secrets+=("${name}|${source}|${digest}") + fi +} + +prevalidate_network() { + local name=$1 required_internal=$2 details + if docker network inspect "${name}" >/dev/null 2>&1; then + details=$(docker network inspect "${name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') + if [[ "${required_internal}" == "true" ]]; then + [[ "${details}" == "overlay swarm true true true" ]] || { + echo "Network ${name} must be an internal, attachable, encrypted Swarm overlay; got ${details}." >&2 + return 1 + } + else + [[ "${details}" == "overlay swarm false true true" ]] || { + echo "Network ${name} must be an external, attachable, encrypted Swarm overlay; got ${details}." >&2 + return 1 + } + fi + else + missing_networks+=("${name}|${required_internal}") + fi +} + +# Validate every immutable object, network, and rendered stack before crossing +# any host or Swarm mutation boundary. +[[ -f "${remote_dir}/compose.yml" && ! -L "${remote_dir}/compose.yml" \ + && -f "${remote_dir}/envs/canary/compose.yml" && ! -L "${remote_dir}/envs/canary/compose.yml" \ + && -f "${remote_dir}/config/runtrace-pg_hba.conf" && ! -L "${remote_dir}/config/runtrace-pg_hba.conf" \ + && -x "${remote_dir}/scripts/deploy-postgres-stack.sh" && ! -L "${remote_dir}/scripts/deploy-postgres-stack.sh" \ + && -x "${remote_dir}/scripts/brio-db-transaction.sh" && ! -L "${remote_dir}/scripts/brio-db-transaction.sh" ]] || { + echo "Canary bundle inputs are missing or symlinked." >&2 + exit 1 +} + +# Recovery and first-deployment journal capture use the pre-existing shared +# PostgreSQL network, because the Brio-only network may not exist yet. Validate +# that transport boundary before a recovery helper can receive a superuser +# credential or connect to the database alias. +prevalidate_network "${db_network}" false +docker network inspect "${db_network}" >/dev/null 2>&1 || { + echo "The pre-existing shared database network ${db_network} is required for crash-safe journal preparation." >&2 + exit 1 +} + +# A SIGKILL can leave immutable objects created by the interrupted run. Recover +# its root-owned journal before inventorying those objects so the current +# attempt cannot mistake interrupted-run state for its own validated pre-state. +recover_incomplete_journals + +prevalidate_swarm_config "${tls_cert_config}" "${runtime_dir}/postgres-server-cert.pem" "TLS certificate" +prevalidate_swarm_secret "${tls_key_secret}" "${runtime_dir}/postgres-server-key.pem" +prevalidate_swarm_config "${hba_config}" "${remote_dir}/config/runtrace-pg_hba.conf" "HBA policy" +prevalidate_network "${le_petit_coin_network}" false +prevalidate_network "${brio_network}" true + +export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" +export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_network}" +export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_network}" +docker compose \ + --env-file "${db_env}" \ + --env-file "${env_deploy}" \ + -f "${remote_dir}/compose.yml" \ + -f "${remote_dir}/envs/canary/compose.yml" \ + config > "${runtime_dir}/candidate-stack.yml" +docker stack config --compose-file "${runtime_dir}/candidate-stack.yml" >/dev/null + +: > "${runtime_dir}/prior-services.list" +: > "${runtime_dir}/prior-service-spec-hashes.list" +install -d -m 0700 "${runtime_dir}/prior-service-specs" +if docker stack services "${stack_name}" --format '{{.Name}}' > "${runtime_dir}/prior-services.list" 2>/dev/null \ + && [[ -s "${runtime_dir}/prior-services.list" ]]; then + stack_preexisting=1 + while IFS= read -r service_name; do + [[ "${service_name}" == "${stack_name}_"* ]] || { + echo "Existing stack contains an unexpected service identity: ${service_name}" >&2 + exit 1 + } + [[ $(docker service inspect "${service_name}" --format '{{index .Spec.Labels "com.docker.stack.namespace"}}') == "${stack_name}" ]] || { + echo "Existing service ${service_name} does not carry the exact stack namespace label." >&2 + exit 1 + } + if [[ "${service_name}" == "${stack_name}_keycloak_brio_staging_backup" ]]; then + legacy_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + echo "Legacy identity backup service ${service_name} is still present (spec sha256 ${legacy_hash})." >&2 + echo "Retire it through a separately reviewed operation only after standalone keycloak_brio_staging backup acceptance; this deployment will not prune it." >&2 + exit 1 + fi + spec_file="${runtime_dir}/prior-service-specs/${service_name}.json" + docker service inspect "${service_name}" --format '{{json .Spec}}' > "${spec_file}" + [[ -s "${spec_file}" && ! -L "${spec_file}" ]] || { echo "Failed to inventory exact prior service spec." >&2; exit 1; } + printf '%s|%s\n' "${service_name}" "$(sha256sum "${spec_file}" | cut -d' ' -f1)" \ + >> "${runtime_dir}/prior-service-spec-hashes.list" + done < "${runtime_dir}/prior-services.list" + grep -Fxq "${stack_name}_postgres" "${runtime_dir}/prior-services.list" || { + echo "Existing canary stack does not contain its expected PostgreSQL service." >&2 + exit 1 + } +fi +[[ "${stack_preexisting}" == 1 ]] || { + echo "Canary Brio provisioning requires a healthy pre-existing PostgreSQL stack so its database state can be journaled before mutation." >&2 + exit 1 +} + +# The durable root-owned transaction journal is staged atomically before the +# first host, Swarm, role, database, ACL, or service mutation. +docker run --rm \ + --mount type=bind,src=/etc,dst=/host/etc,readonly \ + --mount type=bind,src=/var/lib,dst=/host/var/lib,readonly \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${runtime_dir},dst=/runtime,readonly" \ + --mount "type=bind,src=${remote_dir}/scripts/brio-db-transaction.sh,dst=/input/brio-db-transaction.sh,readonly" \ + -e "DEPLOYMENT_ID=${deployment_id}" \ + "${validation_image}" sh -euc ' + case "$DEPLOYMENT_ID" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$DEPLOYMENT_ID" in *-*) ;; *) exit 1 ;; esac + root=/managed-var-lib/postgres-recovery/brio-canary + stage="$root/.${DEPLOYMENT_ID}.staging" + final="$root/$DEPLOYMENT_ID" + for path in /managed-var-lib /managed-var-lib/postgres-recovery "$root"; do [ ! -L "$path" ] || exit 1; done + install -d -o 0 -g 0 -m 0700 /managed-var-lib/postgres-recovery "$root" + [ ! -e "$stage" ] && [ ! -L "$stage" ] && [ ! -e "$final" ] && [ ! -L "$final" ] + install -d -o 0 -g 0 -m 0700 "$stage" "$stage/rollback" "$stage/database" "$stage/swarm" + cat > "$stage/rollback/paths.list" <<"PATHS" +etc/makepad/secrets/postgres-canary-superuser-password +etc/makepad/tls/postgres/ca.crt +etc/makepad/secrets/postgres-brio-app-backup-password +etc/makepad/tls/backups/brio-recipient.crt +var/lib/makepad/postgres-backups/brio-staging +PATHS + : > "$stage/rollback/present.list" + : > "$stage/rollback/absent.list" + while IFS= read -r path; do + current=/host + old_ifs=$IFS; IFS=/; set -- $path; IFS=$old_ifs + for component do current="$current/$component"; [ ! -L "$current" ] || { echo "Snapshot path contains a symlink component." >&2; exit 1; }; done + if [ -e "/host/$path" ]; then printf "%s\n" "$path" >> "$stage/rollback/present.list"; else printf "%s\n" "$path" >> "$stage/rollback/absent.list"; fi + done < "$stage/rollback/paths.list" + tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar" -C /host -T "$stage/rollback/present.list" + backup=/host/var/lib/makepad/postgres-backups/brio-staging + : > "$stage/rollback/backup-entries.list" + if [ -d "$backup" ] && [ ! -L "$backup" ]; then + find "$backup" -mindepth 1 -maxdepth 1 -printf "%f\n" | LC_ALL=C sort > "$stage/rollback/backup-entries.list" + if [ -L "$backup/latest" ]; then readlink "$backup/latest" > "$stage/rollback/prior-latest-target"; fi + if [ -f "$backup/last-success.json" ] && [ ! -L "$backup/last-success.json" ]; then + cp -a "$backup/last-success.json" "$stage/rollback/prior-last-success.json" + fi + fi + cp /runtime/prior-services.list "$stage/swarm/prior-services.list" + cp /runtime/prior-service-spec-hashes.list "$stage/swarm/prior-service-spec-hashes.list" + cp -a /runtime/prior-service-specs "$stage/swarm/specs" + install -o 0 -g 0 -m 0700 /input/brio-db-transaction.sh "$stage/brio-db-transaction.sh" + printf "%s\n" "$DEPLOYMENT_ID" > "$stage/deployment-id" + chown -R 0:0 "$stage" + find "$stage" -type d -exec chmod 0700 {} + + find "$stage" -type f -exec chmod 0600 {} + + sync -f "$stage" 2>/dev/null || sync + ' +journal_stage="${recovery_root}/.${deployment_id}.staging" +run_db_transaction prepare "${journal_stage}" +docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "DEPLOYMENT_ID=${deployment_id}" "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-canary + stage="$root/.${DEPLOYMENT_ID}.staging"; final="$root/$DEPLOYMENT_ID" + [ -s "$stage/database/restore.sql" ] && [ -s "$stage/database/prestate.fingerprint" ] + printf "%s\n" IN_PROGRESS > "$stage/IN_PROGRESS"; chmod 0600 "$stage/IN_PROGRESS" + sync -f "$stage" 2>/dev/null || sync + mv -T "$stage" "$final" + sync -f "$root" 2>/dev/null || sync + ' + +rollback_armed=1 +journal_marker "${journal_dir}" STACK_MUTATION_ARMED +stack_mutated=1 + +for record in "${missing_networks[@]}"; do + IFS='|' read -r name required_internal <<< "${record}" + network_args=(--driver overlay --attachable --opt encrypted=true) + [[ "${required_internal}" != "true" ]] || network_args+=(--internal) + docker network create "${network_args[@]}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" >/dev/null +done +for record in "${missing_configs[@]}"; do + IFS='|' read -r name source digest <<< "${record}" + docker config create --label "content-sha256=${digest}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" "${source}" >/dev/null +done +for record in "${missing_secrets[@]}"; do + IFS='|' read -r name source digest <<< "${record}" + docker secret create --label "content-sha256=${digest}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" "${source}" >/dev/null +done + +# Stage every replacement next to its final destination, then promote with +# same-filesystem renames. The exact snapshot above compensates any partial +# promotion or any later stack/bootstrap/backup failure. +stage_tag=${deployment_id} +docker run --rm \ + --mount "type=bind,src=${runtime_dir},dst=/runtime,readonly" \ + --mount type=bind,src=/etc,dst=/host/etc \ + --mount type=bind,src=/var/lib,dst=/host/var/lib \ + -e "STAGE_TAG=${stage_tag}" \ + "${postgres_image}" sh -euc ' + case "$STAGE_TAG" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "Invalid stage identifier." >&2; exit 1 ;; esac + case "$STAGE_TAG" in *-*) ;; *) echo "Invalid stage identifier." >&2; exit 1 ;; esac + stage_run=${STAGE_TAG%-*} + stage_attempt=${STAGE_TAG#*-} + case "$stage_run:$stage_attempt" in *[!0-9:]*|:*|*:) echo "Invalid stage identifier." >&2; exit 1 ;; esac + for path in \ + /host/etc /host/etc/makepad /host/etc/makepad/secrets \ + /host/etc/makepad/tls /host/etc/makepad/tls/postgres /host/etc/makepad/tls/backups \ + /host/var /host/var/lib /host/var/lib/makepad /host/var/lib/makepad/postgres-backups; do + [ ! -L "$path" ] || { echo "Managed destination contains a symlink: $path" >&2; exit 1; } + done + super_stage="/host/etc/makepad/secrets/.postgres-canary-superuser-password.${STAGE_TAG}.stage" + ca_stage="/host/etc/makepad/tls/postgres/.ca.crt.${STAGE_TAG}.stage" + backup_stage="/host/etc/makepad/secrets/.postgres-brio-app-backup-password.${STAGE_TAG}.stage" + recipient_stage="/host/etc/makepad/tls/backups/.brio-recipient.crt.${STAGE_TAG}.stage" + super_final=/host/etc/makepad/secrets/postgres-canary-superuser-password + ca_final=/host/etc/makepad/tls/postgres/ca.crt + backup_final=/host/etc/makepad/secrets/postgres-brio-app-backup-password + recipient_final=/host/etc/makepad/tls/backups/brio-recipient.crt + cleanup() { rm -f -- "$super_stage" "$ca_stage" "$backup_stage" "$recipient_stage"; } + trap cleanup EXIT HUP INT TERM + for path in "$super_stage" "$ca_stage" "$backup_stage" "$recipient_stage"; do + [ ! -e "$path" ] && [ ! -L "$path" ] || { echo "Refusing an existing stage path." >&2; exit 1; } + done + for path in "$super_final" "$ca_final" "$backup_final" "$recipient_final"; do + [ ! -L "$path" ] && [ ! -d "$path" ] || { echo "Refusing an unsafe final managed path." >&2; exit 1; } + done + install -o 0 -g 0 -m 0600 /runtime/postgres-superuser-password "$super_stage" + install -o 0 -g 0 -m 0444 /runtime/postgres-ca.pem "$ca_stage" + install -o 999 -g 999 -m 0400 /runtime/brio-staging-backup-password "$backup_stage" + install -o 0 -g 0 -m 0444 /runtime/brio-backup-recipient-cert.pem "$recipient_stage" + mv -fT "$super_stage" "$super_final" + mv -fT "$ca_stage" "$ca_final" + mv -fT "$backup_stage" "$backup_final" + mv -fT "$recipient_stage" "$recipient_final" + install -d -o 999 -g 999 -m 0700 /host/var/lib/makepad/postgres-backups/brio-staging + trap - EXIT HUP INT TERM + ' +[[ $(stat -c '%u:%a' "${superuser_host_file}") == "0:600" ]] || { echo "Canary superuser credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_file}") == "999:400" ]] || { echo "Canary backup credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_dir}") == "999:700" && ! -L "${backup_host_dir}" ]] || { echo "Canary backup directory installation failed its ownership/mode check." >&2; exit 1; } +for public_file in "${ca_host_file}" "${recipient_host_file}"; do + public_mode=$(stat -c '%a' "${public_file}") + if [[ $(stat -c '%u' "${public_file}") != "0" ]] || (( (8#${public_mode} & 8#022) != 0 )); then + echo "Canary public certificate installation failed its ownership/mode check." >&2 + exit 1 + fi +done + +case "${failure_injection}" in + after-managed-file-promotion) echo "Injected failure after canary managed-file promotion." >&2; exit 96 ;; + term-after-managed-file-promotion) kill -TERM "$$" ;; + kill-after-managed-file-promotion) kill -KILL "$$" ;; +esac + +"${remote_dir}/scripts/deploy-postgres-stack.sh" "${remote_dir}" "${stack_name}" canary +[[ "${failure_injection}" != "after-stack-deploy" ]] || { echo "Injected failure before canary bootstrap." >&2; exit 95; } + +journal_marker "${journal_dir}" DATABASE_MUTATION_ARMED +db_mutated=1 +docker run --rm --network "${brio_network}" \ + -v "${superuser_host_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${runtime_dir}/brio-staging-app-password:/run/secrets/brio_app_password:ro" \ + -v "${runtime_dir}/brio-staging-backup-password:/run/secrets/brio_backup_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + -v "${remote_dir}/bootstrap/brio-staging-app.sql:/bootstrap/brio-staging-app.sql:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export PGSSLMODE=verify-full PGSSLROOTCERT=/etc/postgresql/ca.crt + export BRIO_APP_PASSWORD="$(cat /run/secrets/brio_app_password)" + export BRIO_BACKUP_PASSWORD="$(cat /run/secrets/brio_backup_password)" + { + printf "%s\n" "\\getenv brio_staging_app_password BRIO_APP_PASSWORD" "\\getenv brio_staging_backup_password BRIO_BACKUP_PASSWORD" + cat /bootstrap/brio-staging-app.sql + } > /tmp/bootstrap.sql + exec psql -X -v ON_ERROR_STOP=1 -h makepad-postgres-brio-staging -U "$1" -d postgres -f /tmp/bootstrap.sql + ' sh "${postgres_user}" >/dev/null + +case "${failure_injection}" in + after-bootstrap) echo "Injected failure after canary database bootstrap." >&2; exit 94 ;; + kill-after-bootstrap) kill -KILL "$$" ;; +esac + +run_role_query() { + local password_file=$1 role=$2 database=$3 sslmode=$4 query=$5 + docker run --rm --network "${brio_network}" \ + -v "${password_file}:/run/secrets/role_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/role_password)" + export PGSSLMODE="$3" PGSSLROOTCERT=/etc/postgresql/ca.crt + exec psql -X -At -h makepad-postgres-brio-staging -U "$1" -d "$2" -c "$4" + ' sh "${role}" "${database}" "${sslmode}" "${query}" +} + +alias_ip=$(docker run --rm --network "${brio_network}" "${postgres_image}" getent hosts makepad-postgres-brio-staging | awk 'NR == 1 {print $1}') +[[ -n "${alias_ip}" ]] || { echo "Brio database alias does not resolve on its isolated network." >&2; exit 1; } +if [[ $(run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app brio_staging verify-full "select current_database() || ':' || current_user") != "brio_staging:brio_staging_app" ]]; then + echo "Brio application role failed the verify-full identity probe." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-app-probe ]] || { echo "Injected failure after canary app probe." >&2; exit 93; } +if run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app brio_staging disable "select 1" >/dev/null 2>&1; then + echo "Plaintext access to brio_staging was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-plaintext-probe ]] || { echo "Injected failure after canary plaintext probe." >&2; exit 92; } +if run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app postgres verify-full "select 1" >/dev/null 2>&1; then + echo "Brio application role was unexpectedly accepted by a non-target database." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-nontarget-probe ]] || { echo "Injected failure after canary non-target probe." >&2; exit 91; } +if [[ $(run_role_query "${runtime_dir}/brio-staging-backup-password" brio_staging_backup brio_staging verify-full "show default_transaction_read_only") != "on" ]]; then + echo "Brio backup role is not read-only." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-backup-role-probe ]] || { echo "Injected failure after canary backup-role probe." >&2; exit 90; } + +previous_latest=$(docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + "${validation_image}" sh -euc 'readlink /backups/latest 2>/dev/null || true') +backup_started_at=$(date +%s) +# A one-shot verifier creates no second Swarm service update, so PreviousSpec +# remains the exact predeployment spec and one rollback can restore it. +docker run --rm --network "${brio_network}" --user 999:999 --read-only \ + --cap-drop ALL --security-opt no-new-privileges:true \ + --tmpfs /tmp:size=64m,mode=0700,uid=999,gid=999 \ + --mount "type=bind,src=${backup_host_dir},dst=/backups" \ + --mount "type=bind,src=${backup_host_file},dst=/run/secrets/postgres_backup_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + --mount "type=bind,src=${recipient_host_file},dst=/etc/postgresql/brio-backup-recipient.crt,readonly" \ + --mount "type=bind,src=${remote_dir}/scripts/run-brio-encrypted-backup.sh,dst=/usr/local/bin/run-brio-encrypted-backup.sh,readonly" \ + -e PGHOST=makepad-postgres-brio-staging -e PGPORT=5432 -e PGUSER=brio_staging_backup \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e POSTGRES_BACKUP_PASSWORD_FILE=/run/secrets/postgres_backup_password \ + -e BRIO_BACKUP_DATABASE=brio_staging -e BRIO_BACKUP_ROOT=/backups \ + -e BRIO_BACKUP_RECIPIENT_CERT=/etc/postgresql/brio-backup-recipient.crt \ + -e BRIO_BACKUP_RETENTION_DAYS=35 \ + "${validation_image}" /usr/local/bin/run-brio-encrypted-backup.sh +backup_verified=0 +for _ in $(seq 1 60); do + if docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + -e "BACKUP_STARTED_AT=${backup_started_at}" -e "PREVIOUS_LATEST=${previous_latest}" \ + "${validation_image}" sh -euc ' + status=/backups/last-success.json + [ -s "$status" ] && [ "$(stat -c %Y "$status")" -ge "$BACKUP_STARTED_AT" ] + grep -q "\"database\":\"brio_staging\"" "$status" + grep -q "\"encrypted\":true" "$status" + latest=$(readlink /backups/latest) + case "$latest" in 20??????T??????Z) ;; *) exit 1 ;; esac + [ "$latest" != "$PREVIOUS_LATEST" ] + directory="/backups/$latest" + [ -s "$directory/brio_staging.dump.cms" ] && [ -s "$directory/SHA256SUMS" ] + (cd "$directory" && sha256sum --check --status SHA256SUMS) + openssl cms -cmsout -inform DER -in "$directory/brio_staging.dump.cms" -noout >/dev/null 2>&1 + '; then + backup_verified=1 + break + fi + sleep 2 +done +[[ "${backup_verified}" == "1" ]] || { echo "A fresh validated encrypted brio_staging backup was not published." >&2; exit 1; } +[[ "${failure_injection}" != after-backup-verification ]] || { echo "Injected failure after canary backup verification." >&2; exit 89; } + +journal_marker "${journal_dir}" COMMITTED +rollback_armed=0 +remove_durable_journal "${journal_dir}" +echo "Brio canary PostgreSQL provisioning, bootstrap, transport policy, alias, and encrypted backup verification passed." diff --git a/scripts/deploy-brio-identity-db-host.sh b/scripts/deploy-brio-identity-db-host.sh new file mode 100755 index 0000000..7b4e032 --- /dev/null +++ b/scripts/deploy-brio-identity-db-host.sh @@ -0,0 +1,806 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 4)); then + echo "Usage: deploy-brio-identity-db-host.sh " >&2 + exit 2 +fi + +bundle_dir=$1 +runtime_dir=$2 +db_hostname=$3 +keycloak_source_cidr=$4 +live_dir=/srv/makepad/postgres +compose_project=postgres +expected_container_name=postgres-postgres-1 +db_env="${bundle_dir}/envs/production/.env.db" +candidate_compose="${bundle_dir}/compose.host.yml" +rollback_armed=0 +validation_image= +prior_postgres_image= +db_mutated=0 +recovery_marker="${runtime_dir}/RECOVERY_REQUIRED" +recovery_root=/var/lib/makepad/postgres-recovery/brio-identity +recovery_id=${runtime_dir##*/postgres-brio-identity-runtime-} +recovery_evidence="${recovery_root}/${recovery_id}" +journal_dir=${recovery_evidence} +failure_injection=${BRIO_DEPLOY_FAILURE_INJECTION:-} + +if [[ -n "${failure_injection}" ]]; then + [[ "${BRIO_DEPLOY_TEST_MODE:-}" == "isolated-container" && -f /.dockerenv ]] || { + echo "Failure injection is permitted only in the isolated deployment test container." >&2 + exit 2 + } + case "${failure_injection}" in + after-managed-file-promotion|term-after-managed-file-promotion|kill-after-managed-file-promotion|after-bootstrap|kill-after-bootstrap|after-app-probe|after-plaintext-probe|after-nontarget-probe|after-backup-role-probe|after-backup-verification|rollback-restore|rollback-recreate) ;; + *) echo "Unsupported identity deployment failure injection." >&2; exit 2 ;; + esac +fi + +[[ "${BRIO_IDENTITY_DB_DEPLOY_CONFIRM:-}" == "restart-standalone-postgres-for-brio-staging" ]] || { + echo "Set BRIO_IDENTITY_DB_DEPLOY_CONFIRM to the exact standalone DB restart acknowledgement." >&2 + exit 2 +} +[[ "${BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED:-}" == "yes" ]] || { + echo "A successful current backup/restore gate must be confirmed before changing the standalone DB VM." >&2 + exit 2 +} +[[ "${bundle_dir}" =~ ^/tmp/postgres-brio-identity-bundle-[0-9]+-[0-9]+$ ]] || { + echo "job-bundle-dir must be /tmp/postgres-brio-identity-bundle--." >&2 + exit 2 +} +[[ "${runtime_dir}" =~ ^/tmp/postgres-brio-identity-runtime-[0-9]+-[0-9]+$ ]] || { + echo "runtime-secret-dir must be /tmp/postgres-brio-identity-runtime--." >&2 + exit 2 +} +[[ "${db_hostname}" == "65.21.134.125" ]] || { + echo "The Brio identity certificate endpoint must be the reviewed standalone DB IP 65.21.134.125." >&2 + exit 2 +} +[[ "${keycloak_source_cidr}" == "88.99.209.165/32" ]] || { + echo "The Keycloak source must be the reviewed exact egress 88.99.209.165/32." >&2 + exit 2 +} + +read_setting() { + local name=$1 file=$2 value + value=$(grep -E "^${name}=" "${file}" | tail -n 1 | cut -d= -f2-) + [[ -n "${value}" ]] || { echo "${name} is missing or empty in ${file}." >&2; exit 1; } + printf '%s' "${value}" +} + +assert_no_symlink_components() { + local path=$1 current='' component + local -a components=() + [[ "${path}" == /* && "${path}" != *"//"* && "${path}" != *"/./"* \ + && "${path}" != */. && "${path}" != *"/../"* && "${path}" != */.. ]] || { + echo "Managed path is not a normalized absolute path: ${path}" >&2 + return 1 + } + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + [[ -n "${component}" && "${component}" != . && "${component}" != .. ]] || return 1 + current="${current}/${component}" + [[ ! -L "${current}" ]] || { + echo "Managed path contains a symlink component: ${current}" >&2 + return 1 + } + done +} + +validate_postgres_target() { + local expected_image=$1 container_id container_name project_label service_label oneoff_label + local network_mode actual_image data_mount running health + container_id=$(docker container inspect "${expected_container_name}" --format '{{.Id}}' 2>/dev/null) || { + echo "Expected standalone container ${expected_container_name} was not found." >&2 + return 1 + } + container_name=$(docker container inspect "${container_id}" --format '{{.Name}}') + project_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.project"}}') + service_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.service"}}') + oneoff_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.oneoff"}}') + network_mode=$(docker container inspect "${container_id}" --format '{{.HostConfig.NetworkMode}}') + actual_image=$(docker container inspect "${container_id}" --format '{{.Config.Image}}') + data_mount=$(docker container inspect "${container_id}" --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{printf "%s|%s|%t\n" .Type .Source .RW}}{{end}}{{end}}') + running=$(docker container inspect "${container_id}" --format '{{.State.Running}}') + health=$(docker container inspect "${container_id}" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}') + [[ "${container_name}" == "/${expected_container_name}" \ + && "${project_label}" == "${compose_project}" \ + && "${service_label}" == "postgres" \ + && "${oneoff_label}" == "False" \ + && "${network_mode}" == "host" \ + && "${actual_image}" == "${expected_image}" \ + && "${data_mount}" == 'bind|/var/lib/makepad/postgres|true' \ + && "${running}" == "true" && "${health}" == "healthy" ]] || { + echo "The standalone target failed its exact Compose label, image, host-network, data-bind, or health contract." >&2 + return 1 + } + printf '%s' "${container_id}" +} + +restore_snapshot() { + local source_journal=${1:-${journal_dir}} + docker run --rm \ + --mount "type=bind,src=${live_dir},dst=/managed/live" \ + --mount type=bind,src=/etc/makepad,dst=/managed/etc \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${source_journal}/rollback,dst=/rollback,readonly" \ + "${validation_image}" sh -euc ' + backup=/managed-var-lib/postgres-backups/keycloak-brio-staging + parent=${backup%/*} + [ -d "$parent" ] && [ ! -L "$parent" ] + if [ -e "$backup" ] || [ -L "$backup" ]; then + [ -d "$backup" ] && [ ! -L "$backup" ] || { echo "Identity backup rollback target is unsafe." >&2; exit 1; } + find "$backup" -depth -delete + fi + if [ -f /rollback/identity-backups.tar ] && [ ! -L /rollback/identity-backups.tar ]; then + tar -tf /rollback/identity-backups.tar | while IFS= read -r member; do + case "$member" in postgres-backups/keycloak-brio-staging|postgres-backups/keycloak-brio-staging/*) ;; *) exit 1 ;; esac + done + tar --numeric-owner -xpf /rollback/identity-backups.tar -C /managed-var-lib + elif [ ! -f /rollback/identity-backup-absent ]; then + echo "Identity backup snapshot contract is incomplete." >&2 + exit 1 + fi + while IFS= read -r path; do + case "$path" in + live/compose.host.yml|live/envs/production/.env.db|live/config/runtrace-pg_hba.conf|live/bootstrap/keycloak-brio-staging.sql|live/scripts/run-runtrace-backup.sh|live/scripts/run-runtrace-backup-loop.sh|live/scripts/run-brio-encrypted-backup.sh|live/scripts/run-brio-encrypted-backup-loop.sh|etc/secrets/postgres-brio-identity-backup-password|etc/tls/backups/brio-recipient.crt) ;; + *) echo "Refusing an unexpected rollback path." >&2; exit 1 ;; + esac + rm -f -- "/managed/$path" + done < /rollback/absent.list + tar --numeric-owner -xpf /rollback/managed.tar -C /managed + ' +} + +run_db_transaction() { + local operation=$1 source_journal=${2:-${journal_dir}} helper + if [[ "${operation}" == prepare ]]; then + helper="${bundle_dir}/scripts/brio-db-transaction.sh" + else + helper="${source_journal}/brio-db-transaction.sh" + fi + [[ -f "${helper}" && ! -L "${helper}" ]] || { + echo "The durable database compensation helper is unavailable." >&2 + return 1 + } + docker run --rm --network host \ + --mount "type=bind,src=${helper},dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + --mount "type=bind,src=${source_journal},dst=/journal" \ + --mount "type=bind,src=${superuser_host_file},dst=/run/secrets/postgres_superuser_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + -e "PGUSER=${postgres_user}" -e "PGHOST=${db_hostname}" -e PGHOSTADDR=127.0.0.1 \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e PGPASSWORD_FILE=/run/secrets/postgres_superuser_password \ + "${postgres_image}" /usr/local/bin/brio-db-transaction.sh "${operation}" keycloak /journal/database +} + +journal_marker() { + local source_journal=$1 marker=$2 + case "${marker}" in IN_PROGRESS|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; + *) echo "Refusing an unsupported journal marker." >&2; return 1 ;; + esac + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 1 ;; esac + [ -d /journal ] && [ ! -L /journal ] + tmp="/journal/.${MARKER}.tmp" + [ ! -e "$tmp" ] && [ ! -L "$tmp" ] + printf "%s\n" "$MARKER" > "$tmp" + chmod 0600 "$tmp" + mv -fT "$tmp" "/journal/$MARKER" + sync -f /journal 2>/dev/null || sync + ' +} + +remove_durable_journal() { + local source_journal=$1 + docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "JOURNAL=${source_journal}" "${validation_image}" sh -euc ' + case "$JOURNAL" in /var/lib/makepad/postgres-recovery/brio-identity/*) identifier=${JOURNAL##*/} ;; *) exit 1 ;; esac + case "$identifier" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$identifier" in *-*) ;; *) exit 1 ;; esac + relative=${JOURNAL#/var/lib/makepad} + target="/managed-var-lib$relative" + [ -d "$target" ] && [ ! -L "$target" ] + find "$target" -depth -delete + sync -f /managed-var-lib/postgres-recovery/brio-identity 2>/dev/null || sync + ' +} + +recover_incomplete_journals() { + local current_id="${recovery_id}" current_journal="${journal_dir}" pending saved_injection="${failure_injection}" marker_list + marker_list=$(docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib,readonly \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-identity + [ ! -e "$root" ] && exit 0 + [ -d "$root" ] && [ ! -L "$root" ] + for candidate in "$root"/* "$root"/.*; do + [ -e "$candidate" ] || continue + name=${candidate##*/} + case "$name" in .|..) continue ;; .*.staging) echo "INCOMPLETE-STAGE:$name"; continue ;; esac + case "$name" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "UNSAFE:$name"; continue ;; esac + case "$name" in *-*) ;; *) echo "UNSAFE:$name"; continue ;; esac + [ -d "$candidate" ] && [ ! -L "$candidate" ] || { echo "UNSAFE:$name"; continue; } + if [ -f "$candidate/COMMITTED" ] && [ ! -L "$candidate/COMMITTED" ]; then + echo "COMMITTED:$name" + elif [ -f "$candidate/ROLLED_BACK" ] && [ ! -L "$candidate/ROLLED_BACK" ]; then + echo "ROLLED_BACK:$name" + elif [ -f "$candidate/IN_PROGRESS" ] && [ ! -L "$candidate/IN_PROGRESS" ]; then + echo "PENDING:$name" + else + echo "UNSAFE:$name" + fi + done + ') + while IFS=: read -r state identifier; do + [[ -n "${state}" ]] || continue + case "${state}" in + COMMITTED|ROLLED_BACK) + remove_durable_journal "${recovery_root}/${identifier}" + ;; + PENDING) + pending="${recovery_root}/${identifier}" + recovery_id=${identifier} + journal_dir=${pending} + recovery_evidence=${pending} + prior_postgres_image=$(docker run --rm --mount "type=bind,src=${pending},dst=/journal,readonly" \ + "${validation_image}" sh -euc 'cat /journal/prior-postgres-image') + [[ "${prior_postgres_image}" == *@sha256:* ]] || { echo "Recovery journal has an invalid prior image." >&2; return 1; } + db_mutated=$(docker run --rm --mount "type=bind,src=${pending},dst=/journal,readonly" \ + "${validation_image}" sh -euc 'if [ -f /journal/DATABASE_MUTATION_ARMED ] && [ ! -L /journal/DATABASE_MUTATION_ARMED ]; then echo 1; else echo 0; fi') + failure_injection= + rollback_armed=1 + if ! rollback_deployment; then + # Do not let the outer EXIT trap attempt the same failed recovery a + # second time. Its durable journal and marker are now authoritative. + rollback_armed=0 + preserve_recovery_evidence || true + echo "An interrupted Brio identity database transaction could not be recovered." >&2 + return 1 + fi + rollback_armed=0 + ;; + INCOMPLETE-STAGE|UNSAFE) + echo "Unsafe or incomplete durable Brio identity journal detected: ${identifier}" >&2 + return 1 + ;; + *) echo "Unexpected durable journal state." >&2; return 1 ;; + esac + done <<< "${marker_list}" + recovery_id=${current_id} + journal_dir=${current_journal} + recovery_evidence=${current_journal} + failure_injection=${saved_injection} + db_mutated=0 +} + +rollback_deployment() { + local rollback_status=0 + echo "Deployment failed after the mutation boundary; restoring the exact managed-file snapshot." >&2 + if [[ "${failure_injection}" == "rollback-restore" ]]; then + rollback_status=1 + else + restore_snapshot "${journal_dir}" || rollback_status=1 + fi + export MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST="${db_hostname}" + export MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH="${live_dir}/config/runtrace-pg_hba.conf" + export MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup.sh" + export MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup-loop.sh" + export MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup.sh" + export MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup-loop.sh" + if [[ "${failure_injection}" == "rollback-recreate" ]]; then + rollback_status=1 + else + docker compose --project-name postgres \ + --env-file "${live_dir}/envs/production/.env.db" \ + -f "${live_dir}/compose.host.yml" \ + up -d --remove-orphans --wait --force-recreate || rollback_status=1 + fi + validate_postgres_target "${prior_postgres_image}" >/dev/null || rollback_status=1 + if [[ "${db_mutated}" == "1" || -f "${journal_dir}/DATABASE_MUTATION_ARMED" ]]; then + run_db_transaction restore "${journal_dir}" || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + journal_marker "${journal_dir}" ROLLED_BACK || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + remove_durable_journal "${journal_dir}" || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + echo "Managed files and the prior healthy postgres Compose target were restored." >&2 + else + echo "Automatic rollback did not restore a healthy exact target; operator intervention is required." >&2 + fi + return "${rollback_status}" +} + +preserve_recovery_evidence() { + # The rollback archive can contain the former managed backup credential. Keep + # it outside /tmp, root-owned and unreadable to the deploy account. The + # runtime marker contains only the fixed evidence path and run identifier. + { + printf 'recovery_evidence=%s\n' "${recovery_evidence}" + printf 'deployment_id=%s\n' "${recovery_id}" + } > "${recovery_marker}" + chmod 0600 "${recovery_marker}" + journal_marker "${journal_dir}" RECOVERY_REQUIRED + echo "Root-protected recovery evidence retained at ${recovery_evidence}." >&2 +} + +cleanup_runtime() { + local cleanup_status=0 + # Incoming application/backup passwords and recipient material are always + # removed, including when rollback itself failed. + for name in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + [[ ! -f "${runtime_dir}/${name}" || -L "${runtime_dir}/${name}" ]] || rm -f -- "${runtime_dir:?}/${name}" || cleanup_status=1 + done + if [[ -f "${recovery_marker}" && ! -L "${recovery_marker}" ]]; then + # The protected snapshot and marker are deliberately exempt from normal and + # TTL cleanup until an operator has completed recovery. + rm -f -- "${runtime_dir}/runtrace-pg_hba.conf" "${runtime_dir}/candidate-compose.yml" || cleanup_status=1 + return "${cleanup_status}" + fi + if [[ -e "${recovery_marker}" || -L "${recovery_marker}" ]]; then + echo "Unsafe recovery marker type; preserving the runtime directory for operator inspection." >&2 + return 1 + fi + if [[ -n "${validation_image}" ]] && docker image inspect "${validation_image}" >/dev/null 2>&1; then + docker run --rm --mount "type=bind,src=${runtime_dir},dst=/runtime" "${validation_image}" \ + sh -euc 'for path in /runtime/runtrace-pg_hba.conf /runtime/candidate-compose.yml; do [ ! -e "$path" ] || find "$path" -depth -delete; done' \ + >/dev/null 2>&1 || cleanup_status=1 + fi + rmdir -- "${runtime_dir}" 2>/dev/null || true + return "${cleanup_status}" +} + +handle_exit() { + local status=$? + trap - EXIT HUP INT TERM + if [[ "${rollback_armed}" == "1" ]]; then + if ! rollback_deployment; then + preserve_recovery_evidence || { + echo "CRITICAL: rollback and durable recovery-evidence preservation both failed." >&2 + status=1 + } + status=1 + fi + fi + cleanup_runtime || status=1 + exit "${status}" +} +trap handle_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +swarm_state=$(docker info --format '{{.Swarm.LocalNodeState}}') +[[ "${swarm_state}" == "inactive" ]] || { + echo "Refusing to run the standalone DB-VM deployment on a Docker Swarm node (state: ${swarm_state})." >&2 + exit 1 +} + +for candidate in \ + "${candidate_compose}" \ + "${db_env}" \ + "${bundle_dir}/config/runtrace-pg_hba.conf" \ + "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" \ + "${bundle_dir}/scripts/run-runtrace-backup.sh" \ + "${bundle_dir}/scripts/run-runtrace-backup-loop.sh" \ + "${bundle_dir}/scripts/run-brio-encrypted-backup.sh" \ + "${bundle_dir}/scripts/run-brio-encrypted-backup-loop.sh" \ + "${bundle_dir}/scripts/brio-db-transaction.sh"; do + [[ -f "${candidate}" && ! -L "${candidate}" ]] || { echo "Candidate bundle file is missing or a symlink: ${candidate}" >&2; exit 1; } +done + +postgres_image=$(read_setting POSTGRES_IMAGE "${db_env}") +validation_image=$(read_setting BRIO_BACKUP_IMAGE "${db_env}") +postgres_user=$(read_setting POSTGRES_USER "${db_env}") +data_host_dir=$(read_setting MAKEPAD_POSTGRES_DATA_PATH "${db_env}") +superuser_host_file=$(read_setting MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH "${db_env}") +server_cert_host_file=$(read_setting MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH "${db_env}") +server_key_host_file=$(read_setting MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH "${db_env}") +ca_host_file=$(read_setting MAKEPAD_POSTGRES_CA_CERT_HOST_PATH "${db_env}") +hba_host_file=$(read_setting MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH "${db_env}") +backup_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH "${db_env}") +recipient_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH "${db_env}") +backup_host_dir=$(read_setting MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH "${db_env}") + +[[ "${postgres_image}" == *@sha256:* && "${validation_image}" == *@sha256:* ]] || { echo "Candidate images must be digest pinned." >&2; exit 1; } +[[ "${data_host_dir}" == "/var/lib/makepad/postgres" \ + && "${superuser_host_file}" == "/etc/makepad/secrets/postgres-superuser-password" \ + && "${server_cert_host_file}" == "/etc/makepad/tls/postgres/server.crt" \ + && "${server_key_host_file}" == "/etc/makepad/secrets/postgres-server.key" \ + && "${ca_host_file}" == "/etc/makepad/tls/postgres/ca.crt" \ + && "${hba_host_file}" == "/srv/makepad/postgres/config/runtrace-pg_hba.conf" \ + && "${backup_host_file}" == "/etc/makepad/secrets/postgres-brio-identity-backup-password" \ + && "${recipient_host_file}" == "/etc/makepad/tls/backups/brio-recipient.crt" \ + && "${backup_host_dir}" == "/var/lib/makepad/postgres-backups/keycloak-brio-staging" ]] || { + echo "Candidate environment does not match the exact standalone DB-VM path contract." >&2 + exit 1 +} +expected_data_bind="\"\${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data\"" +if ! grep -Fq 'network_mode: host' "${candidate_compose}" \ + || ! grep -Fq "${expected_data_bind}" "${candidate_compose}"; then + echo "Candidate Compose does not declare the reviewed host network and exact PostgreSQL data bind." >&2 + exit 1 +fi + +runtime_mode=$(stat -c '%a' "${runtime_dir}") +[[ ! -L "${runtime_dir}" && "${runtime_mode}" == "700" ]] || { echo "Runtime directory must be a non-symlink with mode 0700." >&2; exit 1; } +for input in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + input_path="${runtime_dir}/${input}" + [[ -s "${input_path}" && ! -L "${input_path}" && $(stat -c '%a' "${input_path}") == "600" ]] || { + echo "Required runtime input must be non-empty, non-symlinked, and mode 0600: ${input}" >&2 + exit 1 + } +done +for password_file in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password; do + if [[ $(awk 'END { print NR }' "${runtime_dir}/${password_file}") -ne 1 ]] || grep -q $'\r' "${runtime_dir}/${password_file}"; then + echo "Password input must contain one line and no carriage return: ${password_file}" >&2 + exit 1 + fi +done +cmp -s "${runtime_dir}/keycloak-brio-staging-app-password" "${runtime_dir}/keycloak-brio-staging-backup-password" \ + && { echo "Keycloak Brio application and backup credentials must be distinct." >&2; exit 1; } + +for live_input in \ + "${live_dir}/compose.host.yml" \ + "${live_dir}/envs/production/.env.db" \ + "${live_dir}/config/runtrace-pg_hba.conf" \ + "${live_dir}/scripts/run-runtrace-backup.sh" \ + "${live_dir}/scripts/run-runtrace-backup-loop.sh" \ + "${superuser_host_file}" "${server_cert_host_file}" "${server_key_host_file}" "${ca_host_file}"; do + [[ -s "${live_input}" && ! -L "${live_input}" ]] || { echo "Required existing DB-VM input is unavailable or unsafe: ${live_input}" >&2; exit 1; } +done +for managed_path in \ + "${live_dir}" "${live_dir}/compose.host.yml" "${live_dir}/envs/production" "${live_dir}/envs/production/.env.db" \ + "${live_dir}/config" "${hba_host_file}" "${live_dir}/bootstrap" "${live_dir}/scripts" \ + "${superuser_host_file}" "${server_cert_host_file}" "${server_key_host_file}" "${ca_host_file}" \ + "${backup_host_file}" "${recipient_host_file}" "${backup_host_dir}" \ + /var/lib/makepad/postgres-backups /var/lib/makepad/postgres-recovery; do + assert_no_symlink_components "${managed_path}" +done +for exact_parent in \ + /srv /srv/makepad "${live_dir}" "${live_dir}/envs" "${live_dir}/envs/production" \ + "${live_dir}/config" "${live_dir}/bootstrap" "${live_dir}/scripts" \ + /etc /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres \ + /var /var/lib /var/lib/makepad /var/lib/makepad/postgres-backups; do + [[ -d "${exact_parent}" && ! -L "${exact_parent}" ]] || { + echo "Required managed parent is missing, symlinked, or not a directory: ${exact_parent}" >&2 + exit 1 + } +done +[[ $(stat -c '%u:%a' "${superuser_host_file}") == "0:600" ]] || { echo "Existing PostgreSQL superuser credential must be root-owned with mode 0600." >&2; exit 1; } +for public_file in "${server_cert_host_file}" "${ca_host_file}"; do + public_mode=$(stat -c '%a' "${public_file}") + if [[ $(stat -c '%u' "${public_file}") != "0" ]] || (( (8#${public_mode} & 8#022) != 0 )); then + echo "Existing PostgreSQL public TLS files must be root-owned and not group/world writable." >&2 + exit 1 + fi +done +key_uid=$(stat -c '%u' "${server_key_host_file}") +key_gid=$(stat -c '%g' "${server_key_host_file}") +key_mode=$(stat -c '%a' "${server_key_host_file}") +[[ "${key_uid}:${key_gid}:${key_mode}" == "70:70:400" ]] || { + echo "Existing PostgreSQL server key must retain the verified 70:70 mode-0400 contract." >&2 + exit 1 +} +for optional_managed in "${backup_host_file}" "${recipient_host_file}"; do + [[ ! -L "${optional_managed}" ]] || { echo "Refusing a symlink at a managed identity backup path." >&2; exit 1; } +done +if [[ -L "${backup_host_dir}" || ( -e "${backup_host_dir}" && ! -d "${backup_host_dir}" ) ]]; then + echo "Refusing a symlink or non-directory Brio identity backup path." >&2 + exit 1 +fi + +command -v openssl >/dev/null 2>&1 || { echo "openssl is required." >&2; exit 1; } +openssl x509 -in "${server_cert_host_file}" -noout -checkend 604800 >/dev/null +openssl x509 -in "${server_cert_host_file}" -noout -checkip "${db_hostname}" >/dev/null +openssl verify -purpose sslserver -CAfile "${ca_host_file}" "${server_cert_host_file}" >/dev/null +cert_key_hash=$(openssl x509 -in "${server_cert_host_file}" -pubkey -noout | openssl pkey -pubin -outform DER | sha256sum | cut -d' ' -f1) +private_key_hash=$(docker run --rm --mount "type=bind,src=${server_key_host_file},dst=/runtime/server.key,readonly" \ + "${validation_image}" openssl pkey -in /runtime/server.key -pubout -outform DER | sha256sum | cut -d' ' -f1) +[[ "${cert_key_hash}" == "${private_key_hash}" ]] || { echo "Existing DB-VM server certificate and key do not match." >&2; exit 1; } +if grep -q -- 'PRIVATE KEY' "${runtime_dir}/brio-backup-recipient-cert.pem" \ + || ! openssl x509 -in "${runtime_dir}/brio-backup-recipient-cert.pem" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${runtime_dir}/brio-backup-recipient-cert.pem" -out /dev/null; then + echo "Brio backup recipient is not a valid public encryption certificate." >&2 + exit 1 +fi + +rendered_hba="${runtime_dir}/runtrace-pg_hba.conf" +awk -v cidr="${keycloak_source_cidr}" ' + $1 == "hostssl" && $2 == "keycloak_brio_staging" && $3 == "keycloak_brio_staging_app" && $4 == "all" && $5 == "scram-sha-256" { + print "hostssl keycloak_brio_staging keycloak_brio_staging_app 127.0.0.1/32 scram-sha-256" + print "hostssl keycloak_brio_staging keycloak_brio_staging_app " cidr " scram-sha-256" + next + } + { print } +' "${bundle_dir}/config/runtrace-pg_hba.conf" > "${rendered_hba}" +chmod 0600 "${rendered_hba}" +app_allow_line=$(grep -n -E "^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+${keycloak_source_cidr//./\.}[[:space:]]+scram-sha-256$" "${rendered_hba}" | cut -d: -f1) +backup_allow_line=$(grep -n -E '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_backup[[:space:]]+127\.0\.0\.1/32[[:space:]]+scram-sha-256$' "${rendered_hba}" | cut -d: -f1) +app_reject_line=$(grep -n -E '^host[[:space:]]+all[[:space:]]+keycloak_brio_staging_app[[:space:]]+all[[:space:]]+reject$' "${rendered_hba}" | cut -d: -f1) +backup_reject_line=$(grep -n -E '^host[[:space:]]+all[[:space:]]+keycloak_brio_staging_backup[[:space:]]+all[[:space:]]+reject$' "${rendered_hba}" | cut -d: -f1) +if [[ -z "${app_allow_line}" || -z "${backup_allow_line}" || -z "${app_reject_line}" || -z "${backup_reject_line}" \ + || "${app_allow_line}" -ge "${app_reject_line}" || "${backup_allow_line}" -ge "${backup_reject_line}" \ + || $(grep -Ec '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+127\.0\.0\.1/32[[:space:]]+scram-sha-256$' "${rendered_hba}") -ne 1 \ + || $(grep -Ec "^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+${keycloak_source_cidr//./\.}[[:space:]]+scram-sha-256$" "${rendered_hba}") -ne 1 ]] \ + || grep -Eq '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+all[[:space:]]' "${rendered_hba}"; then + echo "Failed to render ordered, exact source-restricted Keycloak Brio HBA rules." >&2 + exit 1 +fi + +export MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST="${db_hostname}" +export MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH="${hba_host_file}" +export MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup.sh" +export MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup-loop.sh" +export MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup.sh" +export MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup-loop.sh" +candidate_compose_command=(docker compose --project-name postgres --env-file "${db_env}" -f "${candidate_compose}") +"${candidate_compose_command[@]}" config --quiet +"${candidate_compose_command[@]}" config > "${runtime_dir}/candidate-compose.yml" +if ! grep -Fq 'network_mode: host' "${runtime_dir}/candidate-compose.yml" \ + || ! grep -Fq 'source: /var/lib/makepad/postgres' "${runtime_dir}/candidate-compose.yml" \ + || ! grep -Fq 'target: /var/lib/postgresql/data' "${runtime_dir}/candidate-compose.yml"; then + echo "Rendered candidate Compose failed the host-network or exact data-bind prevalidation." >&2 + exit 1 +fi + +prior_container_id=$(validate_postgres_target "${postgres_image}") +prior_postgres_image=$(docker container inspect "${prior_container_id}" --format '{{.Config.Image}}') +docker pull "${postgres_image}" >/dev/null +docker pull "${validation_image}" >/dev/null + +# A SIGKILL cannot run shell traps. Recover any root-owned transaction journal +# left by an interrupted prior deployment before creating a new mutation. +recover_incomplete_journals + +# Build the complete root-owned journal at a same-filesystem staging path. It +# includes exact managed files and a compensating database-state program before +# the first host, Compose, role, database, ACL, or backup-service mutation. +docker run --rm \ + --mount "type=bind,src=${live_dir},dst=/managed/live,readonly" \ + --mount type=bind,src=/etc/makepad,dst=/managed/etc,readonly \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${bundle_dir}/scripts/brio-db-transaction.sh,dst=/input/brio-db-transaction.sh,readonly" \ + -e "RECOVERY_ID=${recovery_id}" -e "PRIOR_POSTGRES_IMAGE=${prior_postgres_image}" \ + "${validation_image}" sh -euc ' + case "$RECOVERY_ID" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$RECOVERY_ID" in *-*) ;; *) exit 1 ;; esac + root=/managed-var-lib/postgres-recovery/brio-identity + stage="$root/.${RECOVERY_ID}.staging" + final="$root/$RECOVERY_ID" + for path in /managed-var-lib /managed-var-lib/postgres-recovery "$root"; do + [ ! -L "$path" ] || { echo "Recovery path contains a symlink component." >&2; exit 1; } + done + install -d -o 0 -g 0 -m 0700 /managed-var-lib/postgres-recovery "$root" + [ ! -e "$stage" ] && [ ! -L "$stage" ] && [ ! -e "$final" ] && [ ! -L "$final" ] + install -d -o 0 -g 0 -m 0700 "$stage" "$stage/rollback" "$stage/database" + cat > "$stage/rollback/paths.list" <<"PATHS" +live/compose.host.yml +live/envs/production/.env.db +live/config/runtrace-pg_hba.conf +live/bootstrap/keycloak-brio-staging.sql +live/scripts/run-runtrace-backup.sh +live/scripts/run-runtrace-backup-loop.sh +live/scripts/run-brio-encrypted-backup.sh +live/scripts/run-brio-encrypted-backup-loop.sh +etc/secrets/postgres-brio-identity-backup-password +etc/tls/backups/brio-recipient.crt +PATHS + : > "$stage/rollback/present.list" + : > "$stage/rollback/absent.list" + while IFS= read -r path; do + current=/managed + old_ifs=$IFS + IFS=/ + set -- $path + IFS=$old_ifs + for component do + current="$current/$component" + [ ! -L "$current" ] || { echo "Managed snapshot path contains a symlink component: $path" >&2; exit 1; } + done + if [ -e "/managed/$path" ]; then printf "%s\n" "$path" >> "$stage/rollback/present.list"; else printf "%s\n" "$path" >> "$stage/rollback/absent.list"; fi + done < "$stage/rollback/paths.list" + tar --numeric-owner -cpf "$stage/rollback/managed.tar" -C /managed -T "$stage/rollback/present.list" + backup=/managed-var-lib/postgres-backups/keycloak-brio-staging + if [ -e "$backup" ] || [ -L "$backup" ]; then + [ -d "$backup" ] && [ ! -L "$backup" ] || { echo "Identity backup snapshot path is unsafe." >&2; exit 1; } + tar --numeric-owner -cpf "$stage/rollback/identity-backups.tar" -C /managed-var-lib postgres-backups/keycloak-brio-staging + else + printf "%s\n" absent > "$stage/rollback/identity-backup-absent" + fi + install -o 0 -g 0 -m 0700 /input/brio-db-transaction.sh "$stage/brio-db-transaction.sh" + printf "%s\n" "$PRIOR_POSTGRES_IMAGE" > "$stage/prior-postgres-image" + printf "%s\n" "$RECOVERY_ID" > "$stage/deployment-id" + find "$stage" -type d -exec chmod 0700 {} + + find "$stage" -type f -exec chmod 0600 {} + + sync -f "$stage" 2>/dev/null || sync + ' +journal_stage="${recovery_root}/.${recovery_id}.staging" +run_db_transaction prepare "${journal_stage}" +docker run --rm \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "RECOVERY_ID=${recovery_id}" \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-identity + stage="$root/.${RECOVERY_ID}.staging" + final="$root/$RECOVERY_ID" + [ -d "$stage/database" ] && [ -s "$stage/database/restore.sql" ] && [ -s "$stage/database/prestate.fingerprint" ] + printf "%s\n" IN_PROGRESS > "$stage/IN_PROGRESS" + chmod 0600 "$stage/IN_PROGRESS" + sync -f "$stage" 2>/dev/null || sync + mv -T "$stage" "$final" + sync -f "$root" 2>/dev/null || sync + ' +rollback_armed=1 + +install_host_path() { + local source=$1 destination=$2 owner=$3 mode=$4 + case "${destination}" in + "${live_dir}"/*|/etc/makepad/*) ;; + *) echo "Refusing unmanaged host path: ${destination}" >&2; return 1 ;; + esac + docker run --rm \ + --mount "type=bind,src=${source},dst=/runtime/input,readonly" \ + --mount type=bind,src=/srv,dst=/host/srv \ + --mount type=bind,src=/etc,dst=/host/etc \ + -e "DESTINATION=${destination}" -e "FILE_OWNER=${owner}" -e "FILE_MODE=${mode}" -e "STAGE_TAG=${recovery_id}" \ + "${postgres_image}" sh -euc ' + case "$DESTINATION" in /srv/makepad/postgres/*|/etc/makepad/*) ;; *) exit 1 ;; esac + case "$STAGE_TAG" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$STAGE_TAG" in *-*) ;; *) exit 1 ;; esac + host_destination="/host$DESTINATION" + parent=${host_destination%/*} + base=${host_destination##*/} + current=/host + old_ifs=$IFS + IFS=/ + set -- ${DESTINATION#/} + IFS=$old_ifs + last=$# + index=0 + for component do + index=$((index + 1)) + current="$current/$component" + [ ! -L "$current" ] || { echo "Managed promotion path contains a symlink component: $current" >&2; exit 1; } + if [ "$index" -lt "$last" ]; then [ -d "$current" ] || { echo "Managed promotion parent is not a directory: $current" >&2; exit 1; }; fi + done + [ -d "$parent" ] && [ ! -L "$parent" ] + [ ! -L "$host_destination" ] && [ ! -d "$host_destination" ] + stage="$parent/.${base}.${STAGE_TAG}.stage" + [ ! -e "$stage" ] && [ ! -L "$stage" ] + cleanup() { rm -f -- "$stage"; } + trap cleanup EXIT HUP INT TERM + install -o "${FILE_OWNER%:*}" -g "${FILE_OWNER#*:}" -m "${FILE_MODE}" /runtime/input "$stage" + mv -fT "$stage" "$host_destination" + trap - EXIT HUP INT TERM + ' +} + +install_host_path "${candidate_compose}" "${live_dir}/compose.host.yml" 0:0 0644 +install_host_path "${db_env}" "${live_dir}/envs/production/.env.db" 0:0 0644 +install_host_path "${rendered_hba}" "${hba_host_file}" 0:0 0444 +install_host_path "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" "${live_dir}/bootstrap/keycloak-brio-staging.sql" 0:0 0644 +for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + install_host_path "${bundle_dir}/scripts/${script}" "${live_dir}/scripts/${script}" 0:0 0755 +done +install_host_path "${runtime_dir}/keycloak-brio-staging-backup-password" "${backup_host_file}" 999:999 0400 +install_host_path "${runtime_dir}/brio-backup-recipient-cert.pem" "${recipient_host_file}" 0:0 0444 +docker run --rm --mount type=bind,src=/var/lib,dst=/host/var/lib \ + "${postgres_image}" sh -euc ' + for path in /host/var /host/var/lib /host/var/lib/makepad /host/var/lib/makepad/postgres-backups; do + [ -d "$path" ] && [ ! -L "$path" ] || { echo "Identity backup promotion path is unsafe: $path" >&2; exit 1; } + done + install -d -o 999 -g 999 -m 0700 /host/var/lib/makepad/postgres-backups/keycloak-brio-staging + ' +[[ $(stat -c '%u:%a' "${hba_host_file}") == "0:444" && ! -L "${hba_host_file}" ]] || { echo "Active HBA installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_file}") == "999:400" && ! -L "${backup_host_file}" ]] || { echo "Identity backup credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${recipient_host_file}") == "0:444" && ! -L "${recipient_host_file}" ]] || { echo "Backup recipient installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_dir}") == "999:700" && ! -L "${backup_host_dir}" ]] || { echo "Identity backup directory installation failed its ownership/mode check." >&2; exit 1; } + +case "${failure_injection}" in + after-managed-file-promotion) echo "Injected failure after identity managed-file promotion." >&2; exit 97 ;; + term-after-managed-file-promotion) kill -TERM "$$" ;; + kill-after-managed-file-promotion) kill -KILL "$$" ;; +esac + +compose=(docker compose --project-name postgres --env-file "${live_dir}/envs/production/.env.db" -f "${live_dir}/compose.host.yml") +"${compose[@]}" up -d --wait --force-recreate postgres +validate_postgres_target "${postgres_image}" >/dev/null + +journal_marker "${journal_dir}" DATABASE_MUTATION_ARMED +db_mutated=1 +docker run --rm --network host \ + -v "${superuser_host_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${runtime_dir}/keycloak-brio-staging-app-password:/run/secrets/keycloak_app_password:ro" \ + -v "${runtime_dir}/keycloak-brio-staging-backup-password:/run/secrets/keycloak_backup_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + -v "${live_dir}/bootstrap/keycloak-brio-staging.sql:/bootstrap/keycloak-brio-staging.sql:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export KEYCLOAK_APP_PASSWORD="$(cat /run/secrets/keycloak_app_password)" + export KEYCLOAK_BACKUP_PASSWORD="$(cat /run/secrets/keycloak_backup_password)" + export PGHOST="$1" PGHOSTADDR=127.0.0.1 PGSSLMODE=verify-full PGSSLROOTCERT=/etc/postgresql/ca.crt + { + printf "%s\n" "\\getenv keycloak_brio_staging_app_password KEYCLOAK_APP_PASSWORD" "\\getenv keycloak_brio_staging_backup_password KEYCLOAK_BACKUP_PASSWORD" + cat /bootstrap/keycloak-brio-staging.sql + } > /tmp/bootstrap.sql + exec psql -X -v ON_ERROR_STOP=1 -U "$2" -d postgres -f /tmp/bootstrap.sql + ' sh "${db_hostname}" "${postgres_user}" >/dev/null + +case "${failure_injection}" in + after-bootstrap) echo "Injected failure after identity database bootstrap." >&2; exit 96 ;; + kill-after-bootstrap) kill -KILL "$$" ;; +esac + +run_role_query() { + local password_file=$1 role=$2 database=$3 sslmode=$4 query=$5 + docker run --rm --network host \ + -v "${password_file}:/run/secrets/role_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/role_password)" + export PGHOST="$1" PGHOSTADDR=127.0.0.1 PGSSLMODE="$4" PGSSLROOTCERT=/etc/postgresql/ca.crt + exec psql -X -At -U "$2" -d "$3" -c "$5" + ' sh "${db_hostname}" "${role}" "${database}" "${sslmode}" "${query}" +} + +if [[ $(run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app keycloak_brio_staging verify-full "select current_database() || ':' || current_user") != "keycloak_brio_staging:keycloak_brio_staging_app" ]]; then + echo "Keycloak Brio role failed the local verify-full identity probe." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-app-probe ]] || { echo "Injected failure after identity app-role probe." >&2; exit 95; } +if run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app keycloak_brio_staging disable "select 1" >/dev/null 2>&1; then + echo "Plaintext Keycloak Brio database access was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-plaintext-probe ]] || { echo "Injected failure after identity plaintext probe." >&2; exit 94; } +if run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app postgres verify-full "select 1" >/dev/null 2>&1; then + echo "Keycloak Brio role was unexpectedly accepted by a non-target database." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-nontarget-probe ]] || { echo "Injected failure after identity non-target probe." >&2; exit 93; } +if [[ $(run_role_query "${runtime_dir}/keycloak-brio-staging-backup-password" keycloak_brio_staging_backup keycloak_brio_staging verify-full "show default_transaction_read_only") != "on" ]]; then + echo "Keycloak Brio backup role is not read-only." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-backup-role-probe ]] || { echo "Injected failure after identity backup-role probe." >&2; exit 92; } + +previous_latest=$(docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + "${validation_image}" sh -euc 'readlink /backups/latest 2>/dev/null || true') +backup_started_at=$(date +%s) +"${compose[@]}" up -d --no-deps --force-recreate keycloak_brio_staging_backup +backup_verified=0 +for _ in $(seq 1 60); do + if docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + -e "BACKUP_STARTED_AT=${backup_started_at}" -e "PREVIOUS_LATEST=${previous_latest}" \ + "${validation_image}" sh -euc ' + status=/backups/last-success.json + [ -s "$status" ] && [ "$(stat -c %Y "$status")" -ge "$BACKUP_STARTED_AT" ] + grep -q "\"database\":\"keycloak_brio_staging\"" "$status" + grep -q "\"encrypted\":true" "$status" + latest=$(readlink /backups/latest) + case "$latest" in 20??????T??????Z) ;; *) exit 1 ;; esac + [ "$latest" != "$PREVIOUS_LATEST" ] + directory="/backups/$latest" + [ -s "$directory/keycloak_brio_staging.dump.cms" ] && [ -s "$directory/SHA256SUMS" ] + (cd "$directory" && sha256sum --check --status SHA256SUMS) + openssl cms -cmsout -inform DER -in "$directory/keycloak_brio_staging.dump.cms" -noout >/dev/null 2>&1 + '; then + backup_verified=1 + break + fi + sleep 2 +done +[[ "${backup_verified}" == "1" ]] || { echo "A fresh validated encrypted keycloak_brio_staging backup was not published." >&2; exit 1; } +[[ "${failure_injection}" != after-backup-verification ]] || { echo "Injected failure after identity backup verification." >&2; exit 91; } + +journal_marker "${journal_dir}" COMMITTED +rollback_armed=0 +remove_durable_journal "${journal_dir}" +echo "Standalone Brio identity PostgreSQL target, rollback snapshot, HBA, bootstrap, local TLS policy, and encrypted backup verification passed." +echo "Release acceptance still requires the protected Keycloak-host Verify Brio Identity Database Path workflow for this PostgreSQL run ID." diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh index 4cfc7ff..0ba486f 100755 --- a/scripts/deploy-postgres-stack.sh +++ b/scripts/deploy-postgres-stack.sh @@ -1,20 +1,50 @@ #!/usr/bin/env bash set -euo pipefail -if (($# != 3)); then - echo "Usage: deploy-postgres-stack.sh " >&2 +if (($# < 3 || $# > 4)); then + echo "Usage: deploy-postgres-stack.sh [production-vif-runtime-secret-dir]" >&2 exit 2 fi remote_dir=$1 stack_name=$2 deploy_env=$3 +vif_runtime_dir=${4:-} +if [[ ! "${remote_dir}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+/\.deploy/postgres-[0-9]+-[0-9]+$ ]] \ + || [[ "${remote_dir}" == *"/../"* || "${remote_dir}" == *"/.." || "${remote_dir}" == *"/./"* || "${remote_dir}" == *"/." || "${remote_dir}" == *"//"* ]]; then + echo "job-bundle-dir must be a unique /srv or /opt .deploy/postgres-- path." >&2 + exit 2 +fi +case "${stack_name}" in ''|*[!a-zA-Z0-9_-]*) echo "stack-name contains unsupported characters." >&2; exit 2 ;; esac +case "${deploy_env}" in canary|production) ;; *) echo "Deployment environment must be canary or production." >&2; exit 2 ;; esac +if [[ "${deploy_env}" == "production" ]]; then + [[ $# -eq 4 && "${vif_runtime_dir}" =~ ^/tmp/postgres-brio-vif-runtime-[0-9]+-[0-9]+$ ]] || { + echo "Production requires a job-scoped /tmp/postgres-brio-vif-runtime-- directory." >&2 + exit 2 + } +elif [[ $# -ne 3 ]]; then + echo "Canary deployment must not receive a VIF credential directory." >&2 + exit 2 +fi + +server_certificate= +cleanup_deploy_material() { + [[ -z "${server_certificate}" ]] || rm -f -- "${server_certificate}" + if [[ -n "${vif_runtime_dir}" && -d "${vif_runtime_dir}" && ! -L "${vif_runtime_dir}" ]]; then + rm -f -- "${vif_runtime_dir}/vif-db-password" + rmdir -- "${vif_runtime_dir}" 2>/dev/null || true + fi +} +trap cleanup_deploy_material EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + env_deploy="${remote_dir}/envs/${deploy_env}/.env.deploy" db_env="${remote_dir}/envs/${deploy_env}/.env.db" db_network=$(grep '^MAKEPAD_POSTGRES_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) le_petit_coin_db_network=$(grep '^MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) -brio_backup_image=$(grep '^BRIO_BACKUP_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) postgres_root_user=$(grep '^POSTGRES_USER=' "${db_env}" | tail -n 1 | cut -d= -f2-) postgres_root_password_file=$(grep '^MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) postgres_tls_cert_config=$(grep '^MAKEPAD_POSTGRES_TLS_CERT_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) @@ -23,11 +53,12 @@ postgres_runtrace_hba_config=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=' "${ runtrace_backup_path=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) runtrace_backup_password_file=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) postgres_ca_cert_file=$(grep '^MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) -brio_backup_recipient_cert=$(grep '^MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) vif_enabled=0 brio_staging_enabled=0 if [[ "${deploy_env}" == "canary" ]]; then brio_staging_enabled=1 + brio_backup_image=$(grep '^BRIO_BACKUP_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_backup_recipient_cert=$(grep '^MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) brio_staging_db_network=$(grep '^MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) brio_backup_path=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) brio_backup_password_file=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) @@ -37,23 +68,23 @@ if [[ "${deploy_env}" == "production" ]]; then vif_db_network=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) vif_db_name=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NAME=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) vif_db_user=$(grep '^MAKEPAD_POSTGRES_VIF_DB_USER=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_password=$(grep '^MAKEPAD_POSTGRES_VIF_DB_PASSWORD=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - brio_backup_path=$(grep '^MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) - brio_backup_password_file=$(grep '^MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) + vif_db_password_file="${vif_runtime_dir}/vif-db-password" fi : "${db_network:?MAKEPAD_POSTGRES_DB_NETWORK is missing or empty in ${env_deploy}}" : "${le_petit_coin_db_network:?MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK is missing or empty in ${env_deploy}}" : "${postgres_image:?POSTGRES_IMAGE is missing or empty in ${db_env}}" -: "${brio_backup_image:?BRIO_BACKUP_IMAGE is missing or empty in ${db_env}}" : "${postgres_root_user:?POSTGRES_USER is missing or empty in ${db_env}}" : "${postgres_root_password_file:?MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" : "${postgres_tls_cert_config:?MAKEPAD_POSTGRES_TLS_CERT_CONFIG is missing or empty in ${db_env}}" : "${postgres_tls_key_secret:?MAKEPAD_POSTGRES_TLS_KEY_SECRET is missing or empty in ${db_env}}" : "${postgres_runtrace_hba_config:?MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG is missing or empty in ${db_env}}" : "${postgres_ca_cert_file:?MAKEPAD_POSTGRES_CA_CERT_HOST_PATH is missing or empty in ${db_env}}" -: "${brio_backup_recipient_cert:?MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH is missing or empty in ${db_env}}" -: "${brio_backup_path:?Brio backup directory is missing or empty in ${db_env}}" -: "${brio_backup_password_file:?Brio backup password-file path is missing or empty in ${db_env}}" +if [[ "${deploy_env}" == "canary" ]]; then + : "${brio_backup_image:?BRIO_BACKUP_IMAGE is missing or empty in ${db_env}}" + : "${brio_backup_recipient_cert:?MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH is missing or empty in ${db_env}}" + : "${brio_backup_path:?Brio backup directory is missing or empty in ${db_env}}" + : "${brio_backup_password_file:?Brio backup password-file path is missing or empty in ${db_env}}" +fi if [[ "${deploy_env}" == "production" ]]; then : "${runtrace_backup_path:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH is missing or empty in ${db_env}}" : "${runtrace_backup_password_file:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" @@ -83,50 +114,50 @@ if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 exit 1 fi -for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do - if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then - echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 +if [[ "${deploy_env}" == "canary" ]]; then + for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then + echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 + exit 1 + fi + done + if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then + echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 + exit 1 + fi + brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") + brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") + if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then + echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then + echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 + exit 1 + fi + brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") + brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") + if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then + echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then + echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 + exit 1 + fi + brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") + brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") + if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then + echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 + exit 1 + fi + if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then + echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 exit 1 fi -done -if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then - echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 - exit 1 -fi -brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") -brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") -if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then - echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 - exit 1 -fi -if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then - echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 - exit 1 -fi -brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") -brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") -if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then - echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 - exit 1 -fi -if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then - echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 - exit 1 -fi -brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") -brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") -if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then - echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 - exit 1 -fi -if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ - || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then - echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 - exit 1 fi server_certificate=$(mktemp) -cleanup_server_certificate() { rm -f "${server_certificate}"; } -trap cleanup_server_certificate EXIT docker config inspect "${postgres_tls_cert_config}" --format '{{printf "%s" .Spec.Data}}' > "${server_certificate}" if ! openssl x509 -in "${server_certificate}" -noout -checkend 604800 >/dev/null; then echo "PostgreSQL TLS certificate is invalid or expires in less than seven days." >&2 @@ -177,7 +208,20 @@ if [[ "${vif_enabled}" == "1" ]]; then : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" - : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" + if [[ ! -s "${vif_db_password_file}" || -L "${vif_db_password_file}" \ + || $(stat -c '%a' "${vif_db_password_file}") != "600" \ + || $(stat -c '%a' "${vif_runtime_dir}") != "700" || -L "${vif_runtime_dir}" ]]; then + echo "VIF credential material must be a mode-0600 file in the mode-0700 job runtime directory." >&2 + exit 1 + fi + if [[ $(awk 'END { print NR }' "${vif_db_password_file}") -ne 1 ]] || grep -q $'\r' "${vif_db_password_file}"; then + echo "VIF credential must contain one line and no carriage return." >&2 + exit 1 + fi + [[ -f "${remote_dir}/bootstrap/vif-app.sql" && ! -L "${remote_dir}/bootstrap/vif-app.sql" ]] || { + echo "The VIF bootstrap SQL is missing from the job-scoped bundle." >&2 + exit 1 + } fi if [[ "${brio_staging_enabled}" == "1" ]]; then : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" @@ -229,14 +273,17 @@ fi export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" +generated_dir="${remote_dir}/generated" +install -d -m 0700 "${generated_dir}" +stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml" docker compose \ --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ --env-file "${env_deploy}" \ -f "${remote_dir}/compose.yml" \ -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ - config > "${remote_dir}/stack.yml" + config > "${stack_file}" -docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" +docker stack deploy --compose-file "${stack_file}" "${stack_name}" wait_for_service_convergence() { local service_name=$1 @@ -276,8 +323,6 @@ wait_for_service_convergence() { wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}" if [[ "${deploy_env}" == "canary" ]]; then wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}" -else - wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}" fi if [[ "${brio_staging_enabled}" == "1" ]]; then @@ -323,24 +368,11 @@ fi docker run --rm --network "${vif_db_network}" \ -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres \ - -v ON_ERROR_STOP=1 \ - -v vif_db="${vif_db_name}" \ - -v vif_user="${vif_db_user}" \ - -v vif_password="${vif_db_password}" <<'SQL' -SELECT format('CREATE ROLE %I LOGIN', :'vif_user') -WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec -SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec -SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') -WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec -SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') -WHERE EXISTS ( - SELECT 1 - FROM pg_database d - JOIN pg_roles r ON r.oid = d.datdba - WHERE d.datname = :'vif_db' - AND r.rolname <> :'vif_user' -) \gexec -SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec -SQL + -v "${vif_db_password_file}:/run/secrets/vif_db_password:ro" \ + -v "${remote_dir}/bootstrap/vif-app.sql:/bootstrap/vif-app.sql:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export VIF_PASSWORD="$(cat /run/secrets/vif_db_password)" + exec psql -X -v ON_ERROR_STOP=1 -h makepad-postgres-vif -U "$1" -d postgres \ + -v vif_db="$2" -v vif_user="$3" -f /bootstrap/vif-app.sql + ' sh "${postgres_root_user}" "${vif_db_name}" "${vif_db_user}" >/dev/null diff --git a/scripts/ensure-brio-tmp-cleaner.sh b/scripts/ensure-brio-tmp-cleaner.sh new file mode 100755 index 0000000..75dc080 --- /dev/null +++ b/scripts/ensure-brio-tmp-cleaner.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +clean_test_root() { + local test_root=$1 + [[ "${test_root}" =~ ^/tmp/postgres-brio-cleaner-test-[A-Za-z0-9._-]+$ ]] || { + echo "Test cleanup root must use /tmp/postgres-brio-cleaner-test-." >&2 + exit 2 + } + find "${test_root}" -mindepth 1 -maxdepth 1 -type d \ + \( -name 'postgres-brio-*' -o -name 'postgres-keycloak-cohort-*' \) -mmin +180 \ + -exec sh -euc 'for directory do + [ ! -L "${directory}" ] || continue + [ ! -f "${directory}/RECOVERY_REQUIRED" ] || continue + find "${directory}" -depth -delete + done' sh {} + +} + +if (($# == 2)) && [[ "$1" == "test-clean-once" ]]; then + clean_test_root "$2" + exit 0 +fi + +if (($# == 3)) && [[ "$1" == "test-clean-production-ownership" ]]; then + test_root=$2 + test_image=$3 + [[ "${test_root}" =~ ^/tmp/postgres-brio-cleaner-test-[A-Za-z0-9._-]+$ \ + && "${test_image}" == *@sha256:* ]] || { echo "Unsafe production-ownership cleaner test inputs." >&2; exit 2; } + docker run --rm --read-only --cap-drop ALL --cap-add DAC_OVERRIDE --cap-add FOWNER \ + --security-opt no-new-privileges --mount "type=bind,src=${test_root},dst=/host-tmp" \ + "${test_image}" sh -euc ' + find /host-tmp -mindepth 1 -maxdepth 1 -type d \ + \( -name "postgres-brio-*" -o -name "postgres-keycloak-cohort-*" \) -mmin +180 \ + -exec sh -euc '\''for directory do + [ ! -L "$directory" ] || continue + [ ! -f "$directory/RECOVERY_REQUIRED" ] || continue + find "$directory" -depth -delete + done'\'' sh {} + + ' + exit 0 +fi + +if (($# != 1)); then + echo "Usage: ensure-brio-tmp-cleaner.sh " >&2 + exit 2 +fi + +db_env=$1 +[[ -f "${db_env}" && ! -L "${db_env}" ]] || { echo "Database environment file is missing or a symlink." >&2; exit 2; } +postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) +: "${postgres_image:?POSTGRES_IMAGE is missing from ${db_env}}" +[[ "${postgres_image}" == *@sha256:* ]] || { echo "The temporary-material cleaner requires a digest-pinned image." >&2; exit 1; } + +cleaner_name=makepad-postgres-brio-tmp-cleaner +cleaner_contract=brio-tmp-cleaner-v5-exact-command +# shellcheck disable=SC2016 # The nested shell expands directory inside the container. +cleaner_command='while :; do + find /host-tmp -mindepth 1 -maxdepth 1 -type d \ + \( -name "postgres-brio-*" -o -name "postgres-keycloak-cohort-*" \) -mmin +180 \ + -exec sh -euc '\''for directory do + [ ! -L "${directory}" ] || continue + [ ! -f "${directory}/RECOVERY_REQUIRED" ] || continue + find "${directory}" -depth -delete + done'\'' sh {} + + sleep 900 +done' + +verify_running() { + local state attempts=10 delay=1 + if [[ "${BRIO_DEPLOY_TEST_MODE:-}" == isolated-container ]]; then attempts=2; delay=0; fi + for _ in $(seq 1 "${attempts}"); do + state=$(docker container inspect "${cleaner_name}" --format '{{.State.Running}}|{{.State.Restarting}}' 2>/dev/null || true) + [[ "${state}" != "true|false" ]] || return 0 + sleep "${delay}" + done + echo "${cleaner_name} did not remain running after startup." >&2 + return 1 +} + +if docker container inspect "${cleaner_name}" >/dev/null 2>&1; then + image=$(docker container inspect "${cleaner_name}" --format '{{.Config.Image}}') + contract=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Labels "makepad.cleanup.contract"}}') + restart_policy=$(docker container inspect "${cleaner_name}" --format '{{.HostConfig.RestartPolicy.Name}}') + readonly_root=$(docker container inspect "${cleaner_name}" --format '{{.HostConfig.ReadonlyRootfs}}') + mount_contract=$(docker container inspect "${cleaner_name}" --format '{{range .Mounts}}{{if eq .Destination "/host-tmp"}}{{printf "%s|%s|%t" .Type .Source .RW}}{{end}}{{end}}') + cap_drop=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.CapDrop}}') + cap_add=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.CapAdd}}') + security_opt=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.SecurityOpt}}') + observed_command_length=$(docker container inspect "${cleaner_name}" --format '{{len .Config.Cmd}}') + observed_shell=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 0}}') + observed_shell_flags=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 1}}') + observed_command=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 2}}') + if [[ "${image}" != "${postgres_image}" || "${contract}" != "${cleaner_contract}" \ + || "${restart_policy}" != "unless-stopped" || "${readonly_root}" != "true" \ + || "${mount_contract}" != "bind|/tmp|true" || "${cap_drop}" != '["ALL"]' \ + || "${cap_add}" != '["DAC_OVERRIDE","FOWNER"]' \ + || "${security_opt}" != *'no-new-privileges'* || "${observed_command_length}" != 3 \ + || "${observed_shell}" != sh || "${observed_shell_flags}" != -euc \ + || "${observed_command}" != "${cleaner_command}" ]]; then + echo "Existing ${cleaner_name} does not match the fail-closed cleanup contract." >&2 + exit 1 + fi + if [[ $(docker container inspect "${cleaner_name}" --format '{{.State.Running}}') != "true" ]]; then + docker start "${cleaner_name}" >/dev/null + fi + verify_running + exit $? +fi + +docker run -d \ + --name "${cleaner_name}" \ + --restart unless-stopped \ + --read-only \ + --cap-drop ALL \ + --cap-add DAC_OVERRIDE \ + --cap-add FOWNER \ + --security-opt no-new-privileges \ + --label "makepad.cleanup.contract=${cleaner_contract}" \ + --mount type=bind,src=/tmp,dst=/host-tmp \ + "${postgres_image}" \ + sh -euc "${cleaner_command}" >/dev/null +verify_running diff --git a/scripts/fixtures/brio-deployment-failure-fixture.sh b/scripts/fixtures/brio-deployment-failure-fixture.sh new file mode 100755 index 0000000..d00bcf0 --- /dev/null +++ b/scripts/fixtures/brio-deployment-failure-fixture.sh @@ -0,0 +1,886 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/brio-mock-bin +install -d -m 0700 "${mock_bin}" + +cat > "${mock_bin}/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail + +remove_exact() { + local target=$1 + if [[ -L "${target}" || -f "${target}" ]]; then + rm -f -- "${target}" + elif [[ -d "${target}" ]]; then + find "${target}" -mindepth 1 -delete + rmdir -- "${target}" + elif [[ -e "${target}" ]]; then + echo "Refusing unexpected mock path type: ${target}" >&2 + exit 1 + fi +} + +map_mount() { + local specification=$1 source= destination= + IFS=',' read -r -a fields <<< "${specification}" + for field in "${fields[@]}"; do + case "${field}" in src=*|source=*) source=${field#*=} ;; dst=*|destination=*|target=*) destination=${field#*=} ;; esac + done + [[ -n "${source}" && -n "${destination}" ]] || return 0 + mock_mount_sources+=("${source}") + mock_mount_destinations+=("${destination}") + mkdir -p "$(dirname "${destination}")" + remove_exact "${destination}" + case "${destination}" in + /managed/live|/managed/etc) + mkdir -p "${destination}" + cp -a "${source}/." "${destination}/" + ;; + *) ln -s "${source}" "${destination}" ;; + esac +} + +create_backup() { + local database=$1 root timestamp directory recipient + case "${database}" in + brio_staging) + root=/var/lib/makepad/postgres-backups/brio-staging + recipient=/etc/makepad/tls/backups/brio-recipient.crt + ;; + keycloak_brio_staging) + root=/var/lib/makepad/postgres-backups/keycloak-brio-staging + recipient=/etc/makepad/tls/backups/brio-recipient.crt + ;; + *) return 1 ;; + esac + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + directory="${root}/${timestamp}" + mkdir -p "${directory}" + printf 'mock-%s-backup\n' "${database}" | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${recipient}" -out "${directory}/${database}.dump.cms" 2>/dev/null + (cd "${directory}" && sha256sum "${database}.dump.cms" > SHA256SUMS) + ln -sfn "${timestamp}" "${root}/latest" + printf '{"database":"%s","encrypted":true}\n' "${database}" > "${root}/last-success.json" +} + +command_name=${1:-} +shift || true +case "${command_name}" in + info) + printf '%s\n' inactive + ;; + pull) + ;; + image) + [[ "${1:-}" == inspect ]] + ;; + container) + [[ "${1:-}" == inspect ]] || exit 1 + shift + target=${1:-} + shift || true + format= + while (($#)); do + if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi + done + if [[ "${target}" == makepad-postgres-brio-tmp-cleaner ]]; then + cleaner_state=/tmp/mock-cleaner-state + [[ -d "${cleaner_state}" ]] || exit 1 + case "${format}" in + '') ;; + *'.State.Running'*'.State.Restarting'*) printf '%s|false\n' "$(< "${cleaner_state}/running")" ;; + *'.Config.Image'*) cat "${cleaner_state}/image" ;; + *'makepad.cleanup.contract'*) printf '%s\n' brio-tmp-cleaner-v5-exact-command ;; + *'.HostConfig.RestartPolicy.Name'*) printf '%s\n' unless-stopped ;; + *'.HostConfig.ReadonlyRootfs'*) printf '%s\n' true ;; + *'.Mounts'*) printf '%s\n' 'bind|/tmp|true' ;; + *'.HostConfig.CapDrop'*) printf '%s\n' '["ALL"]' ;; + *'.HostConfig.CapAdd'*) printf '%s\n' '["DAC_OVERRIDE","FOWNER"]' ;; + *'.HostConfig.SecurityOpt'*) printf '%s\n' '["no-new-privileges"]' ;; + *'len .Config.Cmd'*) printf '%s\n' 3 ;; + *'index .Config.Cmd 0'*) printf '%s\n' sh ;; + *'index .Config.Cmd 1'*) printf '%s\n' -euc ;; + *'index .Config.Cmd 2'*) cat "${cleaner_state}/command" ;; + *'.State.Running'*) cat "${cleaner_state}/running" ;; + *) echo "Unhandled cleaner container format: ${format}" >&2; exit 1 ;; + esac + exit 0 + fi + [[ "${target}" == postgres-postgres-1 || "${target}" == mock-container-id ]] || exit 1 + case "${format}" in + *'.Id'*) printf '%s\n' mock-container-id ;; + *'.Name'*) printf '%s\n' /postgres-postgres-1 ;; + *'com.docker.compose.project'*) printf '%s\n' "${MOCK_COMPOSE_PROJECT:-postgres}" ;; + *'com.docker.compose.service'*) printf '%s\n' postgres ;; + *'com.docker.compose.oneoff'*) printf '%s\n' False ;; + *'.HostConfig.NetworkMode'*) printf '%s\n' host ;; + *'.Config.Image'*) printf '%s\n' "${MOCK_POSTGRES_IMAGE}" ;; + *'.Mounts'*) printf '%s\n' 'bind|/var/lib/makepad/postgres|true' ;; + *'.State.Running'*) printf '%s\n' true ;; + *'.State.Health'*) printf '%s\n' healthy ;; + *) echo "Unhandled container format: ${format}" >&2; exit 1 ;; + esac + ;; + start) + [[ "${1:-}" == makepad-postgres-brio-tmp-cleaner && -d /tmp/mock-cleaner-state ]] || exit 1 + if [[ ! -f /tmp/mock-cleaner-stop-after-start ]]; then printf '%s\n' true > /tmp/mock-cleaner-state/running; fi + ;; + compose) + if printf '%s\n' "$@" | grep -Fxq config; then + if [[ " ${*} " == *' --quiet '* ]]; then exit 0; fi + cat <<'YAML' +services: + postgres: + network_mode: host + volumes: + - type: bind + source: /var/lib/makepad/postgres + target: /var/lib/postgresql/data +YAML + exit 0 + fi + if printf '%s\n' "$@" | grep -Fxq up; then + counter_file=/tmp/mock-compose-up-count + count=0 + [[ ! -f "${counter_file}" ]] || read -r count < "${counter_file}" + count=$((count + 1)) + printf '%s\n' "${count}" > "${counter_file}" + if [[ "${MOCK_COMPOSE_FAIL_FIRST:-0}" == 1 && "${count}" == 1 ]]; then exit 44; fi + if [[ " ${*} " == *' keycloak_brio_staging_backup '* ]]; then + create_backup keycloak_brio_staging + fi + exit 0 + fi + echo "Unhandled docker compose invocation" >&2 + exit 1 + ;; + config|secret|network) + object_kind=${command_name} + action=${1:-} + shift || true + state_root=/tmp/mock-docker-state + case "${action}" in + inspect) + object_name=${1:-} + state_file="${state_root}/${object_kind}/${object_name}" + [[ -f "${state_file}" ]] || exit 1 + format= + while (($#)); do + if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi + done + if [[ "${format}" == *makepad.brio.deployment-id* ]]; then + sed -n 's/^deployment_id=//p' "${state_file}" + elif [[ "${format}" == *content-sha256* ]]; then + sed -n 's/^digest=//p' "${state_file}" + elif [[ "${object_kind}" == config && "${format}" == *'.Spec.Data'* ]]; then + cat "${state_root}/config-data/${object_name}" + elif [[ "${object_kind}" == network && -n "${format}" ]]; then + internal=$(sed -n 's/^internal=//p' "${state_file}") + printf 'overlay swarm %s true true\n' "${internal:-false}" + fi + ;; + create) + mkdir -p "${state_root}/${object_kind}" + declare -a values=("$@") + object_name=${values[${#values[@]}-1]} + [[ "${object_kind}" == network ]] || object_name=${values[${#values[@]}-2]} + deployment_id= + digest= + internal=false + for ((index=0; index<${#values[@]}; index++)); do + if [[ "${values[index]}" == --label && $((index + 1)) -lt ${#values[@]} ]]; then + case "${values[index+1]}" in + makepad.brio.deployment-id=*) deployment_id=${values[index+1]#*=} ;; + content-sha256=*) digest=${values[index+1]#*=} ;; + esac + elif [[ "${values[index]}" == --internal ]]; then + internal=true + fi + done + printf 'deployment_id=%s\ndigest=%s\ninternal=%s\n' "${deployment_id}" "${digest}" "${internal}" > "${state_root}/${object_kind}/${object_name}" + if [[ "${object_kind}" == config ]]; then + mkdir -p "${state_root}/config-data" + cp "${values[${#values[@]}-1]}" "${state_root}/config-data/${object_name}" + fi + ;; + rm) + rm -f -- "${state_root}/${object_kind}/${1:-}" + [[ "${object_kind}" != config ]] || rm -f -- "${state_root}/config-data/${1:-}" + ;; + ls) + requested_owner= + while (($#)); do + case "$1" in + --filter) requested_owner=${2#label=makepad.brio.deployment-id=}; shift 2 ;; + --format) shift 2 ;; + *) shift ;; + esac + done + [[ -d "${state_root}/${object_kind}" ]] || exit 0 + for state_file in "${state_root}/${object_kind}"/*; do + [[ -f "${state_file}" ]] || continue + [[ -z "${requested_owner}" || $(sed -n 's/^deployment_id=//p' "${state_file}") == "${requested_owner}" ]] || continue + basename "${state_file}" + done + ;; + *) exit 1 ;; + esac + ;; + stack) + action=${1:-} + case "${action}" in + config|rm) exit 0 ;; + services) + stack_name=${2:-} + [[ -d /tmp/mock-docker-state/service ]] || exit 1 + for state_file in /tmp/mock-docker-state/service/"${stack_name}"_*; do + [[ -f "${state_file}" ]] || continue + basename "${state_file}" + done + ;; + *) exit 1 ;; + esac + ;; + service) + action=${1:-} + shift || true + state_root=/tmp/mock-docker-state/service + case "${action}" in + inspect) + service_name=${1:-}; shift || true + state_file="${state_root}/${service_name}" + [[ -f "${state_file}" ]] || exit 1 + format= + while (($#)); do if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi; done + current=$(sed -n 's/^current=//p' "${state_file}") + previous=$(sed -n 's/^previous=//p' "${state_file}") + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + case "${format}" in + '') printf '[{"Spec":{"Name":"%s","Marker":"%s"}}]\n' "${service_name}" "${current}" ;; + *'com.docker.stack.namespace'*) printf '%s\n' "${namespace}" ;; + *'json .Spec'*) printf '{"Name":"%s","Marker":"%s"}\n' "${service_name}" "${current}" ;; + *'.PreviousSpec'*) if [[ -n "${previous}" ]]; then printf '%s\n' present; else printf '%s\n' absent; fi ;; + *) echo "Unhandled mock service format: ${format}" >&2; exit 1 ;; + esac + ;; + rollback) + while [[ "${1:-}" == --* ]]; do + if [[ "$1" == --detach=false ]]; then shift; else shift 2; fi + done + service_name=${1:-}; state_file="${state_root}/${service_name}" + [[ -f "${state_file}" ]] || exit 1 + current=$(sed -n 's/^current=//p' "${state_file}") + previous=$(sed -n 's/^previous=//p' "${state_file}") + [[ -n "${previous}" ]] || exit 1 + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + printf 'current=%s\nprevious=%s\nnamespace=%s\n' "${previous}" "${current}" "${namespace}" > "${state_file}" + ;; + rm) rm -f -- "${state_root}/${1:-}" ;; + *) exit 1 ;; + esac + ;; + run) + if printf '%s\n' "$@" | grep -Fxq makepad-postgres-brio-tmp-cleaner; then + cleaner_state=/tmp/mock-cleaner-state + install -d -m 0700 "${cleaner_state}" + declare -a cleaner_args=("$@") + cleaner_image= + cleaner_command= + for ((cleaner_index=0; cleaner_index<${#cleaner_args[@]}; cleaner_index++)); do + if [[ "${cleaner_args[cleaner_index]}" == sh && "${cleaner_args[cleaner_index+1]:-}" == -euc ]]; then + cleaner_image=${cleaner_args[cleaner_index-1]} + cleaner_command=${cleaner_args[cleaner_index+2]:-} + break + fi + done + [[ -n "${cleaner_image}" && -n "${cleaner_command}" ]] || exit 1 + printf '%s\n' "${cleaner_image}" > "${cleaner_state}/image" + printf '%s\n' true > "${cleaner_state}/running" + printf '%s\n' "${cleaner_command}" > "${cleaner_state}/command" + exit 0 + fi + declare -a environment=() + declare -a mock_mount_sources=() + declare -a mock_mount_destinations=() + while (($#)); do + case "$1" in + --rm|-i|--read-only) shift ;; + --mount) map_mount "$2"; shift 2 ;; + -v|--volume) + mount_value=$2 + source=${mount_value%%:*} + remainder=${mount_value#*:} + destination=${remainder%%:*} + map_mount "src=${source},dst=${destination}" + shift 2 + ;; + -e|--env) environment+=("$2"); shift 2 ;; + --network|--entrypoint|--tmpfs|--pids-limit|--memory|--cap-drop|--cap-add|--security-opt|--user) shift 2 ;; + --*) shift ;; + *) image=$1; shift; break ;; + esac + done + : "${image:?missing mock image}" + for assignment in "${environment[@]}"; do export "${assignment}"; done + if [[ "${1:-}" == /usr/local/bin/brio-db-transaction.sh ]]; then + operation=${2:-} + scope=${3:-} + destination=${4:-} + case "${scope}" in brio|keycloak) ;; *) exit 1 ;; esac + case "${operation}" in + prepare) + mkdir -p "${destination}" + if [[ -f /tmp/mock-db-state ]]; then cp /tmp/mock-db-state "${destination}/mock-state"; else printf '%s\n' prior > "${destination}/mock-state"; fi + printf '%s\n' '-- mock exact restore program' > "${destination}/restore.sql" + printf '%s\n' mock-prestate-fingerprint > "${destination}/prestate.fingerprint" + ;; + restore) + [[ -s "${destination}/mock-state" && -s "${destination}/restore.sql" && -s "${destination}/prestate.fingerprint" ]] + cp "${destination}/mock-state" /tmp/mock-db-state + ;; + *) exit 1 ;; + esac + exit 0 + fi + if [[ "${1:-}" == /usr/local/bin/run-brio-encrypted-backup.sh ]]; then + create_backup "${BRIO_BACKUP_DATABASE:?}" + exit 0 + fi + if [[ "${1:-}" == getent && "${2:-}" == hosts ]]; then + printf '%s %s\n' 10.44.0.2 "${3:-makepad-postgres-brio-staging}" + exit 0 + fi + if [[ "${1:-}" == sh && "${2:-}" == -euc && "${3:-}" == *'exec psql'* ]]; then + script=${3} + if [[ "${script}" == *'-f /tmp/bootstrap.sql'* ]]; then + printf '%s\n' candidate > /tmp/mock-db-state + exit 0 + fi + role=${5:-}; database=${6:-}; sslmode=${7:-}; query=${8:-} + [[ "${sslmode}" != disable ]] || exit 1 + [[ "${database}" != postgres ]] || exit 1 + if [[ "${query}" == *current_database* ]]; then printf '%s:%s\n' "${database}" "${role}" + elif [[ "${query}" == *default_transaction_read_only* ]]; then printf '%s\n' on + else printf '%s\n' 1; fi + exit 0 + fi + if [[ "${1:-}" == sh && "${2:-}" == -euc ]]; then + script=${3} + for ((mount_index=0; mount_index<${#mock_mount_sources[@]}; mount_index++)); do + mount_source=${mock_mount_sources[mount_index]} + mount_destination=${mock_mount_destinations[mount_index]} + case "${mount_destination}" in + /journal|/rollback|/managed-var-lib|/host/etc|/host/var/lib) + script=${script//${mount_destination}/${mount_source}} + ;; + esac + done + script=${script//\/host\//\/} + script=${script//\/host/\/} + shift 3 + env "${environment[@]}" sh -euc "${script}" "$@" + command_status=$? + if [[ ${command_status} -eq 0 && "${script}" == *'tar --numeric-owner -xpf'* ]]; then + for ((mount_index=0; mount_index<${#mock_mount_sources[@]}; mount_index++)); do + mount_source=${mock_mount_sources[mount_index]} + mount_destination=${mock_mount_destinations[mount_index]} + case "${mount_destination}" in + /managed/live|/managed/etc) + find "${mount_source}" -mindepth 1 -delete + cp -a "${mount_destination}/." "${mount_source}/" + ;; + esac + done + fi + exit "${command_status}" + fi + env "${environment[@]}" "$@" + ;; + *) + echo "Unhandled mock docker command: ${command_name} $*" >&2 + exit 1 + ;; +esac +MOCK +chmod 0755 "${mock_bin}/docker" +export PATH="${mock_bin}:${PATH}" + +generate_certificates() { + local destination=$1 + install -d -m 0700 "${destination}" + openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=Brio-Test-CA \ + -keyout "${destination}/ca.key" -out "${destination}/ca.crt" >/dev/null 2>&1 + openssl req -newkey rsa:2048 -nodes -subj /CN=makepad-postgres-brio-staging \ + -keyout "${destination}/server.key" -out "${destination}/server.csr" >/dev/null 2>&1 + cat > "${destination}/server.ext" <<'EOF' +subjectAltName=DNS:makepad-postgres-brio-staging,IP:65.21.134.125 +extendedKeyUsage=serverAuth +EOF + openssl x509 -req -days 30 -in "${destination}/server.csr" \ + -CA "${destination}/ca.crt" -CAkey "${destination}/ca.key" -CAcreateserial \ + -extfile "${destination}/server.ext" -out "${destination}/server.crt" >/dev/null 2>&1 + openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=Brio-Backup-Test \ + -keyout "${destination}/recipient.key" -out "${destination}/recipient.crt" >/dev/null 2>&1 +} + +pki=/tmp/brio-fixture-pki +generate_certificates "${pki}" +export MOCK_POSTGRES_IMAGE='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +validation_image='postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825' + +remove_exact() { + local target=$1 + if [[ -L "${target}" || -f "${target}" ]]; then + rm -f -- "${target}" + elif [[ -d "${target}" ]]; then + find "${target}" -mindepth 1 -delete + rmdir -- "${target}" + elif [[ -e "${target}" ]]; then + echo "Refusing unexpected fixture path type: ${target}" >&2 + exit 1 + fi +} + +reset_canary_host() { + for target in /etc/makepad /var/lib/makepad /host /runtime /rollback /managed /tmp/mock-docker-state /tmp/mock-db-state; do remove_exact "${target}"; done + install -d -m 0755 /etc/makepad/secrets /etc/makepad/tls/postgres /etc/makepad/tls/backups /var/lib/makepad/postgres-backups + printf '%s\n' old-canary-superuser > /etc/makepad/secrets/postgres-canary-superuser-password + printf '%s\n' old-canary-backup > /etc/makepad/secrets/postgres-brio-app-backup-password + cp "${pki}/ca.crt" /etc/makepad/tls/postgres/ca.crt + cp "${pki}/recipient.crt" /etc/makepad/tls/backups/brio-recipient.crt + install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/brio-staging + printf '%s\n' preserve-me > /var/lib/makepad/postgres-backups/brio-staging/sentinel + chmod 0600 /etc/makepad/secrets/postgres-canary-superuser-password + chown 999:999 /etc/makepad/secrets/postgres-brio-app-backup-password + chmod 0400 /etc/makepad/secrets/postgres-brio-app-backup-password + chmod 0444 /etc/makepad/tls/postgres/ca.crt /etc/makepad/tls/backups/brio-recipient.crt + install -d -m 0700 /tmp/mock-docker-state/service + docker network create --driver overlay --attachable --opt encrypted=true makepad_canary_primary_db >/dev/null + docker network create --driver overlay --attachable --opt encrypted=true makepad_canary_lpc_db >/dev/null + printf 'current=prior-postgres\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_postgres + printf 'current=prior-backup\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_brio_staging_backup + printf '%s\n' prior > /tmp/mock-db-state +} + +prepare_canary() { + local id=$1 + canary_bundle="/srv/test/.deploy/postgres-${id}" + canary_runtime="/tmp/postgres-brio-canary-runtime-${id}" + remove_exact "${canary_bundle}" + remove_exact "${canary_runtime}" + install -d -m 0700 "${canary_bundle}/envs/canary" "${canary_bundle}/config" "${canary_bundle}/scripts" "${canary_bundle}/bootstrap" "${canary_runtime}" + cat > "${canary_bundle}/envs/canary/.env.db" < "${canary_bundle}/envs/canary/.env.deploy" <<'EOF' +MAKEPAD_POSTGRES_DB_NETWORK=makepad_canary_primary_db +MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=makepad_canary_lpc_db +MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK=makepad_brio_staging_db +EOF + printf '%s\n' 'services: {}' > "${canary_bundle}/compose.yml" + printf '%s\n' 'services: {}' > "${canary_bundle}/envs/canary/compose.yml" + printf '%s\n' 'host all all all scram-sha-256' > "${canary_bundle}/config/runtrace-pg_hba.conf" + printf '%s\n' '# test' > "${canary_bundle}/bootstrap/brio-staging-app.sql" + cat > "${canary_bundle}/scripts/deploy-postgres-stack.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "${MOCK_STACK_RESULT:-success}" == success ]] +for state_file in /tmp/mock-docker-state/service/brio-canary_*; do + [[ -f "${state_file}" ]] || continue + current=$(sed -n 's/^current=//p' "${state_file}") + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + printf 'current=candidate-%s\nprevious=%s\nnamespace=%s\n' "${state_file##*/}" "${current}" "${namespace}" > "${state_file}" +done +EOF + chmod 0755 "${canary_bundle}/scripts/deploy-postgres-stack.sh" + cp "${repo}/scripts/brio-db-transaction.sh" "${canary_bundle}/scripts/brio-db-transaction.sh" + chmod 0755 "${canary_bundle}/scripts/brio-db-transaction.sh" + printf '%s\n' new-superuser > "${canary_runtime}/postgres-superuser-password" + printf '%s\n' new-app-password > "${canary_runtime}/brio-staging-app-password" + printf '%s\n' new-backup-password > "${canary_runtime}/brio-staging-backup-password" + cp "${pki}/ca.crt" "${canary_runtime}/postgres-ca.pem" + cp "${pki}/server.crt" "${canary_runtime}/postgres-server-cert.pem" + cp "${pki}/server.key" "${canary_runtime}/postgres-server-key.pem" + cp "${pki}/recipient.crt" "${canary_runtime}/brio-backup-recipient-cert.pem" + chmod 0600 "${canary_runtime}"/* +} + +assert_canary_restored() { + [[ $(< /etc/makepad/secrets/postgres-canary-superuser-password) == old-canary-superuser ]] + [[ $(< /etc/makepad/secrets/postgres-brio-app-backup-password) == old-canary-backup ]] + [[ $(< /var/lib/makepad/postgres-backups/brio-staging/sentinel) == preserve-me ]] + [[ $(sed -n 's/^current=//p' /tmp/mock-docker-state/service/brio-canary_postgres) == prior-postgres ]] + [[ $(sed -n 's/^current=//p' /tmp/mock-docker-state/service/brio-canary_brio_staging_backup) == prior-backup ]] + [[ $(< /tmp/mock-db-state) == prior ]] + [[ -f /tmp/mock-docker-state/network/makepad_canary_primary_db ]] + [[ -f /tmp/mock-docker-state/network/makepad_canary_lpc_db ]] + [[ ! -e /tmp/mock-docker-state/network/makepad_brio_staging_db ]] + ! find /tmp/mock-docker-state/config /tmp/mock-docker-state/secret -type f -print -quit 2>/dev/null | grep -q . +} + +run_canary_failure() { + local id=$1 injection=$2 stack_result=${3:-success} + reset_canary_host + prepare_canary "${id}" + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${injection}" MOCK_STACK_RESULT="${stack_result}" \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] +} + +run_canary_failure 101-1 after-managed-file-promotion +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-101-1 ]] + +run_canary_failure 102-1 term-after-managed-file-promotion +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-102-1 ]] + +run_canary_failure 103-1 after-stack-deploy success +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-103-1 ]] + +canary_injections=(after-bootstrap after-app-probe after-plaintext-probe after-nontarget-probe after-backup-role-probe after-backup-verification) +canary_case=110 +for injection in "${canary_injections[@]}"; do + run_canary_failure "${canary_case}-1" "${injection}" + assert_canary_restored + [[ ! -e "/var/lib/makepad/postgres-recovery/brio-canary/${canary_case}-1" ]] + canary_case=$((canary_case + 1)) +done + +# A SIGKILL cannot execute traps. The next attempt must recover the durable +# transaction before taking its own snapshot, at both host and DB boundaries. +for kill_injection in kill-after-managed-file-promotion kill-after-bootstrap; do + reset_canary_host + prepare_canary 120-1 + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${kill_injection}" \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-kill-output 2>&1 + kill_status=$? + set -e + [[ ${kill_status} -eq 137 ]] + [[ -f /var/lib/makepad/postgres-recovery/brio-canary/120-1/IN_PROGRESS ]] + prepare_canary 120-2 + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-recovery-output 2>&1 + recovery_status=$? + set -e + [[ ${recovery_status} -ne 0 ]] + assert_canary_restored + [[ ! -e /var/lib/makepad/postgres-recovery/brio-canary/120-1 ]] + remove_exact /tmp/postgres-brio-canary-runtime-120-1 +done + +# stack deploy has no implicit --prune. A stale legacy identity backup service +# is inventoried and blocks the release without mutating its exact prior Spec. +reset_canary_host +printf 'current=legacy-exact\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_keycloak_brio_staging_backup +legacy_before=$(docker service inspect brio-canary_keycloak_brio_staging_backup --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) +prepare_canary 121-1 +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-legacy-output 2>&1 +legacy_status=$? +set -e +[[ ${legacy_status} -ne 0 ]] +grep -q 'Retire it through a separately reviewed operation' /tmp/canary-legacy-output +legacy_after=$(docker service inspect brio-canary_keycloak_brio_staging_backup --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) +[[ "${legacy_before}" == "${legacy_after}" && $(< /tmp/mock-db-state) == prior ]] + +reset_canary_host +prepare_canary 103-2 +remove_exact /var/lib/makepad/postgres-backups/brio-staging +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-absent-output 2>&1 +status=$? +set -e +[[ ${status} -ne 0 ]] +[[ ! -e /var/lib/makepad/postgres-backups/brio-staging ]] + +run_canary_failure 104-1 rollback-restore fail +[[ -f /tmp/postgres-brio-canary-runtime-104-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-canary/104-1/rollback/managed.tar ]] +[[ $(stat -c '%u:%a' /var/lib/makepad/postgres-recovery/brio-canary/104-1) == 0:700 ]] +for secret in postgres-superuser-password brio-staging-app-password brio-staging-backup-password postgres-server-key.pem; do + [[ ! -e "/tmp/postgres-brio-canary-runtime-104-1/${secret}" ]] +done + +# Check every canary-controlled parent before the journal snapshot. No bind +# mount may hide a symlink in a mutable host path or touch its outside target. +canary_outside=/tmp/postgres-brio-canary-outside-sentinel +remove_exact "${canary_outside}" +install -d -m 0700 "${canary_outside}" +printf '%s\n' untouched > "${canary_outside}/sentinel" +for parent in \ + /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres /etc/makepad/tls/backups \ + /var/lib/makepad /var/lib/makepad/postgres-backups /var/lib/makepad/postgres-recovery; do + reset_canary_host + prepare_canary 105-1 + remove_exact "${parent}" + ln -s "${canary_outside}" "${parent}" + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-symlink-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] + grep -Eq 'symlink component|missing, not a directory, or a symlink' /tmp/canary-symlink-output + [[ $(< "${canary_outside}/sentinel") == untouched ]] + remove_exact "${parent}" +done +remove_exact "${canary_outside}" + +reset_identity_host() { + for target in /srv/makepad/postgres /etc/makepad /var/lib/makepad /host /runtime /rollback /managed /managed-var-lib /tmp/mock-compose-up-count /tmp/mock-db-state; do remove_exact "${target}"; done + install -d -m 0755 /srv/makepad/postgres/{bootstrap,config,envs/production,scripts} /etc/makepad/secrets /etc/makepad/tls/postgres /etc/makepad/tls/backups /var/lib/makepad/postgres /var/lib/makepad/postgres-backups + printf '%s\n' old-live-compose > /srv/makepad/postgres/compose.host.yml + printf '%s\n' old-live-env > /srv/makepad/postgres/envs/production/.env.db + printf '%s\n' old-live-hba > /srv/makepad/postgres/config/runtrace-pg_hba.conf + for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh; do printf '%s\n' old > "/srv/makepad/postgres/scripts/${script}"; done + printf '%s\n' old-superuser > /etc/makepad/secrets/postgres-superuser-password + cp "${pki}/server.crt" /etc/makepad/tls/postgres/server.crt + cp "${pki}/server.key" /etc/makepad/secrets/postgres-server.key + cp "${pki}/ca.crt" /etc/makepad/tls/postgres/ca.crt + chmod 0600 /etc/makepad/secrets/postgres-superuser-password + chown 70:70 /etc/makepad/secrets/postgres-server.key + chmod 0400 /etc/makepad/secrets/postgres-server.key + chmod 0444 /etc/makepad/tls/postgres/server.crt /etc/makepad/tls/postgres/ca.crt + install -d -o 999 -g 999 -m 0700 \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z + printf '%s\n' preserved-encrypted-backup > \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z/sentinel + ln -s 20200101T000000Z /var/lib/makepad/postgres-backups/keycloak-brio-staging/latest + printf '%s\n' '{"database":"keycloak_brio_staging","encrypted":true,"backup":"20200101T000000Z"}' > \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/last-success.json + chown -R 999:999 /var/lib/makepad/postgres-backups/keycloak-brio-staging + printf '%s\n' prior > /tmp/mock-db-state +} + +prepare_identity() { + local id=$1 + identity_bundle="/tmp/postgres-brio-identity-bundle-${id}" + identity_runtime="/tmp/postgres-brio-identity-runtime-${id}" + remove_exact "${identity_bundle}" + remove_exact "${identity_runtime}" + install -d -m 0700 "${identity_bundle}/envs/production" "${identity_bundle}/config" "${identity_bundle}/bootstrap" "${identity_bundle}/scripts" "${identity_runtime}" + cat > "${identity_bundle}/envs/production/.env.db" < "${identity_bundle}/compose.host.yml" <<'EOF' +services: + postgres: + network_mode: host + volumes: + - "${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data" +EOF + cat > "${identity_bundle}/config/runtrace-pg_hba.conf" <<'EOF' +hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_backup 127.0.0.1/32 scram-sha-256 +host all keycloak_brio_staging_app all reject +host all keycloak_brio_staging_backup all reject +EOF + printf '%s\n' '# bootstrap' > "${identity_bundle}/bootstrap/keycloak-brio-staging.sql" + for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + printf '%s\n' '#!/bin/sh' 'exit 0' > "${identity_bundle}/scripts/${script}" + chmod 0755 "${identity_bundle}/scripts/${script}" + done + cp "${repo}/scripts/brio-db-transaction.sh" "${identity_bundle}/scripts/brio-db-transaction.sh" + chmod 0755 "${identity_bundle}/scripts/brio-db-transaction.sh" + printf '%s\n' identity-app-new > "${identity_runtime}/keycloak-brio-staging-app-password" + printf '%s\n' identity-backup-new > "${identity_runtime}/keycloak-brio-staging-backup-password" + cp "${pki}/recipient.crt" "${identity_runtime}/brio-backup-recipient-cert.pem" + chmod 0600 "${identity_runtime}"/* +} + +assert_identity_restored() { + [[ $(< /srv/makepad/postgres/compose.host.yml) == old-live-compose ]] + [[ $(< /srv/makepad/postgres/config/runtrace-pg_hba.conf) == old-live-hba ]] + [[ ! -e /etc/makepad/secrets/postgres-brio-identity-backup-password ]] + [[ $(< /tmp/mock-db-state) == prior ]] + [[ $(readlink /var/lib/makepad/postgres-backups/keycloak-brio-staging/latest) == 20200101T000000Z ]] + [[ $(< /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z/sentinel) == preserved-encrypted-backup ]] + [[ $(< /var/lib/makepad/postgres-backups/keycloak-brio-staging/last-success.json) == '{"database":"keycloak_brio_staging","encrypted":true,"backup":"20200101T000000Z"}' ]] + [[ $(find /var/lib/makepad/postgres-backups/keycloak-brio-staging -mindepth 1 -maxdepth 1 -printf '%f\n' | sort | tr '\n' ' ') == '20200101T000000Z last-success.json latest ' ]] +} + +run_identity_failure() { + local id=$1 injection=$2 fail_first=${3:-0} + reset_identity_host + prepare_identity "${id}" + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${injection}" \ + MOCK_COMPOSE_FAIL_FIRST="${fail_first}" \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] +} + +run_identity_failure 201-1 after-managed-file-promotion +assert_identity_restored +[[ ! -e /tmp/postgres-brio-identity-runtime-201-1 ]] + +run_identity_failure 202-1 term-after-managed-file-promotion +assert_identity_restored +[[ ! -e /tmp/postgres-brio-identity-runtime-202-1 ]] + +identity_injections=(after-bootstrap after-app-probe after-plaintext-probe after-nontarget-probe after-backup-role-probe after-backup-verification) +identity_case=210 +for injection in "${identity_injections[@]}"; do + run_identity_failure "${identity_case}-1" "${injection}" + assert_identity_restored + [[ ! -e "/var/lib/makepad/postgres-recovery/brio-identity/${identity_case}-1" ]] + identity_case=$((identity_case + 1)) +done + +for kill_injection in kill-after-managed-file-promotion kill-after-bootstrap; do + reset_identity_host + prepare_identity 220-1 + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes BRIO_DEPLOY_TEST_MODE=isolated-container \ + BRIO_DEPLOY_FAILURE_INJECTION="${kill_injection}" \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-kill-output 2>&1 + kill_status=$? + set -e + [[ ${kill_status} -eq 137 ]] + [[ -f /var/lib/makepad/postgres-recovery/brio-identity/220-1/IN_PROGRESS ]] + prepare_identity 220-2 + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes BRIO_DEPLOY_TEST_MODE=isolated-container \ + BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-recovery-output 2>&1 + recovery_status=$? + set -e + [[ ${recovery_status} -ne 0 ]] + assert_identity_restored + [[ ! -e /var/lib/makepad/postgres-recovery/brio-identity/220-1 ]] + remove_exact /tmp/postgres-brio-identity-runtime-220-1 +done + +run_identity_failure 203-1 rollback-restore 1 +[[ -f /tmp/postgres-brio-identity-runtime-203-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/203-1/rollback/managed.tar ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/203-1/RECOVERY_REQUIRED ]] +[[ $(stat -c '%u:%a' /var/lib/makepad/postgres-recovery/brio-identity/203-1) == 0:700 ]] +for secret in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + [[ ! -e "/tmp/postgres-brio-identity-runtime-203-1/${secret}" ]] +done + +run_identity_failure 204-1 rollback-recreate 1 +assert_identity_restored +[[ -f /tmp/postgres-brio-identity-runtime-204-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/204-1/RECOVERY_REQUIRED ]] + +reset_identity_host +prepare_identity 205-1 +set +e +BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ +BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes MOCK_COMPOSE_PROJECT=unexpected-project \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-target-output 2>&1 +status=$? +set -e +[[ ${status} -ne 0 ]] +grep -q 'exact Compose label' /tmp/identity-target-output +assert_identity_restored + +# Every repository-controlled mutable parent is checked component-by-component +# both before snapshot and within promotion. Symlink targets remain untouched. +outside=/tmp/postgres-brio-outside-sentinel +remove_exact "${outside}" +install -d -m 0700 "${outside}" +printf '%s\n' untouched > "${outside}/sentinel" +for parent in \ + /srv/makepad /srv/makepad/postgres \ + /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres /etc/makepad/tls/backups \ + /var/lib/makepad /var/lib/makepad/postgres-backups; do + reset_identity_host + prepare_identity 230-1 + remove_exact "${parent}" + ln -s "${outside}" "${parent}" + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-parent-symlink-output 2>&1 + symlink_status=$? + set -e + [[ ${symlink_status} -ne 0 ]] + grep -Eq 'symlink|missing|unavailable|unsafe' /tmp/identity-parent-symlink-output + [[ $(< "${outside}/sentinel") == untouched ]] + remove_exact "${parent}" +done +remove_exact "${outside}" + +cleaner_root=/tmp/postgres-brio-cleaner-test-failures +remove_exact "${cleaner_root}" +install -d "${cleaner_root}/postgres-brio-delete" "${cleaner_root}/postgres-brio-preserve" +printf '%s\n' recovery > "${cleaner_root}/postgres-brio-preserve/RECOVERY_REQUIRED" +touch -t 202001010000 "${cleaner_root}/postgres-brio-delete" "${cleaner_root}/postgres-brio-preserve" +"${repo}/scripts/ensure-brio-tmp-cleaner.sh" test-clean-once "${cleaner_root}" +[[ ! -e "${cleaner_root}/postgres-brio-delete" ]] +[[ -f "${cleaner_root}/postgres-brio-preserve/RECOVERY_REQUIRED" ]] + +# Existing host cleaners are trusted only when their entire command contract +# matches, and a stopped instance must remain running after an attempted start. +cleaner_env=/tmp/postgres-brio-cleaner-test-failures.env +printf 'POSTGRES_IMAGE=%s\n' "${MOCK_POSTGRES_IMAGE}" > "${cleaner_env}" +remove_exact /tmp/mock-cleaner-state +remove_exact /tmp/mock-cleaner-stop-after-start +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" +[[ $(< /tmp/mock-cleaner-state/running) == true ]] +printf '%s\n' 'unexpected cleaner command' > /tmp/mock-cleaner-state/command +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" >/tmp/cleaner-command-output 2>&1 +cleaner_command_status=$? +set -e +[[ ${cleaner_command_status} -ne 0 ]] +grep -q 'does not match the fail-closed cleanup contract' /tmp/cleaner-command-output +remove_exact /tmp/mock-cleaner-state +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" +printf '%s\n' false > /tmp/mock-cleaner-state/running +touch /tmp/mock-cleaner-stop-after-start +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" >/tmp/cleaner-running-output 2>&1 +cleaner_running_status=$? +set -e +[[ ${cleaner_running_status} -ne 0 ]] +grep -q 'did not remain running after startup' /tmp/cleaner-running-output +remove_exact /tmp/mock-cleaner-stop-after-start + +echo "Brio deployment failure-injection tests passed." diff --git a/scripts/test-brio-db-transaction.sh b/scripts/test-brio-db-transaction.sh new file mode 100755 index 0000000..e20d06a --- /dev/null +++ b/scripts/test-brio-db-transaction.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +postgres_image=$(awk -F= '$1 == "POSTGRES_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +container="brio-db-transaction-${RANDOM}-$$" +superuser_password='transaction-superuser-only' +old_app_password='transaction-old-app-only' +old_backup_password='transaction-old-backup-only' +new_app_password='transaction-new-app-only' +new_backup_password='transaction-new-backup-only' + +cleanup() { + local status=$? + trap - EXIT + docker rm -f "${container}" >/dev/null 2>&1 || true + exit "${status}" +} +trap cleanup EXIT + +docker run -d --name "${container}" \ + -e "POSTGRES_PASSWORD=${superuser_password}" \ + -e POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 \ + --mount "type=bind,src=${repo_root}/bootstrap,dst=/bootstrap,readonly" \ + --mount "type=bind,src=${repo_root}/scripts/brio-db-transaction.sh,dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + "${postgres_image}" >/dev/null +for _ in $(seq 1 300); do + docker exec -e "PGPASSWORD=${superuser_password}" "${container}" \ + psql -X -h 127.0.0.1 -U postgres -d postgres -c 'select 1' >/dev/null 2>&1 && break + sleep 0.1 +done +docker exec -e "PGPASSWORD=${superuser_password}" "${container}" \ + psql -X -h 127.0.0.1 -U postgres -d postgres -c 'select 1' >/dev/null +docker exec "${container}" sh -euc 'install -d -m 0700 /journal/keycloak /journal/brio; printf "%s" "$1" > /run/superuser-password; chmod 0600 /run/superuser-password' sh "${superuser_password}" + +psql_admin() { + docker exec -e "PGPASSWORD=${superuser_password}" "${container}" psql -X -v ON_ERROR_STOP=1 -U postgres "$@" +} +transaction() { + docker exec \ + -e PGUSER=postgres -e PGHOST=127.0.0.1 -e PGSSLMODE=disable -e PGPASSWORD_FILE=/run/superuser-password \ + "${container}" /usr/local/bin/brio-db-transaction.sh "$@" +} + +psql_admin -d postgres \ + -c "CREATE ROLE brio_legacy_owner NOLOGIN" \ + -c "CREATE ROLE keycloak_brio_staging_app LOGIN NOINHERIT CONNECTION LIMIT 7 PASSWORD '${old_app_password}' VALID UNTIL '2035-01-02 03:04:05+00'" \ + -c "CREATE ROLE keycloak_brio_staging_backup LOGIN INHERIT CONNECTION LIMIT 3 PASSWORD '${old_backup_password}'" \ + -c "ALTER ROLE keycloak_brio_staging_app SET statement_timeout TO '17s'" \ + -c "CREATE DATABASE keycloak_brio_staging OWNER brio_legacy_owner" \ + -c "REVOKE CONNECT ON DATABASE keycloak_brio_staging FROM PUBLIC" \ + -c "GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_app" \ + -c "GRANT CREATE ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "ALTER ROLE keycloak_brio_staging_backup IN DATABASE keycloak_brio_staging SET lock_timeout TO '19s'" >/dev/null +psql_admin -d keycloak_brio_staging \ + -c "REVOKE ALL ON SCHEMA public FROM PUBLIC" \ + -c "GRANT CREATE ON SCHEMA public TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "CREATE TABLE public.preexisting_acl_probe(id integer)" \ + -c "GRANT UPDATE ON public.preexisting_acl_probe TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT INSERT ON TABLES TO keycloak_brio_staging_backup WITH GRANT OPTION" >/dev/null + +transaction prepare keycloak /journal/keycloak +pre_fingerprint=$(transaction fingerprint keycloak /journal/keycloak | sha256sum | cut -d' ' -f1) +psql_admin -d postgres \ + -v "keycloak_brio_staging_app_password=${new_app_password}" \ + -v "keycloak_brio_staging_backup_password=${new_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null +docker exec -e "PGPASSWORD=${new_app_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null +docker exec -e "PGPASSWORD=${new_backup_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null +if docker exec -e "PGPASSWORD=${old_app_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null 2>&1; then + echo "Old application credential unexpectedly survived the simulated mutation." >&2 + exit 1 +fi +if docker exec -e "PGPASSWORD=${old_backup_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null 2>&1; then + echo "Old backup credential unexpectedly survived the simulated mutation." >&2 + exit 1 +fi +transaction restore keycloak /journal/keycloak +post_fingerprint=$(transaction fingerprint keycloak /journal/keycloak | sha256sum | cut -d' ' -f1) +[[ "${post_fingerprint}" == "${pre_fingerprint}" ]] || { echo "Exact Keycloak Brio state fingerprint was not restored." >&2; exit 1; } +docker exec -e "PGPASSWORD=${old_app_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null +docker exec -e "PGPASSWORD=${old_backup_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null +if docker exec -e "PGPASSWORD=${new_app_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null 2>&1; then + echo "New application credential remained valid after compensation." >&2 + exit 1 +fi +if docker exec -e "PGPASSWORD=${new_backup_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null 2>&1; then + echo "New backup credential remained valid after compensation." >&2 + exit 1 +fi + +# The absent-state path must remove every object introduced by a failed first +# deployment, not merely rotate credentials back. +transaction prepare brio /journal/brio +psql_admin -d postgres \ + -v "brio_staging_app_password=${new_app_password}" \ + -v "brio_staging_backup_password=${new_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null +transaction restore brio /journal/brio +[[ $(psql_admin -At -d postgres -c "SELECT count(*) FROM pg_database WHERE datname='brio_staging'") == 0 ]] +[[ $(psql_admin -At -d postgres -c "SELECT count(*) FROM pg_roles WHERE rolname IN ('brio_staging_app','brio_staging_backup')") == 0 ]] + +# A recovery mismatch must fail closed without leaking the stored fingerprint. +# That fingerprint includes SCRAM verifiers and therefore must never appear in +# CI or remote deployment diagnostics. +docker exec "${container}" sh -euc 'printf "%s\n" sentinel-secret-verifier >> /journal/brio/prestate.fingerprint' +if mismatch_output=$(transaction restore brio /journal/brio 2>&1); then + echo "A mismatched recovery fingerprint was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${mismatch_output}" == *"Database compensation did not restore the exact prior Brio state."* ]] || { + echo "Recovery mismatch did not return the expected redacted diagnostic." >&2 + exit 1 +} +[[ "${mismatch_output}" != *sentinel-secret-verifier* ]] || { + echo "Recovery mismatch leaked protected fingerprint material." >&2 + exit 1 +} +unset mismatch_output + +echo "Brio database transaction journal restores roles, SCRAM verifiers, attributes, database ownership, ACLs, defaults, and absent state." diff --git a/scripts/test-brio-deploy-guards.sh b/scripts/test-brio-deploy-guards.sh new file mode 100755 index 0000000..4562dd6 --- /dev/null +++ b/scripts/test-brio-deploy-guards.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +expect_refusal() { + local expected=$1 + shift + local output status + set +e + output=$("$@" 2>&1) + status=$? + set -e + if [[ ${status} -eq 0 || "${output}" != *"${expected}"* ]]; then + echo "Expected deployment guard refusal containing: ${expected}" >&2 + exit 1 + fi +} + +expect_refusal "unique /srv or /opt" \ + "${script_dir}/deploy-brio-canary-postgres.sh" relative-dir brio /tmp/postgres-brio-canary-runtime-1-1 + +expect_refusal "exact standalone DB restart acknowledgement" \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 65.21.134.125 88.99.209.165/32 + +expect_refusal "reviewed standalone DB IP" \ + env BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 127.0.0.1 88.99.209.165/32 + +expect_refusal "reviewed exact egress" \ + env BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 65.21.134.125 10.80.0.1/32 + +expect_refusal "unique /srv or /opt" \ + "${script_dir}/deploy-postgres-stack.sh" /srv/makepad/postgres postgres canary + +expect_refusal "Production requires a job-scoped" \ + "${script_dir}/deploy-postgres-stack.sh" /srv/makepad/postgres/.deploy/postgres-1-1 postgres production + +echo "Brio deployment guard tests passed." diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh new file mode 100755 index 0000000..c36feb8 --- /dev/null +++ b/scripts/test-brio-deployment-contracts.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) + +REPO_ROOT="${repo_root}" python3 - <<'PY' +import os +from pathlib import Path + +root = Path(os.environ["REPO_ROOT"]) +manual = (root / ".github/workflows/manual-deploy.yml").read_text() +identity_workflow = (root / ".github/workflows/deploy-brio-identity-db.yml").read_text() +release_workflow = (root / ".github/workflows/release-brio-identity-db.yml").read_text() +ci_workflow = (root / ".github/workflows/ci.yml").read_text() +finalizer_workflow = (root / ".github/workflows/pr-ci-result.yml").read_text() +check_publisher = (root / "scripts/publish-pr-ci-check.mjs").read_text() +jit_launcher = (root / "scripts/run-postgres-ci-jit-vm.sh").read_text() +queue_controller = (root / "scripts/postgres-ci-queue-controller.mjs").read_text() +cohort_workflow = (root / ".github/workflows/verify-keycloak-cohort-restores.yml").read_text() +cohort_validator = (root / "scripts/verify-keycloak-cohort-evidence.py").read_text() +identity = (root / "scripts/deploy-brio-identity-db-host.sh").read_text() +canary = (root / "scripts/deploy-brio-canary-postgres.sh").read_text() +stack = (root / "scripts/deploy-postgres-stack.sh").read_text() +vif = (root / "bootstrap/vif-app.sql").read_text() +hba = (root / "config/runtrace-pg_hba.conf").read_text().splitlines() +canary_env = (root / "envs/canary/.env.db").read_text() +production_env = (root / "envs/production/.env.db").read_text() + +def require(condition, message): + if not condition: + raise SystemExit(message) + +require("makepad_postgres_canary_runtrace_hba_v3" in canary_env, "canary HBA must use immutable v3") +require("makepad_postgres_runtrace_hba_v3" in production_env, "production HBA must use immutable v3") +require("runtrace_hba_v2" not in canary_env + production_env, "active HBA v2 names must not drift") + +records = [tuple(line.split()) for line in hba if line.strip() and not line.lstrip().startswith("#")] +for allow, reject in ( + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_app", "all", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_app", "all", "reject")), + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_backup", "127.0.0.1/32", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_backup", "all", "reject")), +): + require(records.index(allow) < records.index(reject), "identity HBA allow must precede its role rejection") + +require("group: postgres-shared-swarm-target" in manual, "shared Swarm target needs one concurrency group") +require('remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' in manual, "Swarm bundle must be unique per attempt") +require("${REMOTE_DIR}/stack.yml" not in manual + stack, "shared stack.yml is forbidden") +require('stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml"' in stack, "stack config must stay in the run bundle") +require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual + stack, "VIF secret must not persist in .env.deploy") +require("-v vif_password=" not in stack, "VIF secret must not enter psql argv") +require("\\getenv vif_password VIF_PASSWORD" in vif, "VIF bootstrap must use getenv") + +for marker in ( + "compose_project=postgres", + "expected_container_name=postgres-postgres-1", + "com.docker.compose.project", + "com.docker.compose.service", + "com.docker.compose.oneoff", + "bind|/var/lib/makepad/postgres|true", + '"${network_mode}" == "host"', + 'tar --numeric-owner -cpf "$stage/rollback/managed.tar"', + "restore_snapshot", + "rollback_deployment", + "trap handle_exit EXIT", + "trap 'exit 129' HUP", + "trap 'exit 130' INT", + "trap 'exit 143' TERM", + "up -d --remove-orphans --wait --force-recreate", +): + require(marker in identity, f"standalone contract missing: {marker}") + +snapshot = identity.index('tar --numeric-owner -cpf "$stage/rollback/managed.tar"') +armed = identity.index("rollback_armed=1", snapshot) +mutation = identity.index('install_host_path "${candidate_compose}"', armed) +backup = identity.index('[[ "${backup_verified}" == "1" ]]', mutation) +disarmed = identity.index("rollback_armed=0", backup) +require(snapshot < armed < mutation < backup < disarmed, "rollback boundary/order is unsafe") + +require("/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" in identity_workflow, "identity bundle must be per attempt") +require("POSTGRES_HOST_COMPOSE_PROJECT" not in identity_workflow, "standalone project must not be user-selected") +require("brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }}" in identity_workflow, "phase one must publish its exact immutable deployment artifact") +require(identity_workflow.count("actions/upload-artifact@") == 1, "phase one must publish exactly one artifact") +require("brio-db-deployment-evidence.json" in identity_workflow and "makepad.brio-db-deployment-evidence.v1" in identity_workflow, "phase one must use the canonical single-file schema") +for marker in ( + "environment: release-brio-identity-db", + "KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN", + "verify-brio-release-evidence.py postgres-run", + "verify-brio-release-evidence.py postgres-evidence", + "verify-brio-database.yml/dispatches", + "verify-brio-release-evidence.py verifier-run", + "verify-brio-release-evidence.py attestation", + 'release_token=${RELEASE_ORCHESTRATOR_TOKEN}', + 'unset RELEASE_ORCHESTRATOR_TOKEN', + '--config -', +): + require(marker in release_workflow, f"protected release orchestrator missing: {marker}") +require("actions/upload-artifact@" not in release_workflow, "release orchestrator must not synthesize or republish attestation") +require("pull_request_target:" in ci_workflow, "PR CI must use protected-base workflow code") +require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks") +require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact head") +require("repository_dispatch:" in finalizer_workflow and "types: [postgres-pr-ci-attestation]" in finalizer_workflow and "environment: postgres-ci-attestation" in finalizer_workflow, "PR CI result must require signed hypervisor teardown") +require("POSTGRES_PR_CHECK_APP_PRIVATE_KEY" in finalizer_workflow and 'CHECK_NAMES = ["postgres-ci"]' in check_publisher, "required PR check must be App-bound") +for marker in ( + "makepad.postgres.ci-attestation.v1", + "verifySignature", + "registration_absent", + "runnerLookupStatus !== 404", + "makepad-postgres-pr-ephemeral", +): + require(marker in check_publisher + jit_launcher, f"signed disposable PR boundary missing: {marker}") +for marker in ("generate-jitconfig", "--jitconfig", "virsh undefine", "nft delete table", "dispatch-ci-attestation.mjs", "resources.json", "--reconcile", "POSTGRES_CI_RESULT_POLL_ATTEMPTS"): + require(marker in jit_launcher, f"JIT hypervisor teardown contract missing: {marker}") +require('job.name === "policy-and-integration"' in queue_controller and "await runLauncher" in queue_controller, "queue controller must bind and supervise the exact disposable PR job") +require("await reconcileIncompleteJobs" in queue_controller and "launchID" in queue_controller, "queue controller must reconcile deterministic incomplete launches before polling") +require('association.base?.sha !== run.head_sha' in queue_controller, "queue controller must bind the exact PR base SHA") +require('association.base?.sha !== attestation.run.workflow_sha' in check_publisher, "attestor must bind the exact PR base SHA") +for marker in ( + "name: Verify Keycloak Cohort Restore Compatibility", + "keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "makepad.keycloak-cohort-restore-evidence.v2", + "restored-databases-compatible", + "keycloak_release_sha", +): + require(marker in cohort_workflow + cohort_validator, f"six-database cohort evidence contract missing: {marker}") +require("vars." not in cohort_workflow, "cohort evidence cannot rely on a mutable repository variable") +require("Ensure interrupted cohort material expires on the release host" in cohort_workflow, "cohort workflow must verify the release-host TTL guard before credentials or dumps") +require(cohort_workflow.index("Ensure interrupted cohort material expires on the release host") < cohort_workflow.index("Configure isolated SSH and registry state"), "release-host TTL guard must precede credential material") +require('"probe ${helper_digest} ${cleaner_digest}"' in cohort_workflow and cohort_workflow.index('"probe ${helper_digest} ${cleaner_digest}"') < cohort_workflow.index('"capture ${GITHUB_RUN_ID}'), "remote helper/cleaner digest and TTL probe must precede every dump") +require("scp " not in cohort_workflow and "remote_script=" not in cohort_workflow, "cohort workflow must use only the forced-command capture protocol") +require('/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}' in cohort_workflow, "cohort sensitive material must live under the exact TTL-cleaned namespace") +shared_network_validation = canary.index('prevalidate_network "${db_network}" false') +incomplete_recovery = canary.index("recover_incomplete_journals", shared_network_validation) +database_journal = canary.index('run_db_transaction prepare "${journal_stage}"', incomplete_recovery) +require(shared_network_validation < incomplete_recovery < database_journal, "shared DB transport validation must precede recovery and first-deploy journaling") +for marker in ( + "assert_no_symlink_components", + "prevalidate_swarm_config", + "prevalidate_network", + "docker stack config", + 'tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar"', + "rollback_canary", + "prior-service-spec-hashes.list", + 'mv -fT "$super_stage"', + "postgres-recovery/brio-canary", + "DATABASE_MUTATION_ARMED", + "STACK_MUTATION_ARMED", + "keycloak_brio_staging_backup", + "/usr/local/bin/run-brio-encrypted-backup.sh", +): + require(marker in canary, f"canary transactional contract missing: {marker}") +for marker in ("preserve_recovery_evidence", "postgres-recovery/brio-identity", "DATABASE_MUTATION_ARMED", "RECOVERY_REQUIRED"): + require(marker in identity, f"identity durable recovery contract missing: {marker}") +for marker in ("identity-backups.tar", "identity-backup-absent", "recovery_id=${identifier}"): + require(marker in identity, f"identity exact backup/recovery contract missing: {marker}") + +for workflow in (manual, identity_workflow): + cleaner = workflow.index("ensure-brio-tmp-cleaner.sh") + secret_copy = workflow.index('scp "${scp_opts[@]}" "${runtime_dir}') + require(cleaner < secret_copy, "host TTL cleaner must precede secret transfer") +PY + +cleaner_root=$(mktemp -d /tmp/postgres-brio-cleaner-test-contract-XXXXXX) +cleanup_test_root() { + [[ "${cleaner_root}" =~ ^/tmp/postgres-brio-cleaner-test-contract-[A-Za-z0-9]+$ ]] || return 1 + find "${cleaner_root}" -depth -delete +} +trap cleanup_test_root EXIT +mkdir "${cleaner_root}/postgres-brio-old" "${cleaner_root}/postgres-brio-recovery" \ + "${cleaner_root}/postgres-brio-fresh" "${cleaner_root}/postgres-keycloak-cohort-old" \ + "${cleaner_root}/unrelated-old" +printf '%s\n' recovery-required > "${cleaner_root}/postgres-brio-recovery/RECOVERY_REQUIRED" +touch -t 202001010000 "${cleaner_root}/postgres-brio-old" "${cleaner_root}/postgres-brio-recovery" \ + "${cleaner_root}/postgres-keycloak-cohort-old" "${cleaner_root}/unrelated-old" +"${script_dir}/ensure-brio-tmp-cleaner.sh" test-clean-once "${cleaner_root}" +[[ ! -e "${cleaner_root}/postgres-brio-old" ]] || { echo "TTL cleaner retained an expired Brio directory." >&2; exit 1; } +[[ ! -e "${cleaner_root}/postgres-keycloak-cohort-old" ]] || { echo "TTL cleaner retained expired Keycloak cohort material." >&2; exit 1; } +[[ -d "${cleaner_root}/postgres-brio-fresh" ]] || { echo "TTL cleaner removed a fresh Brio directory." >&2; exit 1; } +[[ -f "${cleaner_root}/postgres-brio-recovery/RECOVERY_REQUIRED" ]] || { echo "TTL cleaner removed required recovery evidence." >&2; exit 1; } +[[ -d "${cleaner_root}/unrelated-old" ]] || { echo "TTL cleaner removed unrelated content." >&2; exit 1; } + +# Reproduce production ownership: SSH-created runtime directories are mode 0700 +# and owned by the deploy UID, not by the cleaner container. Minimal DAC/FOWNER +# capabilities must delete expired material while retaining recovery markers. +ownership_root=/tmp/postgres-brio-cleaner-test-production-ownership +[[ ! -e "${ownership_root}" && ! -L "${ownership_root}" ]] || find "${ownership_root}" -depth -delete +install -d -m 0700 "${ownership_root}" +cleaner_image=$(awk -F= '$1 == "POSTGRES_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc ' + mkdir /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + printf "%s\n" secret > /fixture/postgres-brio-deploy-owned/credential + printf "%s\n" recovery > /fixture/postgres-brio-recovery-owned/RECOVERY_REQUIRED + chown -R 12345:12345 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + chmod 0700 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + touch -t 202001010000 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned +' +"${script_dir}/ensure-brio-tmp-cleaner.sh" test-clean-production-ownership "${ownership_root}" "${cleaner_image}" +[[ ! -e "${ownership_root}/postgres-brio-deploy-owned" ]] || { echo "Production cleaner retained a deploy-UID-owned expired secret directory." >&2; exit 1; } +[[ -f "${ownership_root}/postgres-brio-recovery-owned/RECOVERY_REQUIRED" ]] || { echo "Production cleaner removed recovery evidence." >&2; exit 1; } +docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc 'find /fixture -mindepth 1 -depth -delete' + +echo "Brio deployment ordering, rollback, interruption, secret, and TTL contracts passed." diff --git a/scripts/test-brio-deployment-failures.sh b/scripts/test-brio-deployment-failures.sh new file mode 100755 index 0000000..2b0d746 --- /dev/null +++ b/scripts/test-brio-deployment-failures.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) +test_image=$(grep '^BRIO_BACKUP_IMAGE=' "${repo_root}/envs/canary/.env.db" | cut -d= -f2-) +: "${test_image:?BRIO_BACKUP_IMAGE is required for isolated failure testing}" +[[ "${test_image}" == *@sha256:* ]] || { echo "Failure tests require a digest-pinned container image." >&2; exit 1; } + +docker run --rm \ + --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" \ + "${test_image}" bash /repo/scripts/fixtures/brio-deployment-failure-fixture.sh From b901946df97c674ca685f1085f4e61fcad8bda91 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 04:27:09 +0200 Subject: [PATCH 09/20] feat(release): attest Brio identity database promotion --- .../workflows/release-brio-identity-db.yml | 226 +++++++++++++ scripts/test-brio-release-evidence.sh | 172 ++++++++++ scripts/verify-brio-release-evidence.py | 317 ++++++++++++++++++ 3 files changed, 715 insertions(+) create mode 100644 .github/workflows/release-brio-identity-db.yml create mode 100755 scripts/test-brio-release-evidence.sh create mode 100755 scripts/verify-brio-release-evidence.py diff --git a/.github/workflows/release-brio-identity-db.yml b/.github/workflows/release-brio-identity-db.yml new file mode 100644 index 0000000..47a209e --- /dev/null +++ b/.github/workflows/release-brio-identity-db.yml @@ -0,0 +1,226 @@ +name: Release Brio Identity Database +run-name: Release Brio DB deployment ${{ inputs.postgres_deployment_run_id }}/${{ inputs.postgres_deployment_run_attempt }} + +on: + workflow_dispatch: + inputs: + postgres_deployment_run_id: + description: Exact completed Deploy Brio Identity Database run ID + required: true + type: string + postgres_deployment_run_attempt: + description: Exact completed deployment run attempt + required: true + type: string + +concurrency: + group: brio-identity-database-release-orchestrator + cancel-in-progress: false + +permissions: + contents: read + +jobs: + attest: + name: protected-cross-repository-attestation + runs-on: + group: Postgres Release + labels: [self-hosted, linux, x64, makepad, makepad-postgres-release] + environment: release-brio-identity-db + timeout-minutes: 45 + steps: + - name: Require protected main release context + shell: bash + env: + POSTGRES_RUN_ID: ${{ inputs.postgres_deployment_run_id }} + POSTGRES_RUN_ATTEMPT: ${{ inputs.postgres_deployment_run_attempt }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" && "${GITHUB_REF}" == "refs/heads/main" ]] || { + echo "The Brio database release orchestrator is restricted to Makepad-fr/postgres main." >&2 + exit 1 + } + [[ "${POSTGRES_RUN_ID}" =~ ^[1-9][0-9]*$ && "${POSTGRES_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] || { + echo "Deployment run and attempt must be positive integers." >&2 + exit 1 + } + + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate deployment evidence and dispatch exact Keycloak verifier + id: verify + shell: bash + env: + RELEASE_ORCHESTRATOR_TOKEN: ${{ secrets.KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN }} + POSTGRES_RUN_ID: ${{ inputs.postgres_deployment_run_id }} + POSTGRES_RUN_ATTEMPT: ${{ inputs.postgres_deployment_run_attempt }} + run: | + set -euo pipefail + : "${RELEASE_ORCHESTRATOR_TOKEN:?release-brio-identity-db must define KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN}" + release_token=${RELEASE_ORCHESTRATOR_TOKEN} + unset RELEASE_ORCHESTRATOR_TOKEN + work="${RUNNER_TEMP}/brio-db-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${work}" && ! -L "${work}" ]] || { echo "Refusing to reuse release workspace." >&2; exit 1; } + install -d -m 0700 "${work}" + umask 077 + api=https://api.github.com + headers=(--header 'Accept: application/vnd.github+json' --header 'X-GitHub-Api-Version: 2022-11-28') + github_api() { + # Supply authentication through curl's stdin-only config. The + # bearer value is never written to the persistent self-hosted + # runner filesystem or exposed in argv. + printf 'header = "Authorization: Bearer %s"\n' "${release_token}" | \ + curl --config - "$@" + } + fetch_complete_listing() { + local endpoint=$1 collection=$2 destination=$3 page=1 collected=0 returned total page_file + local pages="${work}/pages-${collection}-$RANDOM" + install -d -m 0700 "${pages}" + while ((page <= 1000)); do + page_file="${pages}/$(printf '%04d' "${page}").json" + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}${endpoint}&page=${page}" > "${page_file}" + read -r returned total < <(python3 - "${page_file}" "${collection}" <<'PY' + import json, sys + value=json.load(open(sys.argv[1], encoding="utf-8")); rows=value.get(sys.argv[2]); total=value.get("total_count") + if not isinstance(rows, list) or isinstance(total, bool) or not isinstance(total, int) or total < 0: + raise SystemExit("GitHub listing page has an invalid shape") + print(len(rows), total) + PY + ) + collected=$((collected + returned)) + ((collected <= total)) || { echo "GitHub listing grew or duplicated while paginating." >&2; return 1; } + if ((collected == total)); then break; fi + ((returned > 0)) || { echo "GitHub listing ended before total_count." >&2; return 1; } + page=$((page + 1)) + done + ((collected == total && page <= 1000)) || { echo "GitHub listing exceeded the pagination safety bound." >&2; return 1; } + python3 - "${destination}" "${collection}" "${pages}"/*.json <<'PY' + import json, pathlib, sys + destination=pathlib.Path(sys.argv[1]); field=sys.argv[2]; pages=[json.load(open(path, encoding="utf-8")) for path in sys.argv[3:]] + totals={page.get("total_count") for page in pages}; rows=[row for page in pages for row in page.get(field, [])] + ids=[row.get("id") for row in rows] + if len(totals) != 1 or totals != {len(rows)} or any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in ids) or len(ids) != len(set(ids)): + raise SystemExit("GitHub paginated listing is incomplete, changed, or duplicated") + destination.write_text(json.dumps({"total_count":len(rows),field:rows}, sort_keys=True, separators=(",", ":"))) + PY + find "${pages}" -mindepth 1 -delete + rmdir "${pages}" + } + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/runs/${POSTGRES_RUN_ID}" > "${work}/postgres-run.json" + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/workflows/deploy-brio-identity-db.yml" > "${work}/postgres-workflow.json" + fetch_complete_listing "/repos/Makepad-fr/postgres/actions/runs/${POSTGRES_RUN_ID}/artifacts?per_page=100" \ + artifacts "${work}/postgres-artifacts.json" + read -r deployment_artifact_id postgres_head_sha < <( + python3 scripts/verify-brio-release-evidence.py postgres-run \ + "${work}/postgres-run.json" "${work}/postgres-workflow.json" "${work}/postgres-artifacts.json" \ + "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" + ) + github_api --fail --silent --show-error --location "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/artifacts/${deployment_artifact_id}/zip" \ + > "${work}/postgres-evidence.zip" + python3 scripts/verify-brio-release-evidence.py postgres-evidence \ + "${work}/postgres-evidence.zip" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" + + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/git/ref/heads/main" > "${work}/keycloak-main.json" + keycloak_sha=$(python3 scripts/verify-brio-release-evidence.py keycloak-main "${work}/keycloak-main.json") + dispatch_started=$(date -u +%Y-%m-%dT%H:%M:%SZ) + python3 - "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" "${keycloak_sha}" \ + > "${work}/dispatch.json" <<'PY' + import json, sys + print(json.dumps({"ref":"main","inputs":{ + "postgres_deployment_run_id":sys.argv[1], + "postgres_deployment_run_attempt":sys.argv[2], + "postgres_deployment_head_sha":sys.argv[3], + "keycloak_release_sha":sys.argv[4], + }}, separators=(",", ":"))) + PY + code=$(github_api --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --request POST "${headers[@]}" --data-binary "@${work}/dispatch.json" \ + "${api}/repos/Makepad-fr/keycloak/actions/workflows/verify-brio-database.yml/dispatches") + [[ "${code}" == 204 ]] || { echo "Keycloak verifier dispatch returned HTTP ${code}." >&2; exit 1; } + + verifier_run_id= + for _ in $(seq 1 60); do + fetch_complete_listing "/repos/Makepad-fr/keycloak/actions/workflows/verify-brio-database.yml/runs?branch=main&event=workflow_dispatch&per_page=100" \ + workflow_runs "${work}/keycloak-runs.json" + verifier_run_id=$(python3 scripts/verify-brio-release-evidence.py verifier-run-select \ + "${work}/keycloak-runs.json" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${keycloak_sha}" "${dispatch_started}") + [[ -z "${verifier_run_id}" ]] || break + sleep 10 + done + [[ -n "${verifier_run_id}" ]] || { echo "The exact Keycloak verifier run was not found." >&2; exit 1; } + + verifier_status= + verifier_conclusion= + for _ in $(seq 1 180); do + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/actions/runs/${verifier_run_id}" > "${work}/keycloak-run.json" + read -r verifier_status verifier_conclusion < <(python3 - "${work}/keycloak-run.json" <<'PY' + import json, sys + run=json.load(open(sys.argv[1], encoding="utf-8")) + print(run.get("status") or "missing", run.get("conclusion") or "pending") + PY + ) + [[ "${verifier_status}" != completed ]] || break + sleep 10 + done + [[ "${verifier_status}" == completed && "${verifier_conclusion}" == success ]] || { + echo "Keycloak verifier did not complete successfully." >&2 + exit 1 + } + python3 scripts/verify-brio-release-evidence.py verifier-run \ + "${work}/keycloak-run.json" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${keycloak_sha}" "${verifier_run_id}" + fetch_complete_listing "/repos/Makepad-fr/keycloak/actions/runs/${verifier_run_id}/artifacts?per_page=100" \ + artifacts "${work}/keycloak-artifacts.json" + attestation_artifact_id=$(python3 scripts/verify-brio-release-evidence.py attestation-artifact \ + "${work}/keycloak-artifacts.json" "${verifier_run_id}" 1) + github_api --fail --silent --show-error --location "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/actions/artifacts/${attestation_artifact_id}/zip" \ + > "${work}/keycloak-attestation.zip" + python3 scripts/verify-brio-release-evidence.py attestation \ + "${work}/keycloak-attestation.zip" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" \ + "${verifier_run_id}" 1 "${keycloak_sha}" + unset release_token + { + echo "verifier_run_id=${verifier_run_id}" + echo "keycloak_sha=${keycloak_sha}" + echo "postgres_sha=${postgres_head_sha}" + } >> "${GITHUB_OUTPUT}" + + - name: Record validated external attestation + shell: bash + run: | + set -euo pipefail + { + echo '### Brio identity database release attested' + echo + echo "- PostgreSQL deployment: \`${{ inputs.postgres_deployment_run_id }}/${{ inputs.postgres_deployment_run_attempt }}\`" + echo "- PostgreSQL commit: \`${{ steps.verify.outputs.postgres_sha }}\`" + echo "- Keycloak verifier run: \`${{ steps.verify.outputs.verifier_run_id }}/1\`" + echo "- Keycloak commit: \`${{ steps.verify.outputs.keycloak_sha }}\`" + echo '- Artifact source: exact Keycloak verifier output (not synthesized by this workflow)' + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Remove release material + if: always() + shell: bash + run: | + set -euo pipefail + work="${RUNNER_TEMP}/brio-db-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + case "${work}" in "${RUNNER_TEMP}"/brio-db-release-*) ;; *) exit 1 ;; esac + if [[ -L "${work}" ]]; then + echo "Refusing to remove a symlinked release workspace." >&2 + exit 1 + elif [[ -d "${work}" ]]; then + find "${work}" -mindepth 1 -delete + rmdir "${work}" + elif [[ -e "${work}" ]]; then + echo "Release workspace has an unexpected type." >&2 + exit 1 + fi diff --git a/scripts/test-brio-release-evidence.sh b/scripts/test-brio-release-evidence.sh new file mode 100755 index 0000000..126740a --- /dev/null +++ b/scripts/test-brio-release-evidence.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +validator="${script_dir}/verify-brio-release-evidence.py" +fixture=$(mktemp -d /tmp/postgres-brio-release-evidence-XXXXXX) +cleanup() { + [[ "${fixture}" =~ ^/tmp/postgres-brio-release-evidence-[A-Za-z0-9]+$ ]] || return 1 + find "${fixture}" -depth -delete +} +trap cleanup EXIT + +pg_run=123 +pg_attempt=2 +pg_sha=1111111111111111111111111111111111111111 +kc_run=456 +kc_attempt=1 +kc_sha=2222222222222222222222222222222222222222 + +python3 - "${fixture}" <<'PY' +import json +import pathlib +import stat +import sys +import zipfile + +root = pathlib.Path(sys.argv[1]) +pg_sha = "1" * 40 +kc_sha = "2" * 40 + +def dump(name, value): + (root / name).write_text(json.dumps(value, separators=(",", ":")), encoding="utf-8") + +def archive(name, entry, value, *, mode=None, extra=None): + with zipfile.ZipFile(root / name, "w", compression=zipfile.ZIP_DEFLATED) as target: + info = zipfile.ZipInfo(entry) + if mode is not None: + info.create_system = 3 + info.external_attr = mode << 16 + target.writestr(info, json.dumps(value, separators=(",", ":"))) + if extra: + target.writestr(extra, "unexpected") + +pg_evidence = { + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": 123, + "postgres_run_attempt": 2, + "postgres_head_sha": pg_sha, + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", +} +attestation = { + "schema": "makepad.brio-db-path-attestation.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": 123, + "postgres_run_attempt": 2, + "postgres_head_sha": pg_sha, + "keycloak_repository": "Makepad-fr/keycloak", + "keycloak_workflow": ".github/workflows/verify-brio-database.yml", + "keycloak_verifier_run_id": 456, + "keycloak_verifier_run_attempt": 1, + "keycloak_release_sha": kc_sha, + "probe": "brio-db-path-ok", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", +} +dump("pg-run.json", { + "id": 123, "run_attempt": 2, "name": "Deploy Brio Identity Database", + "path": ".github/workflows/deploy-brio-identity-db.yml", "event": "workflow_dispatch", + "head_branch": "main", "head_sha": pg_sha, "status": "completed", "conclusion": "success", + "repository": {"full_name": "Makepad-fr/postgres"}, +}) +dump("pg-workflow.json", { + "name": "Deploy Brio Identity Database", + "path": ".github/workflows/deploy-brio-identity-db.yml", "state": "active", +}) +dump("pg-artifacts.json", {"total_count": 1, "artifacts": [{ + "id": 789, "name": "brio-db-deployment-evidence-123-2", "expired": False, "size_in_bytes": 1024, +}]}) +archive("pg.zip", "brio-db-deployment-evidence.json", pg_evidence) +dump("kc-main.json", {"object": {"sha": kc_sha}}) +kc_run_value = { + "id": 456, "run_attempt": 1, "name": "Verify Brio Identity Database Path", + "display_title": f"Verify Brio DB path for PostgreSQL run 123/2 at Keycloak {kc_sha}", + "path": ".github/workflows/verify-brio-database.yml", "event": "workflow_dispatch", + "head_branch": "main", "head_sha": kc_sha, "status": "completed", "conclusion": "success", + "created_at": "2026-09-05T10:00:05Z", "repository": {"full_name": "Makepad-fr/keycloak"}, +} +dump("kc-run.json", kc_run_value) +dump("kc-runs.json", {"total_count": 1, "workflow_runs": [kc_run_value]}) +dump("kc-artifacts.json", {"total_count": 1, "artifacts": [{ + "id": 987, "name": "brio-db-path-attestation-456-1", "expired": False, "size_in_bytes": 1024, +}]}) +archive("kc.zip", "brio-db-path-attestation.json", attestation) + +bad = dict(pg_evidence); bad["schema"] = "wrong"; archive("wrong-schema.zip", "brio-db-deployment-evidence.json", bad) +bad = dict(pg_evidence); bad["extra"] = True; archive("extra-field.zip", "brio-db-deployment-evidence.json", bad) +archive("extra-entry.zip", "brio-db-deployment-evidence.json", pg_evidence, extra="extra") +archive("symlink-entry.zip", "brio-db-deployment-evidence.json", pg_evidence, mode=stat.S_IFLNK | 0o777) +with zipfile.ZipFile(root / "oversized-entry.zip", "w", compression=zipfile.ZIP_STORED) as target: + target.writestr("brio-db-deployment-evidence.json", b"x" * 65537) +bad = dict(attestation); bad["postgres_run_id"] = 999; archive("wrong-binding.zip", "brio-db-path-attestation.json", bad) +PY + +expect_failure() { + local label=$1 + shift + if "$@" >"${fixture}/${label}.out" 2>&1; then + echo "Expected ${label} to fail closed." >&2 + exit 1 + fi +} + +[[ $(python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/pg-artifacts.json" "${pg_run}" "${pg_attempt}") == "789 ${pg_sha}" ]] +python3 "${validator}" postgres-evidence "${fixture}/pg.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +[[ $(python3 "${validator}" keycloak-main "${fixture}/kc-main.json") == "${kc_sha}" ]] +[[ $(python3 "${validator}" verifier-run-select "${fixture}/kc-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z) == "${kc_run}" ]] +python3 "${validator}" verifier-run "${fixture}/kc-run.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" "${kc_run}" +[[ $(python3 "${validator}" attestation-artifact "${fixture}/kc-artifacts.json" "${kc_run}" "${kc_attempt}") == 987 ]] +python3 "${validator}" attestation "${fixture}/kc.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" "${kc_run}" "${kc_attempt}" "${kc_sha}" + +python3 - "${fixture}/pg-run.json" "${fixture}/in-progress.json" <<'PY' +import json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")); value["status"]="in_progress"; value["conclusion"]=None +json.dump(value, open(sys.argv[2], "w", encoding="utf-8")) +PY +expect_failure in-progress python3 "${validator}" postgres-run "${fixture}/in-progress.json" "${fixture}/pg-workflow.json" "${fixture}/pg-artifacts.json" "${pg_run}" "${pg_attempt}" + +python3 - "${fixture}/pg-artifacts.json" "${fixture}/duplicate-artifacts.json" "${fixture}/oversized-artifact.json" "${fixture}/truncated-artifacts.json" <<'PY' +import copy, json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")) +duplicate=copy.deepcopy(value); duplicate["artifacts"].append(copy.deepcopy(duplicate["artifacts"][0])); duplicate["total_count"]=2; json.dump(duplicate, open(sys.argv[2], "w", encoding="utf-8")) +oversized=copy.deepcopy(value); oversized["artifacts"][0]["size_in_bytes"]=131073; json.dump(oversized, open(sys.argv[3], "w", encoding="utf-8")) +truncated=copy.deepcopy(value); truncated["total_count"]=2; json.dump(truncated, open(sys.argv[4], "w", encoding="utf-8")) +PY +expect_failure duplicate-artifact python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/duplicate-artifacts.json" "${pg_run}" "${pg_attempt}" +expect_failure oversized-artifact python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/oversized-artifact.json" "${pg_run}" "${pg_attempt}" +expect_failure truncated-artifact-page python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/truncated-artifacts.json" "${pg_run}" "${pg_attempt}" +expect_failure wrong-schema python3 "${validator}" postgres-evidence "${fixture}/wrong-schema.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure extra-field python3 "${validator}" postgres-evidence "${fixture}/extra-field.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure extra-entry python3 "${validator}" postgres-evidence "${fixture}/extra-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure symlink-entry python3 "${validator}" postgres-evidence "${fixture}/symlink-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure oversized-entry python3 "${validator}" postgres-evidence "${fixture}/oversized-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +ln -s "${fixture}/pg.zip" "${fixture}/symlink-archive.zip" +expect_failure symlink-archive python3 "${validator}" postgres-evidence "${fixture}/symlink-archive.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure wrong-attestation-binding python3 "${validator}" attestation "${fixture}/wrong-binding.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" "${kc_run}" "${kc_attempt}" "${kc_sha}" + +python3 - "${fixture}/kc-runs.json" "${fixture}/duplicate-runs.json" <<'PY' +import copy, json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")); value["workflow_runs"].append(copy.deepcopy(value["workflow_runs"][0])); value["total_count"] = 2 +json.dump(value, open(sys.argv[2], "w", encoding="utf-8")) +PY +expect_failure duplicate-verifier-run python3 "${validator}" verifier-run-select "${fixture}/duplicate-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z + +python3 - "${fixture}/kc-runs.json" "${fixture}/truncated-runs.json" "${fixture}/kc-artifacts.json" "${fixture}/truncated-kc-artifacts.json" <<'PY' +import json, sys +runs=json.load(open(sys.argv[1], encoding="utf-8")); runs["total_count"]=2; json.dump(runs, open(sys.argv[2], "w", encoding="utf-8")) +artifacts=json.load(open(sys.argv[3], encoding="utf-8")); artifacts["total_count"]=2; json.dump(artifacts, open(sys.argv[4], "w", encoding="utf-8")) +PY +expect_failure truncated-verifier-page python3 "${validator}" verifier-run-select "${fixture}/truncated-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z +expect_failure truncated-keycloak-artifact-page python3 "${validator}" attestation-artifact "${fixture}/truncated-kc-artifacts.json" "${kc_run}" "${kc_attempt}" + +echo "Brio two-phase release evidence validation tests passed." diff --git a/scripts/verify-brio-release-evidence.py b/scripts/verify-brio-release-evidence.py new file mode 100755 index 0000000..867bc3c --- /dev/null +++ b/scripts/verify-brio-release-evidence.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Fail-closed validation for the two-phase Brio database release contract.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +import stat +import sys +import zipfile +from pathlib import Path + + +POSTGRES_REPOSITORY = "Makepad-fr/postgres" +POSTGRES_WORKFLOW = ".github/workflows/deploy-brio-identity-db.yml" +KEYCLOAK_REPOSITORY = "Makepad-fr/keycloak" +KEYCLOAK_WORKFLOW = ".github/workflows/verify-brio-database.yml" +SHA = re.compile(r"[0-9a-f]{40}") +POSITIVE = re.compile(r"[1-9][0-9]*") + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def load(path: str) -> object: + with Path(path).open(encoding="utf-8") as source: + return json.load(source) + + +def positive(value: str, label: str) -> int: + if not POSITIVE.fullmatch(value): + fail(f"{label} must be a positive integer") + return int(value) + + +def full_sha(value: str, label: str) -> str: + if not SHA.fullmatch(value): + fail(f"{label} must be a lowercase full commit SHA") + return value + + +def exact_object(value: object, expected: dict[str, object], label: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != set(expected): + fail(f"{label} must have exactly the canonical fields") + for key, expected_value in expected.items(): + if value.get(key) != expected_value: + fail(f"{label} {key} mismatch") + return value + + +def complete_listing(value: object, collection: str, label: str) -> list[object]: + if not isinstance(value, dict): + fail(f"{label} must be a JSON object") + rows = value.get(collection) + total = value.get("total_count") + if ( + not isinstance(rows, list) + or isinstance(total, bool) + or not isinstance(total, int) + or total < 0 + or total != len(rows) + ): + fail(f"{label} is truncated or has an invalid total_count") + identifiers = [row.get("id") for row in rows if isinstance(row, dict)] + if len(identifiers) != len(rows) or any(isinstance(identifier, bool) or not isinstance(identifier, int) or identifier <= 0 for identifier in identifiers) or len(set(identifiers)) != len(identifiers): + fail(f"{label} has missing or duplicate IDs") + return rows + + +def safe_single_json(archive_path: str, expected_name: str) -> object: + path = Path(archive_path) + if not path.is_file() or path.is_symlink() or not 1 <= path.stat().st_size <= 131072: + fail("Evidence archive is missing, symlinked, empty, or oversized") + try: + with zipfile.ZipFile(path) as archive: + entries = archive.infolist() + if len(entries) != 1 or entries[0].filename != expected_name: + fail("Evidence archive must contain exactly its canonical JSON file") + entry = entries[0] + if stat.S_ISLNK(entry.external_attr >> 16) or not 1 <= entry.file_size <= 65536: + fail("Evidence archive entry is symlinked, empty, or oversized") + if entry.compress_size > 131072: + fail("Evidence archive entry has an unsafe compressed size") + return json.loads(archive.read(entry)) + except (zipfile.BadZipFile, UnicodeDecodeError, json.JSONDecodeError) as error: + fail(f"Evidence archive is invalid: {error}") + + +def postgres_run(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("postgres-run expects run, workflow, artifacts, run ID, and attempt") + run = load(arguments[0]) + workflow = load(arguments[1]) + artifacts = load(arguments[2]) + run_id = positive(arguments[3], "PostgreSQL run ID") + attempt = positive(arguments[4], "PostgreSQL run attempt") + if not isinstance(run, dict) or not isinstance(workflow, dict) or not isinstance(artifacts, dict): + fail("GitHub metadata must be JSON objects") + head_sha = full_sha(str(run.get("head_sha", "")), "PostgreSQL head SHA") + expected = { + "id": run_id, + "run_attempt": attempt, + "name": "Deploy Brio Identity Database", + "path": POSTGRES_WORKFLOW, + "event": "workflow_dispatch", + "head_branch": "main", + "head_sha": head_sha, + "status": "completed", + "conclusion": "success", + } + for key, expected_value in expected.items(): + if run.get(key) != expected_value: + fail(f"PostgreSQL deployment run {key} mismatch") + if run.get("repository", {}).get("full_name") != POSTGRES_REPOSITORY: + fail("PostgreSQL deployment repository mismatch") + if (workflow.get("name"), workflow.get("path"), workflow.get("state")) != ( + "Deploy Brio Identity Database", + POSTGRES_WORKFLOW, + "active", + ): + fail("PostgreSQL deployment workflow identity mismatch") + expected_name = f"brio-db-deployment-evidence-{run_id}-{attempt}" + artifact_rows = complete_listing(artifacts, "artifacts", "PostgreSQL artifact listing") + matches = [ + artifact + for artifact in artifact_rows + if artifact.get("name") == expected_name and not artifact.get("expired") + ] + if len(matches) != 1: + fail("Expected exactly one unexpired PostgreSQL deployment evidence artifact") + artifact = matches[0] + artifact_id = artifact.get("id") + size = artifact.get("size_in_bytes") + if not isinstance(artifact_id, int) or artifact_id <= 0: + fail("Invalid PostgreSQL deployment artifact ID") + if not isinstance(size, int) or not 1 <= size <= 131072: + fail("Unsafe PostgreSQL deployment artifact size") + print(f"{artifact_id} {head_sha}") + + +def postgres_evidence(arguments: list[str]) -> None: + if len(arguments) != 4: + fail("postgres-evidence expects archive, run ID, attempt, and head SHA") + run_id = positive(arguments[1], "PostgreSQL run ID") + attempt = positive(arguments[2], "PostgreSQL run attempt") + head_sha = full_sha(arguments[3], "PostgreSQL head SHA") + evidence = safe_single_json(arguments[0], "brio-db-deployment-evidence.json") + exact_object( + evidence, + { + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": POSTGRES_REPOSITORY, + "postgres_workflow": POSTGRES_WORKFLOW, + "postgres_run_id": run_id, + "postgres_run_attempt": attempt, + "postgres_head_sha": head_sha, + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, + "PostgreSQL deployment evidence", + ) + + +def keycloak_main(arguments: list[str]) -> None: + if len(arguments) != 1: + fail("keycloak-main expects the ref response") + payload = load(arguments[0]) + if not isinstance(payload, dict): + fail("Keycloak ref response must be an object") + print(full_sha(str(payload.get("object", {}).get("sha", "")), "Keycloak main SHA")) + + +def verifier_run_select(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("verifier-run-select expects runs, PostgreSQL run/attempt, Keycloak SHA, and dispatch time") + payload = load(arguments[0]) + run_id = positive(arguments[1], "PostgreSQL run ID") + attempt = positive(arguments[2], "PostgreSQL run attempt") + keycloak_sha = full_sha(arguments[3], "Keycloak release SHA") + try: + started = dt.datetime.fromisoformat(arguments[4].replace("Z", "+00:00")) + except ValueError as error: + fail(f"Invalid dispatch timestamp: {error}") + expected_title = f"Verify Brio DB path for PostgreSQL run {run_id}/{attempt} at Keycloak {keycloak_sha}" + matches: list[int] = [] + if not isinstance(payload, dict): + fail("Verifier run listing must be an object") + for run in complete_listing(payload, "workflow_runs", "Keycloak verifier run listing"): + try: + created = dt.datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) + except (KeyError, ValueError): + continue + if ( + run.get("display_title") == expected_title + and run.get("name") == "Verify Brio Identity Database Path" + and run.get("path") == KEYCLOAK_WORKFLOW + and run.get("repository", {}).get("full_name") == KEYCLOAK_REPOSITORY + and run.get("head_branch") == "main" + and run.get("head_sha") == keycloak_sha + and run.get("event") == "workflow_dispatch" + and created >= started - dt.timedelta(minutes=2) + ): + matches.append(run.get("id")) + if len(matches) > 1: + fail("Ambiguous Keycloak verifier runs") + if len(matches) == 1: + if not isinstance(matches[0], int) or matches[0] <= 0: + fail("Invalid Keycloak verifier run ID") + print(matches[0]) + + +def verifier_run(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("verifier-run expects run, PostgreSQL run/attempt, Keycloak SHA, and verifier run ID") + run = load(arguments[0]) + postgres_run_id = positive(arguments[1], "PostgreSQL run ID") + postgres_attempt = positive(arguments[2], "PostgreSQL run attempt") + keycloak_sha = full_sha(arguments[3], "Keycloak release SHA") + verifier_id = positive(arguments[4], "Keycloak verifier run ID") + expected = { + "id": verifier_id, + "run_attempt": 1, + "name": "Verify Brio Identity Database Path", + "display_title": f"Verify Brio DB path for PostgreSQL run {postgres_run_id}/{postgres_attempt} at Keycloak {keycloak_sha}", + "path": KEYCLOAK_WORKFLOW, + "event": "workflow_dispatch", + "head_branch": "main", + "head_sha": keycloak_sha, + "status": "completed", + "conclusion": "success", + } + if not isinstance(run, dict): + fail("Keycloak verifier run must be an object") + for key, expected_value in expected.items(): + if run.get(key) != expected_value: + fail(f"Keycloak verifier run {key} mismatch") + if run.get("repository", {}).get("full_name") != KEYCLOAK_REPOSITORY: + fail("Keycloak verifier repository mismatch") + + +def attestation_artifact(arguments: list[str]) -> None: + if len(arguments) != 3: + fail("attestation-artifact expects artifacts, verifier run ID, and attempt") + payload = load(arguments[0]) + run_id = positive(arguments[1], "Keycloak verifier run ID") + attempt = positive(arguments[2], "Keycloak verifier run attempt") + expected_name = f"brio-db-path-attestation-{run_id}-{attempt}" + artifacts = complete_listing(payload, "artifacts", "Keycloak attestation artifact listing") + matches = [ + artifact + for artifact in artifacts + if artifact.get("name") == expected_name and not artifact.get("expired") + ] + if len(matches) != 1: + fail("Expected exactly one unexpired Keycloak path attestation artifact") + artifact = matches[0] + artifact_id, size = artifact.get("id"), artifact.get("size_in_bytes") + if not isinstance(artifact_id, int) or artifact_id <= 0: + fail("Invalid Keycloak attestation artifact ID") + if not isinstance(size, int) or not 1 <= size <= 131072: + fail("Unsafe Keycloak attestation artifact size") + print(artifact_id) + + +def attestation(arguments: list[str]) -> None: + if len(arguments) != 7: + fail("attestation expects archive, PostgreSQL run/attempt/SHA, Keycloak run/attempt/SHA") + postgres_run = positive(arguments[1], "PostgreSQL run ID") + postgres_attempt = positive(arguments[2], "PostgreSQL run attempt") + postgres_sha = full_sha(arguments[3], "PostgreSQL head SHA") + keycloak_run = positive(arguments[4], "Keycloak verifier run ID") + keycloak_attempt = positive(arguments[5], "Keycloak verifier run attempt") + keycloak_sha = full_sha(arguments[6], "Keycloak release SHA") + evidence = safe_single_json(arguments[0], "brio-db-path-attestation.json") + exact_object( + evidence, + { + "schema": "makepad.brio-db-path-attestation.v1", + "postgres_repository": POSTGRES_REPOSITORY, + "postgres_workflow": POSTGRES_WORKFLOW, + "postgres_run_id": postgres_run, + "postgres_run_attempt": postgres_attempt, + "postgres_head_sha": postgres_sha, + "keycloak_repository": KEYCLOAK_REPOSITORY, + "keycloak_workflow": KEYCLOAK_WORKFLOW, + "keycloak_verifier_run_id": keycloak_run, + "keycloak_verifier_run_attempt": keycloak_attempt, + "keycloak_release_sha": keycloak_sha, + "probe": "brio-db-path-ok", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, + "Keycloak database-path attestation", + ) + + +COMMANDS = { + "postgres-run": postgres_run, + "postgres-evidence": postgres_evidence, + "keycloak-main": keycloak_main, + "verifier-run-select": verifier_run_select, + "verifier-run": verifier_run, + "attestation-artifact": attestation_artifact, + "attestation": attestation, +} + +if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: + fail("Usage: verify-brio-release-evidence.py [arguments ...]") +COMMANDS[sys.argv[1]](sys.argv[2:]) From 6940040be09c31bad4d6cc43edbff9784722c07e Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 04:27:35 +0200 Subject: [PATCH 10/20] test(keycloak): verify six-realm restore cohort --- .../verify-keycloak-cohort-restores.yml | 267 ++++++++++++++++++ .../makepad-keycloak-cohort-cleaner.service | 14 + .../makepad-keycloak-cohort-cleaner.timer | 12 + scripts/capture-keycloak-cohort-backups.sh | 90 ++++++ scripts/clean-keycloak-cohort-resources.sh | 46 +++ .../keycloak-cohort-cleaner-fixture.sh | 81 ++++++ .../keycloak-cohort-dispatch-fixture.sh | 49 ++++ .../install-keycloak-cohort-capture-host.sh | 43 +++ scripts/install-keycloak-cohort-cleaner.sh | 24 ++ scripts/keycloak-cohort-capture-dispatch.sh | 64 +++++ scripts/restore-keycloak-cohort-backups.sh | 231 +++++++++++++++ scripts/test-keycloak-cohort-evidence.sh | 129 +++++++++ scripts/test-keycloak-cohort-hardening.sh | 51 ++++ scripts/verify-keycloak-cohort-evidence.py | 176 ++++++++++++ 14 files changed, 1277 insertions(+) create mode 100644 .github/workflows/verify-keycloak-cohort-restores.yml create mode 100644 host/systemd/makepad-keycloak-cohort-cleaner.service create mode 100644 host/systemd/makepad-keycloak-cohort-cleaner.timer create mode 100755 scripts/capture-keycloak-cohort-backups.sh create mode 100755 scripts/clean-keycloak-cohort-resources.sh create mode 100755 scripts/fixtures/keycloak-cohort-cleaner-fixture.sh create mode 100755 scripts/fixtures/keycloak-cohort-dispatch-fixture.sh create mode 100755 scripts/install-keycloak-cohort-capture-host.sh create mode 100755 scripts/install-keycloak-cohort-cleaner.sh create mode 100755 scripts/keycloak-cohort-capture-dispatch.sh create mode 100755 scripts/restore-keycloak-cohort-backups.sh create mode 100755 scripts/test-keycloak-cohort-evidence.sh create mode 100755 scripts/test-keycloak-cohort-hardening.sh create mode 100755 scripts/verify-keycloak-cohort-evidence.py diff --git a/.github/workflows/verify-keycloak-cohort-restores.yml b/.github/workflows/verify-keycloak-cohort-restores.yml new file mode 100644 index 0000000..1881a1a --- /dev/null +++ b/.github/workflows/verify-keycloak-cohort-restores.yml @@ -0,0 +1,267 @@ +name: Verify Keycloak Cohort Restore Compatibility + +on: + workflow_dispatch: + inputs: + keycloak_release_sha: + description: Exact protected Keycloak main SHA whose pinned 26.7.3 runtime must start all six restored databases + required: true + type: string + +permissions: + contents: read + +concurrency: + group: postgres-keycloak-cohort-restore + cancel-in-progress: false + +jobs: + verify: + name: restore-six-databases-and-start-keycloak + runs-on: + group: Postgres Release + labels: [self-hosted, linux, x64, makepad, makepad-postgres-release] + environment: keycloak-cohort-restore + timeout-minutes: 120 + steps: + - name: Enforce protected immutable release inputs + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + KEYCLOAK_SOURCE_TOKEN: ${{ secrets.KEYCLOAK_COHORT_SOURCE_TOKEN }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == Makepad-fr/postgres && "${GITHUB_REF}" == refs/heads/main ]] + [[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ && "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + [[ "${GITHUB_SHA}" =~ ^[a-f0-9]{40}$ && "${KEYCLOAK_RELEASE_SHA}" =~ ^[a-f0-9]{40}$ ]] + : "${KEYCLOAK_SOURCE_TOKEN:?set the dedicated Keycloak Contents-read token}" + observed_main=$(GH_TOKEN="${KEYCLOAK_SOURCE_TOKEN}" gh api \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + repos/Makepad-fr/keycloak/git/ref/heads/main --jq .object.sha) + [[ "${observed_main}" == "${KEYCLOAK_RELEASE_SHA}" ]] || { + echo "The requested Keycloak release is not the exact current protected main SHA." >&2 + exit 1 + } + + - name: Check out protected PostgreSQL producer + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + ref: ${{ github.sha }} + fetch-depth: 1 + + - name: Check out exact Keycloak release source + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + repository: Makepad-fr/keycloak + token: ${{ secrets.KEYCLOAK_COHORT_SOURCE_TOKEN }} + persist-credentials: false + ref: ${{ inputs.keycloak_release_sha }} + path: _keycloak + fetch-depth: 1 + + - name: Verify exact protected sources and runtime pin + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + [[ "$(git -C _keycloak rev-parse HEAD)" == "${KEYCLOAK_RELEASE_SHA}" ]] + grep -Fxq 'KEYCLOAK_UPSTREAM_VERSION=26.7.3' _keycloak/envs/production/.env.keycloak + grep -Fxq 'KEYCLOAK_IMAGE=dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' \ + _keycloak/envs/production/.env.keycloak + + - name: Ensure interrupted cohort material expires on the release host + shell: bash + run: | + set -euo pipefail + ./scripts/ensure-brio-tmp-cleaner.sh envs/canary/.env.db + systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer + systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer + + - name: Configure isolated SSH and registry state + shell: bash + env: + SSH_PRIVATE_KEY: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS }} + DHI_USERNAME: ${{ secrets.DHI_REGISTRY_USERNAME }} + DHI_PASSWORD: ${{ secrets.DHI_REGISTRY_PASSWORD }} + run: | + set -euo pipefail + : "${SSH_PRIVATE_KEY:?}" "${SSH_KNOWN_HOSTS:?}" "${DHI_USERNAME:?}" "${DHI_PASSWORD:?}" + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${job_root}" && ! -L "${job_root}" ]] + install -d -m 0700 "${job_root}/ssh" "${job_root}/backups" "${job_root}/docker" + umask 077 + printf '%s\n' "${SSH_PRIVATE_KEY}" >"${job_root}/ssh/id_ed25519" + printf '%s\n' "${SSH_KNOWN_HOSTS}" >"${job_root}/ssh/known_hosts" + chmod 0600 "${job_root}/ssh/"* + printf '%s' "${DHI_PASSWORD}" | docker --config "${job_root}/docker" login dhi.io --username "${DHI_USERNAME}" --password-stdin >/dev/null + + - name: Capture exact live cohort backups + shell: bash + env: + REMOTE_HOST: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_HOST }} + REMOTE_PORT: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PORT }} + REMOTE_USER: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_USER }} + run: | + set -euo pipefail + : "${REMOTE_HOST:?}" "${REMOTE_USER:?}" + [[ "${REMOTE_USER}" != root ]] + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${job_root}/ssh/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${job_root}/ssh/id_ed25519" -p "${remote_port}") + target="${REMOTE_USER}@${REMOTE_HOST}" + helper_digest=$(sha256sum scripts/capture-keycloak-cohort-backups.sh | cut -d' ' -f1) + cleaner_digest=$(sha256sum scripts/clean-keycloak-cohort-resources.sh | cut -d' ' -f1) + # The reviewed digests are intentionally expanded by this client. + # shellcheck disable=SC2029 + [[ $(ssh "${ssh_opts[@]}" "${target}" "probe ${helper_digest} ${cleaner_digest}") == cohort-capture-contract-ok ]] + # The SSH key is restricted by an operator-installed forced command. + # Only exact probe/capture/fetch/cleanup verbs and this reviewed helper + # digest can reach Docker on the database host. + # Run identity and digest are intentionally client-expanded forced-command arguments. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "capture ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${helper_digest} ${cleaner_digest}" + for database in keycloak_betacrew keycloak_catwlk keycloak_makepad keycloak_runtrace keycloak_vestiaire keycloak_vif; do + partial="${job_root}/backups/.${database}.dump.partial" + # The forced-command fetch tuple is intentionally client-expanded. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" \ + "fetch ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${database}.dump ${helper_digest} ${cleaner_digest}" >"${partial}" + [[ -s "${partial}" && ! -L "${partial}" ]] + mv -T "${partial}" "${job_root}/backups/${database}.dump" + done + [[ $(find "${job_root}/backups" -mindepth 1 -maxdepth 1 -type f -name '*.dump' | wc -l) -eq 6 ]] + + - name: Build exact checked-out Catwlk provider runtime + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + base='dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' + tag="makepad/keycloak-catwlk-cohort:${KEYCLOAK_RELEASE_SHA}" + manifest_runtime=$(python3 - <<'PY' + import json + value=json.load(open("_keycloak/instances/catwlk/manifest.json", encoding="utf-8"))["runtime"] + assert value == {"source_dir":"providers/catwlk-email-router","dockerfile":"providers/catwlk-email-router/Dockerfile","image":"makepad/keycloak-catwlk"} + print(value["dockerfile"]) + PY + ) + DOCKER_BUILDKIT=1 docker build \ + --build-arg "KEYCLOAK_BASE_IMAGE=${base}" \ + --label "org.makepad.keycloak.base-image=${base}" \ + --label "org.makepad.keycloak.source-sha=${KEYCLOAK_RELEASE_SHA}" \ + --label org.makepad.keycloak.runtime=catwlk-custom-provider \ + --file "_keycloak/${manifest_runtime}" --tag "${tag}" _keycloak >/dev/null + image_id=$(docker image inspect "${tag}" --format '{{.Id}}') + [[ "${image_id}" =~ ^sha256:[a-f0-9]{64}$ ]] + docker run --rm --network none --read-only --entrypoint sh "${tag}" -euc \ + 'test -s /opt/keycloak/providers/catwlk-keycloak-email-router.jar' + printf '%s\n' "${tag}" > "/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/catwlk-image-tag" + + - name: Restore every database and start its reviewed Keycloak runtime + shell: bash + run: | + set -euo pipefail + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + export DOCKER_CONFIG="${job_root}/docker" + docker pull 'dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' >/dev/null + docker pull 'postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' >/dev/null + catwlk_image=$(<"${job_root}/catwlk-image-tag") + scripts/restore-keycloak-cohort-backups.sh "${job_root}/backups" "${job_root}/results" _keycloak "${catwlk_image}" + + - name: Build and validate canonical cohort evidence + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + evidence_dir="${job_root}/evidence" + install -d -m 0700 "${evidence_dir}" + python3 - "${job_root}/results/instances.json" "${evidence_dir}/keycloak-cohort-restore-evidence.json" \ + "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT}" "${GITHUB_SHA}" "${KEYCLOAK_RELEASE_SHA}" <<'PY' + import json + import pathlib + import sys + + results = json.loads(pathlib.Path(sys.argv[1]).read_text()) + payload = { + "schema": "makepad.keycloak-cohort-restore-evidence.v2", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/verify-keycloak-cohort-restores.yml", + "postgres_run_id": int(sys.argv[3]), + "postgres_run_attempt": int(sys.argv[4]), + "postgres_head_sha": sys.argv[5], + "postgres_ref": "refs/heads/main", + "keycloak_release_sha": sys.argv[6], + "keycloak_base_image": "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "catwlk_runtime_image_id": results["catwlk_runtime_image_id"], + "fingerprint_schema": "makepad.keycloak-config-fingerprint.v2", + "keycloak_upstream_version": "26.7.3", + "result": "restored-databases-compatible", + "instances": results["instances"], + } + pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + PY + chmod 0600 "${evidence_dir}/keycloak-cohort-restore-evidence.json" + python3 scripts/verify-keycloak-cohort-evidence.py \ + "${evidence_dir}/keycloak-cohort-restore-evidence.json" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --head-sha "${GITHUB_SHA}" --keycloak-release-sha "${KEYCLOAK_RELEASE_SHA}" + [[ $(find "${evidence_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n') == keycloak-cohort-restore-evidence.json ]] + + - name: Publish immutable six-database compatibility evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/postgres-keycloak-cohort-${{ github.run_id }}-${{ github.run_attempt }}/evidence/keycloak-cohort-restore-evidence.json + if-no-files-found: error + retention-days: 35 + + - name: Remove remote and local cohort material + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_HOST }} + REMOTE_PORT: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PORT }} + REMOTE_USER: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_USER }} + run: | + set -euo pipefail + cleanup_status=0 + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -r "${job_root}/ssh/id_ed25519" && -r "${job_root}/ssh/known_hosts" && -n "${REMOTE_HOST}" && -n "${REMOTE_USER}" ]]; then + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${job_root}/ssh/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${job_root}/ssh/id_ed25519" -p "${remote_port}") + if [[ -f scripts/capture-keycloak-cohort-backups.sh && -f scripts/clean-keycloak-cohort-resources.sh ]]; then + helper_digest=$(sha256sum scripts/capture-keycloak-cohort-backups.sh | cut -d' ' -f1) + cleaner_digest=$(sha256sum scripts/clean-keycloak-cohort-resources.sh | cut -d' ' -f1) + # The forced-command cleanup tuple is intentionally client-expanded. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" \ + "cleanup ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${helper_digest} ${cleaner_digest}" || cleanup_status=1 + else + echo "Reviewed capture/cleaner sources are unavailable for constrained remote cleanup; the host TTL remains authoritative." >&2 + cleanup_status=1 + fi + fi + if [[ -f "${job_root}/catwlk-image-tag" && ! -L "${job_root}/catwlk-image-tag" ]]; then + catwlk_image=$(<"${job_root}/catwlk-image-tag") + if [[ "${catwlk_image}" == "makepad/keycloak-catwlk-cohort:${{ inputs.keycloak_release_sha }}" ]]; then + docker image rm "${catwlk_image}" >/dev/null 2>&1 || cleanup_status=1 + else + echo "Refusing to remove an unexpected Catwlk image tag." >&2 + cleanup_status=1 + fi + fi + if [[ -d "${job_root}" && ! -L "${job_root}" ]]; then + find "${job_root}" -mindepth 1 -delete || cleanup_status=1 + rmdir "${job_root}" || cleanup_status=1 + elif [[ -e "${job_root}" || -L "${job_root}" ]]; then + echo "Refusing cleanup of an unsafe local cohort path." >&2 + cleanup_status=1 + fi + exit "${cleanup_status}" diff --git a/host/systemd/makepad-keycloak-cohort-cleaner.service b/host/systemd/makepad-keycloak-cohort-cleaner.service new file mode 100644 index 0000000..2dd19e7 --- /dev/null +++ b/host/systemd/makepad-keycloak-cohort-cleaner.service @@ -0,0 +1,14 @@ +[Unit] +Description=Remove expired Keycloak cohort restore material and Docker resources +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +User=root +Group=root +ExecStart=/usr/local/libexec/makepad/clean-keycloak-cohort-resources +NoNewPrivileges=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/tmp /var/run/docker.sock diff --git a/host/systemd/makepad-keycloak-cohort-cleaner.timer b/host/systemd/makepad-keycloak-cohort-cleaner.timer new file mode 100644 index 0000000..44ef37a --- /dev/null +++ b/host/systemd/makepad-keycloak-cohort-cleaner.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Periodically reconcile Keycloak cohort restore resources + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min +RandomizedDelaySec=2min +Persistent=true +Unit=makepad-keycloak-cohort-cleaner.service + +[Install] +WantedBy=timers.target diff --git a/scripts/capture-keycloak-cohort-backups.sh b/scripts/capture-keycloak-cohort-backups.sh new file mode 100755 index 0000000..7803899 --- /dev/null +++ b/scripts/capture-keycloak-cohort-backups.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +if (($# != 1)); then + echo "usage: capture-keycloak-cohort-backups.sh /tmp/postgres-keycloak-cohort--" >&2 + exit 2 +fi + +output_dir=$1 +run_id=${COHORT_CAPTURE_RUN_ID:-} +run_attempt=${COHORT_CAPTURE_RUN_ATTEMPT:-} +postgres_image='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +container=postgres-postgres-1 +[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ ]] || { + echo "Exact positive workflow run and attempt are required." >&2 + exit 2 +} +[[ "${output_dir}" == "/tmp/postgres-keycloak-cohort-${run_id}-${run_attempt}" ]] || { + echo "Backup output must be the exact run-scoped /tmp path." >&2 + exit 2 +} +[[ -d /tmp && ! -L /tmp && ! -e "${output_dir}" && ! -L "${output_dir}" ]] || { + echo "Refusing an unsafe or reused cohort backup output path." >&2 + exit 1 +} +for command_name in docker sha256sum; do + command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; } +done + +cleanup_on_failure() { + local status=$? + trap - EXIT HUP INT TERM + if ((status != 0)) && [[ -d "${output_dir}" && ! -L "${output_dir}" ]]; then + find "${output_dir}" -mindepth 1 -delete + rmdir "${output_dir}" + fi + exit "${status}" +} +trap cleanup_on_failure EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +umask 077 +install -d -m 0700 "${output_dir}" + +inspect=$(docker inspect "${container}" --format '{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.service"}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}') +IFS='|' read -r compose_project compose_service observed_image state health <<<"${inspect}" +[[ "${compose_project}" == postgres && "${compose_service}" == postgres && "${observed_image}" == "${postgres_image}" \ + && "${state}" == running && "${health}" == healthy ]] || { + echo "The exact healthy production PostgreSQL container is not present." >&2 + exit 1 +} + +databases=( + keycloak_betacrew + keycloak_catwlk + keycloak_makepad + keycloak_runtrace + keycloak_vestiaire + keycloak_vif +) +for database in "${databases[@]}"; do + exists=$(docker exec "${container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + exec psql -h 127.0.0.1 -U postgres -d postgres -v ON_ERROR_STOP=1 -Atqc \ + "SELECT count(*) FROM pg_database WHERE datname = '\''$1'\'' AND datallowconn" + ' sh "${database}") + [[ "${exists}" == 1 ]] || { echo "Required database ${database} is missing or disallows connections." >&2; exit 1; } + partial="${output_dir}/.${database}.dump.partial" + final="${output_dir}/${database}.dump" + docker exec "${container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + exec pg_dump -h 127.0.0.1 -U postgres --format=custom --compress=6 \ + --no-owner --no-privileges --dbname "$1" + ' sh "${database}" >"${partial}" + [[ -s "${partial}" && ! -L "${partial}" ]] || { echo "Empty backup for ${database}." >&2; exit 1; } + docker run --rm --read-only --network none --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${partial},dst=/backup.dump,readonly" \ + "${postgres_image}" pg_restore --list /backup.dump >/dev/null + chmod 0600 "${partial}" + mv -T "${partial}" "${final}" +done + +expected=$(printf '%s\n' "${databases[@]/%/.dump}" | sort) +observed=$(find "${output_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | sort) +[[ "${observed}" == "${expected}" ]] || { echo "Cohort backup directory has an unexpected entry set." >&2; exit 1; } +sha256sum "${output_dir}"/*.dump >/dev/null +trap - EXIT HUP INT TERM +echo "Captured and structurally validated the exact six Keycloak databases." diff --git a/scripts/clean-keycloak-cohort-resources.sh b/scripts/clean-keycloak-cohort-resources.sh new file mode 100755 index 0000000..f4a22a9 --- /dev/null +++ b/scripts/clean-keycloak-cohort-resources.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +[[ "$(id -u)" -eq 0 ]] || { echo "The cohort resource cleaner must run as root." >&2; exit 1; } +for command_name in date docker find; do command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; }; done + +now=$(date +%s) +contract=makepad-keycloak-cohort-restore-v1 + +remove_expired_container() { + local identifier=$1 details name observed_contract expires + details=$(docker container inspect "${identifier}" --format '{{.Name}}|{{index .Config.Labels "makepad.cleanup.contract"}}|{{index .Config.Labels "makepad.cleanup.expires-epoch"}}') + IFS='|' read -r name observed_contract expires <<<"${details}" + name=${name#/} + [[ "${name}" =~ ^pg-kc-(db|app)-[1-9][0-9]*-[1-9][0-9]*-(betacrew|catwlk|makepad|runtrace|vestiaire|vif)$ \ + && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { + echo "Refusing malformed labeled cohort container ${identifier}." >&2 + return 1 + } + ((expires > now)) || docker rm -f "${identifier}" >/dev/null +} + +remove_expired_network() { + local identifier=$1 details name observed_contract expires + details=$(docker network inspect "${identifier}" --format '{{.Name}}|{{index .Labels "makepad.cleanup.contract"}}|{{index .Labels "makepad.cleanup.expires-epoch"}}') + IFS='|' read -r name observed_contract expires <<<"${details}" + [[ "${name}" =~ ^pg-kc-[1-9][0-9]*-[1-9][0-9]*-(betacrew|catwlk|makepad|runtrace|vestiaire|vif)$ \ + && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { + echo "Refusing malformed labeled cohort network ${identifier}." >&2 + return 1 + } + ((expires > now)) || docker network rm "${identifier}" >/dev/null +} + +while IFS= read -r identifier; do [[ -z "${identifier}" ]] || remove_expired_container "${identifier}"; done \ + < <(docker container ls -aq --filter "label=makepad.cleanup.contract=${contract}") +while IFS= read -r identifier; do [[ -z "${identifier}" ]] || remove_expired_network "${identifier}"; done \ + < <(docker network ls -q --filter "label=makepad.cleanup.contract=${contract}") + +find /tmp -mindepth 1 -maxdepth 1 -type d -name 'postgres-keycloak-cohort-*' -mmin +180 \ + -exec sh -euc 'for directory do + [ ! -L "$directory" ] || continue + [ ! -f "$directory/RECOVERY_REQUIRED" ] || continue + find "$directory" -depth -delete + done' sh {} + diff --git a/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh b/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh new file mode 100755 index 0000000..18f06ec --- /dev/null +++ b/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/keycloak-cohort-cleaner-mock-bin +state=/tmp/keycloak-cohort-cleaner-mock-state +install -d -m 0700 "${mock_bin}" "${state}" + +cat >"${mock_bin}/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +state=/tmp/keycloak-cohort-cleaner-mock-state +kind=${1:-} +action=${2:-} +shift 2 || true +case "${kind}:${action}" in + container:ls) + [[ " $* " == *' -aq '* && " $* " == *' label=makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 '* ]] + cat "${state}/containers" + ;; + container:inspect) + identifier=${1:-} + case "${identifier}" in + c-expired) printf '/pg-kc-db-101-1-catwlk|makepad-keycloak-cohort-restore-v1|1\n' ;; + c-future) printf '/pg-kc-app-101-1-catwlk|makepad-keycloak-cohort-restore-v1|4102444800\n' ;; + c-malformed) printf '/unrelated-container|makepad-keycloak-cohort-restore-v1|1\n' ;; + *) exit 1 ;; + esac + ;; + network:ls) + [[ " $* " == *' -q '* && " $* " == *' label=makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 '* ]] + cat "${state}/networks" + ;; + network:inspect) + identifier=${1:-} + case "${identifier}" in + n-expired) printf 'pg-kc-101-1-catwlk|makepad-keycloak-cohort-restore-v1|1\n' ;; + n-future) printf 'pg-kc-102-1-catwlk|makepad-keycloak-cohort-restore-v1|4102444800\n' ;; + *) exit 1 ;; + esac + ;; + rm:-f) + printf '%s\n' "${1:-}" >>"${state}/removed-containers" + ;; + network:rm) + printf '%s\n' "${1:-}" >>"${state}/removed-networks" + ;; + *) + echo "Unhandled mocked Docker operation: ${kind} ${action} $*" >&2 + exit 1 + ;; +esac +MOCK +chmod 0755 "${mock_bin}/docker" +export PATH="${mock_bin}:${PATH}" + +printf '%s\n' c-expired c-future >"${state}/containers" +printf '%s\n' n-expired n-future >"${state}/networks" +install -d -m 0700 /tmp/postgres-keycloak-cohort-expired /tmp/postgres-keycloak-cohort-future \ + /tmp/postgres-keycloak-cohort-recovery +printf '%s\n' preserve > /tmp/postgres-keycloak-cohort-recovery/RECOVERY_REQUIRED +touch -t 202001010000 /tmp/postgres-keycloak-cohort-expired /tmp/postgres-keycloak-cohort-recovery + +"${repo}/scripts/clean-keycloak-cohort-resources.sh" +[[ $(<"${state}/removed-containers") == c-expired ]] +[[ $(<"${state}/removed-networks") == n-expired ]] +[[ ! -e /tmp/postgres-keycloak-cohort-expired ]] +[[ -d /tmp/postgres-keycloak-cohort-future ]] +[[ -f /tmp/postgres-keycloak-cohort-recovery/RECOVERY_REQUIRED ]] + +printf '%s\n' c-malformed >"${state}/containers" +: >"${state}/networks" +: >"${state}/removed-containers" +if "${repo}/scripts/clean-keycloak-cohort-resources.sh" >/tmp/cohort-cleaner-malformed-output 2>&1; then + echo "Cohort cleaner accepted a malformed resource carrying its label." >&2 + exit 1 +fi +grep -q 'Refusing malformed labeled cohort container' /tmp/cohort-cleaner-malformed-output +[[ ! -s "${state}/removed-containers" ]] + +echo "Keycloak cohort resource cleaner tests passed." diff --git a/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh b/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh new file mode 100755 index 0000000..f8d6d52 --- /dev/null +++ b/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/keycloak-cohort-dispatch-mock-bin +install -d -m 0755 /usr/local/libexec/makepad +install -d -m 0700 "${mock_bin}" +install -o root -g root -m 0755 "${repo}/scripts/capture-keycloak-cohort-backups.sh" \ + /usr/local/libexec/makepad/capture-keycloak-cohort-backups +install -o root -g root -m 0755 "${repo}/scripts/clean-keycloak-cohort-resources.sh" \ + /usr/local/libexec/makepad/clean-keycloak-cohort-resources +install -o root -g root -m 0755 "${repo}/scripts/keycloak-cohort-capture-dispatch.sh" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch +cat >"${mock_bin}/systemctl" <<'MOCK' +#!/usr/bin/env sh +case "$1:$2" in + is-enabled:--quiet|is-active:--quiet) exit 0 ;; + show:--property=Result) printf '%s\n' success ;; + show:--property=ExecMainStatus) printf '%s\n' 0 ;; + *) exit 1 ;; +esac +MOCK +chmod 0755 "${mock_bin}/systemctl" +export PATH="${mock_bin}:${PATH}" + +helper_digest=$(sha256sum /usr/local/libexec/makepad/capture-keycloak-cohort-backups | cut -d' ' -f1) +cleaner_digest=$(sha256sum /usr/local/libexec/makepad/clean-keycloak-cohort-resources | cut -d' ' -f1) +result=$(SSH_ORIGINAL_COMMAND="probe ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch) +[[ "${result}" == cohort-capture-contract-ok ]] + +wrong_digest=$(printf '0%.0s' {1..64}) +if SSH_ORIGINAL_COMMAND="probe ${helper_digest} ${wrong_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted a substituted cleaner digest." >&2 + exit 1 +fi +if SSH_ORIGINAL_COMMAND="shell ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted an arbitrary operation." >&2 + exit 1 +fi +if SSH_ORIGINAL_COMMAND="fetch 1 1 ../../etc/passwd ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted an unsupported fetch path." >&2 + exit 1 +fi + +echo "Keycloak cohort forced-command tests passed." diff --git a/scripts/install-keycloak-cohort-capture-host.sh b/scripts/install-keycloak-cohort-capture-host.sh new file mode 100755 index 0000000..b269b11 --- /dev/null +++ b/scripts/install-keycloak-cohort-capture-host.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: install-keycloak-cohort-capture-host.sh " >&2 + exit 2 +fi +[[ "$(id -u)" -eq 0 ]] || { echo "The capture host installer must run as root." >&2; exit 1; } +capture_user=$1 +public_key_file=$2 +[[ "${capture_user}" =~ ^[a-z_][a-z0-9_-]{0,30}$ && "${capture_user}" != root ]] || { echo "Capture user is invalid." >&2; exit 2; } +id "${capture_user}" >/dev/null +id -nG "${capture_user}" | tr ' ' '\n' | grep -Fxq docker || { echo "Capture user must already belong to the Docker group." >&2; exit 1; } +[[ -f "${public_key_file}" && ! -L "${public_key_file}" ]] || { echo "Public key input is unsafe." >&2; exit 2; } +read -r key_type key_body _ <"${public_key_file}" +case "${key_type}" in ssh-ed25519|sk-ssh-ed25519@openssh.com) ;; *) echo "Only Ed25519 capture keys are accepted." >&2; exit 2 ;; esac +[[ "${key_body}" =~ ^[A-Za-z0-9+/]+={0,3}$ ]] || { echo "Public key encoding is invalid." >&2; exit 2; } + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +"${script_dir}/install-keycloak-cohort-cleaner.sh" +install -d -o root -g root -m 0755 /usr/local/libexec/makepad /etc/makepad/keycloak-cohort-capture +install -o root -g root -m 0755 "${script_dir}/capture-keycloak-cohort-backups.sh" \ + /usr/local/libexec/makepad/capture-keycloak-cohort-backups +install -o root -g root -m 0755 "${script_dir}/keycloak-cohort-capture-dispatch.sh" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch +capture_group=$(id -gn "${capture_user}") +printf 'restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch" %s %s\n' \ + "${key_type}" "${key_body}" > /etc/makepad/keycloak-cohort-capture/authorized_keys +chown root:"${capture_group}" /etc/makepad/keycloak-cohort-capture/authorized_keys +chmod 0640 /etc/makepad/keycloak-cohort-capture/authorized_keys +cat > /etc/ssh/sshd_config.d/70-makepad-keycloak-cohort-capture.conf </dev/null || systemctl reload sshd.service +printf 'capture_helper_sha256=%s\n' "$(sha256sum /usr/local/libexec/makepad/capture-keycloak-cohort-backups | cut -d' ' -f1)" +printf 'cohort_cleaner_sha256=%s\n' "$(sha256sum /usr/local/libexec/makepad/clean-keycloak-cohort-resources | cut -d' ' -f1)" diff --git a/scripts/install-keycloak-cohort-cleaner.sh b/scripts/install-keycloak-cohort-cleaner.sh new file mode 100755 index 0000000..53eccb1 --- /dev/null +++ b/scripts/install-keycloak-cohort-cleaner.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +[[ $# -eq 0 ]] || { echo "usage: install-keycloak-cohort-cleaner.sh" >&2; exit 2; } +[[ "$(id -u)" -eq 0 ]] || { echo "The cohort cleaner installer must run as root." >&2; exit 1; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +repo_root=$(cd "${script_dir}/.." && pwd -P) +for source in "${script_dir}/clean-keycloak-cohort-resources.sh" \ + "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.service" \ + "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.timer"; do + [[ -f "${source}" && ! -L "${source}" ]] || { echo "Missing safe installer input: ${source}" >&2; exit 1; } +done +install -d -o root -g root -m 0755 /usr/local/libexec/makepad +install -o root -g root -m 0755 "${script_dir}/clean-keycloak-cohort-resources.sh" \ + /usr/local/libexec/makepad/clean-keycloak-cohort-resources +install -o root -g root -m 0644 "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.service" \ + /etc/systemd/system/makepad-keycloak-cohort-cleaner.service +install -o root -g root -m 0644 "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.timer" \ + /etc/systemd/system/makepad-keycloak-cohort-cleaner.timer +systemctl daemon-reload +systemctl enable --now makepad-keycloak-cohort-cleaner.timer >/dev/null +systemctl start makepad-keycloak-cohort-cleaner.service +systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer +systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer diff --git a/scripts/keycloak-cohort-capture-dispatch.sh b/scripts/keycloak-cohort-capture-dispatch.sh new file mode 100755 index 0000000..a7943aa --- /dev/null +++ b/scripts/keycloak-cohort-capture-dispatch.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C +set -f + +helper=/usr/local/libexec/makepad/capture-keycloak-cohort-backups +cleaner=/usr/local/libexec/makepad/clean-keycloak-cohort-resources +command_text=${SSH_ORIGINAL_COMMAND:-} +read -r -a words <<<"${command_text}" +[[ ${#words[@]} -ge 3 ]] || { echo "A constrained cohort capture command is required." >&2; exit 2; } +operation=${words[0]} +helper_digest=${words[${#words[@]}-2]} +cleaner_digest=${words[${#words[@]}-1]} +[[ "${helper_digest}" =~ ^[a-f0-9]{64}$ && "${cleaner_digest}" =~ ^[a-f0-9]{64}$ ]] || { + echo "The reviewed capture-helper and cleaner digests are required." >&2 + exit 2 +} +[[ -x "${helper}" && ! -L "${helper}" && -x "${cleaner}" && ! -L "${cleaner}" ]] || { echo "The root-owned cohort capture contract is not installed." >&2; exit 1; } +[[ $(sha256sum "${helper}" | cut -d' ' -f1) == "${helper_digest}" ]] || { echo "The installed capture helper differs from the reviewed release." >&2; exit 1; } +[[ $(sha256sum "${cleaner}" | cut -d' ' -f1) == "${cleaner_digest}" ]] || { echo "The installed cohort cleaner differs from the reviewed release." >&2; exit 1; } +systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer +systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer +[[ $(systemctl show --property=Result --value makepad-keycloak-cohort-cleaner.service) == success ]] +[[ $(systemctl show --property=ExecMainStatus --value makepad-keycloak-cohort-cleaner.service) == 0 ]] + +validate_run() { + [[ "$1" =~ ^[1-9][0-9]*$ && "$2" =~ ^[1-9][0-9]*$ ]] || { echo "Invalid cohort run identity." >&2; exit 2; } + cohort_dir="/tmp/postgres-keycloak-cohort-$1-$2" +} + +case "${operation}" in + probe) + [[ ${#words[@]} -eq 3 ]] + printf 'cohort-capture-contract-ok\n' + ;; + capture) + [[ ${#words[@]} -eq 5 ]] + validate_run "${words[1]}" "${words[2]}" + COHORT_CAPTURE_RUN_ID=${words[1]} COHORT_CAPTURE_RUN_ATTEMPT=${words[2]} \ + "${helper}" "${cohort_dir}" + ;; + fetch) + [[ ${#words[@]} -eq 6 ]] + validate_run "${words[1]}" "${words[2]}" + file=${words[3]} + case "${file}" in + keycloak_betacrew.dump|keycloak_catwlk.dump|keycloak_makepad.dump|keycloak_runtrace.dump|keycloak_vestiaire.dump|keycloak_vif.dump) ;; + *) echo "Unsupported cohort artifact." >&2; exit 2 ;; + esac + [[ -f "${cohort_dir}/${file}" && ! -L "${cohort_dir}/${file}" && -s "${cohort_dir}/${file}" ]] || exit 1 + exec cat "${cohort_dir}/${file}" + ;; + cleanup) + [[ ${#words[@]} -eq 5 ]] + validate_run "${words[1]}" "${words[2]}" + if [[ -d "${cohort_dir}" && ! -L "${cohort_dir}" ]]; then + find "${cohort_dir}" -depth -delete + elif [[ -e "${cohort_dir}" || -L "${cohort_dir}" ]]; then + echo "Refusing unsafe cohort cleanup target." >&2 + exit 1 + fi + ;; + *) echo "Unsupported constrained cohort capture operation." >&2; exit 2 ;; +esac diff --git a/scripts/restore-keycloak-cohort-backups.sh b/scripts/restore-keycloak-cohort-backups.sh new file mode 100755 index 0000000..9806669 --- /dev/null +++ b/scripts/restore-keycloak-cohort-backups.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +if (($# != 4)); then + echo "usage: restore-keycloak-cohort-backups.sh " >&2 + exit 2 +fi + +backup_dir=$1 +result_dir=$2 +keycloak_source=$3 +catwlk_image=$4 +keycloak_image='dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' +postgres_image='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +run_id=${GITHUB_RUN_ID:-1} +run_attempt=${GITHUB_RUN_ATTEMPT:-1} +[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ ]] || { echo "Invalid run identity." >&2; exit 2; } +[[ -d "${backup_dir}" && ! -L "${backup_dir}" && ! -e "${result_dir}" && ! -L "${result_dir}" \ + && -d "${keycloak_source}" && ! -L "${keycloak_source}" ]] || { + echo "Backup/source inputs must be real directories and result output must be new." >&2 + exit 1 +} +[[ "${catwlk_image}" == "makepad/keycloak-catwlk-cohort:"* ]] || { echo "Catwlk runtime tag is outside the reviewed namespace." >&2; exit 2; } +for command_name in docker git python3 sha256sum; do command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; }; done + +declare -A databases=( + [betacrew]=keycloak_betacrew + [catwlk]=keycloak_catwlk + [makepad]=keycloak_makepad + [runtrace]=keycloak_runtrace + [vestiaire]=keycloak_vestiaire + [vif]=keycloak_vif +) +declare -A category_tables=( + [realm]='realm realm_attribute realm_default_groups realm_enabled_event_types realm_events_listeners realm_localizations realm_required_credential realm_smtp_config realm_supported_locales' + [authentication]='authentication_flow authentication_execution authenticator_config authenticator_config_entry' + [roles]='keycloak_role role_attribute composite_role scope_mapping' + [clients]='client client_attributes client_auth_flow_bindings redirect_uris web_origins client_scope client_scope_attributes client_scope_client client_scope_role_mapping default_client_scope protocol_mapper protocol_mapper_config' + [identity_providers]='identity_provider identity_provider_config identity_provider_mapper idp_mapper_config' + [components]='component component_config' + [required_actions]='required_action_provider' +) +expected=$(for slug in "${!databases[@]}"; do printf '%s.dump\n' "${databases[$slug]}"; done | sort) +observed=$(find "${backup_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | sort) +[[ "${observed}" == "${expected}" ]] || { echo "Backup input is not the exact six-database cohort." >&2; exit 1; } +if find "${backup_dir}" -mindepth 1 -maxdepth 1 -type l -print -quit | grep -q .; then echo "Backup input contains a symlink." >&2; exit 1; fi + +release_sha=$(git -C "${keycloak_source}" rev-parse HEAD) +[[ "${release_sha}" =~ ^[a-f0-9]{40}$ ]] || { echo "Checked-out Keycloak release SHA is invalid." >&2; exit 1; } +python3 - "${keycloak_source}/instances/catwlk/manifest.json" <<'PY' +import json, pathlib, sys +value=json.loads(pathlib.Path(sys.argv[1]).read_text())["runtime"] +expected={"source_dir":"providers/catwlk-email-router","dockerfile":"providers/catwlk-email-router/Dockerfile","image":"makepad/keycloak-catwlk"} +if value != expected: raise SystemExit("Catwlk manifest does not select the reviewed custom runtime") +PY +grep -Fxq 'com.makepad.catwlk.keycloak.email.RoutingEmailSenderProviderFactory' \ + "${keycloak_source}/providers/catwlk-email-router/src/main/resources/META-INF/services/org.keycloak.email.EmailSenderProviderFactory" +grep -Fxq 'com.makepad.catwlk.keycloak.apple.CatwlkAppleIdentityProviderFactory' \ + "${keycloak_source}/providers/catwlk-email-router/src/main/resources/META-INF/services/org.keycloak.broker.provider.IdentityProviderFactory" +catwlk_image_id=$(docker image inspect "${catwlk_image}" --format '{{.Id}}') +catwlk_base=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.base-image"}}') +catwlk_source_sha=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.source-sha"}}') +catwlk_runtime=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.runtime"}}') +[[ "${catwlk_image_id}" =~ ^sha256:[a-f0-9]{64}$ && "${catwlk_base}" == "${keycloak_image}" \ + && "${catwlk_source_sha}" == "${release_sha}" && "${catwlk_runtime}" == catwlk-custom-provider ]] || { + echo "Catwlk image is not the exact custom-provider runtime from the checked-out release." >&2 + exit 1 +} +docker run --rm --network none --read-only --entrypoint sh "${catwlk_image}" -euc \ + 'test -s /opt/keycloak/providers/catwlk-keycloak-email-router.jar' + +umask 077 +install -d -m 0700 "${result_dir}" "${result_dir}/.runtime" +records="${result_dir}/.instances.jsonl" +: >"${records}" +active_network= +active_database_container= +active_keycloak_container= +cleanup_active() { + local status=$? + trap - EXIT HUP INT TERM + set +e + [[ -z "${active_keycloak_container}" ]] || docker rm -f "${active_keycloak_container}" >/dev/null 2>&1 + [[ -z "${active_database_container}" ]] || docker rm -f "${active_database_container}" >/dev/null 2>&1 + [[ -z "${active_network}" ]] || docker network rm "${active_network}" >/dev/null 2>&1 + if ((status != 0)) && [[ -d "${result_dir}" && ! -L "${result_dir}" ]]; then find "${result_dir}" -depth -delete; fi + exit "${status}" +} +trap cleanup_active EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +db_query() { + local sql=$1 + docker exec "${active_database_container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres-password)" + exec psql -X -v ON_ERROR_STOP=1 -U keycloak -d keycloak -At -c "$1" + ' sh "${sql}" +} + +fingerprint_configuration() { + local destination_name=$1 category table exists digest + local -n destination="${destination_name}" + for category in $(printf '%s\n' "${!category_tables[@]}" | sort); do + for table in ${category_tables[$category]}; do + exists=$(db_query "SELECT CASE WHEN to_regclass('public.${table}') IS NULL THEN 0 ELSE 1 END") + [[ "${exists}" == 1 ]] || { echo "Keycloak configuration fingerprint schema is missing required table ${table}." >&2; return 1; } + done + digest=$( + { + for table in ${category_tables[$category]}; do + printf 'table\t%s\n' "${table}" + db_query "SELECT to_jsonb(value)::text FROM ${table} AS value ORDER BY to_jsonb(value)::text" + done + } | sha256sum | cut -d' ' -f1 + ) + [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || return 1 + # shellcheck disable=SC2034 # destination is an associative nameref output. + destination["${category}"]=${digest} + done +} + +for slug in $(printf '%s\n' "${!databases[@]}" | sort); do + database=${databases[$slug]} + dump="${backup_dir}/${database}.dump" + [[ -s "${dump}" && -f "${dump}" && ! -L "${dump}" ]] || { echo "Invalid dump for ${slug}." >&2; exit 1; } + docker run --rm --read-only --network none --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${dump},dst=/backup.dump,readonly" "${postgres_image}" pg_restore --list /backup.dump >/dev/null + backup_sha=$(sha256sum "${dump}" | cut -d' ' -f1) + suffix="${run_id}-${run_attempt}-${slug}" + active_network="pg-kc-${suffix}" + active_database_container="pg-kc-db-${suffix}" + active_keycloak_container="pg-kc-app-${suffix}" + secret_file="${result_dir}/.runtime/${slug}-postgres-password" + python3 - "${secret_file}" <<'PY' +import pathlib, secrets, sys +pathlib.Path(sys.argv[1]).write_text(secrets.token_urlsafe(48) + "\n") +PY + chmod 0600 "${secret_file}" + expires_epoch=$(( $(date +%s) + 10800 )) + labels=(--label makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 \ + --label "makepad.cleanup.run=${run_id}-${run_attempt}" --label "makepad.cleanup.expires-epoch=${expires_epoch}") + docker network create --internal "${labels[@]}" "${active_network}" >/dev/null + docker run -d --name "${active_database_container}" --network "${active_network}" --network-alias db \ + "${labels[@]}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=64m --tmpfs /run/postgresql:rw,noexec,nosuid,size=16m \ + --tmpfs /var/lib/postgresql/data:rw,nosuid,size=2g --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password -e POSTGRES_DB=postgres \ + "${postgres_image}" >/dev/null + ready=0 + for _ in $(seq 1 60); do + if docker exec "${active_database_container}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then ready=1; break; fi + [[ $(docker inspect "${active_database_container}" --format '{{.State.Running}}') == true ]] || break + sleep 1 + done + [[ "${ready}" == 1 ]] || { docker logs "${active_database_container}" >&2; echo "Disposable PostgreSQL did not become ready." >&2; exit 1; } + docker exec "${active_database_container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres-password)" + export KEYCLOAK_PASSWORD="$PGPASSWORD" + { printf "%s\n" "\\getenv keycloak_password KEYCLOAK_PASSWORD"; printf "%s\n" "SELECT format('\''CREATE ROLE keycloak LOGIN PASSWORD %L'\'', :'\''keycloak_password'\'') \\gexec" "CREATE DATABASE keycloak OWNER keycloak;"; } | + psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null + ' + docker run --rm --network "${active_network}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=64m \ + --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${dump},dst=/backup.dump,readonly" \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + "${postgres_image}" sh -euc 'export PGPASSWORD="$(cat /run/secrets/postgres-password)"; exec pg_restore -h db -U keycloak -d keycloak --no-owner --no-privileges --exit-on-error /backup.dump' >/dev/null + realm_count=$(db_query 'SELECT count(*) FROM realm') + [[ "${realm_count}" =~ ^[1-9][0-9]*$ ]] || { echo "Restored ${slug} database has no realm." >&2; exit 1; } + declare -A before=() after=() + fingerprint_configuration before + runtime_image=${keycloak_image} + runtime_kind=keycloak-base + runtime_evidence_image=${keycloak_image} + if [[ "${slug}" == catwlk ]]; then runtime_image=${catwlk_image}; runtime_evidence_image=${catwlk_image_id}; runtime_kind=catwlk-custom-provider; fi + keycloak_uid=$(docker run --rm --network none --entrypoint id "${runtime_image}" -u) + [[ "${keycloak_uid}" =~ ^[1-9][0-9]*$ ]] || { echo "Keycloak runtime must use a non-root numeric user." >&2; exit 1; } + docker run --rm --network none --cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges:true \ + --mount "type=bind,src=${secret_file},dst=/secret" "${postgres_image}" chown "${keycloak_uid}:0" /secret + chmod 0400 "${secret_file}" + docker run -d --name "${active_keycloak_container}" --network "${active_network}" --network-alias keycloak \ + "${labels[@]}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=256m \ + --tmpfs "/opt/keycloak/data:rw,nosuid,size=256m,uid=${keycloak_uid},gid=0,mode=0770" \ + --cap-drop ALL --security-opt no-new-privileges:true --entrypoint sh \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + -e KC_DB=postgres -e KC_DB_URL=jdbc:postgresql://db:5432/keycloak -e KC_DB_USERNAME=keycloak \ + -e KC_HEALTH_ENABLED=true -e KC_HTTP_ENABLED=true -e KC_HOSTNAME_STRICT=false \ + "${runtime_image}" -euc 'export KC_DB_PASSWORD="$(cat /run/secrets/postgres-password)"; exec /opt/keycloak/bin/kc.sh start-dev' >/dev/null + ready=0 + for _ in $(seq 1 180); do + if docker run --rm --network "${active_network}" --read-only --cap-drop ALL --security-opt no-new-privileges:true \ + "${postgres_image}" sh -euc 'wget -qO- http://keycloak:9000/health/ready' 2>/dev/null | grep -q '"status"[[:space:]]*:[[:space:]]*"UP"'; then ready=1; break; fi + [[ $(docker inspect "${active_keycloak_container}" --format '{{.State.Running}}') == true ]] || break + sleep 1 + done + [[ "${ready}" == 1 ]] || { docker logs "${active_keycloak_container}" >&2; echo "Keycloak 26.7.3 did not become ready for restored ${slug}." >&2; exit 1; } + fingerprint_configuration after + fingerprints=() + for category in $(printf '%s\n' "${!category_tables[@]}" | sort); do + [[ "${after[$category]}" == "${before[$category]}" ]] || { echo "Keycloak ${category} configuration regression detected for ${slug}." >&2; exit 1; } + fingerprints+=("${category}=${after[$category]}") + done + combined=$(printf '%s\n' "${fingerprints[@]}" | sha256sum | cut -d' ' -f1) + python3 - "${records}" "${slug}" "${database}" "${backup_sha}" "${runtime_kind}" "${runtime_evidence_image}" "${combined}" "${fingerprints[@]}" <<'PY' +import json, pathlib, sys +fingerprints=dict(item.split("=", 1) for item in sys.argv[8:]) +record={"slug":sys.argv[2],"database":sys.argv[3],"backup_sha256":sys.argv[4],"runtime":sys.argv[5],"runtime_image":sys.argv[6],"configuration_fingerprint":sys.argv[7],"configuration_fingerprints":fingerprints,"restore":"passed","keycloak_startup":"passed","configuration_regression":"passed"} +with pathlib.Path(sys.argv[1]).open("a", encoding="utf-8") as target: target.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") +PY + docker rm -f "${active_keycloak_container}" "${active_database_container}" >/dev/null + active_keycloak_container= + active_database_container= + docker network rm "${active_network}" >/dev/null + active_network= + rm -f "${secret_file}" + unset before after +done + +python3 - "${records}" "${result_dir}/instances.json" "${catwlk_image_id}" <<'PY' +import json, pathlib, sys +instances=[json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +payload={"catwlk_runtime_image_id":sys.argv[3],"instances":instances} +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY +rm -f "${records}" +rmdir "${result_dir}/.runtime" +chmod 0600 "${result_dir}/instances.json" +trap - EXIT HUP INT TERM +echo "Restored all six Keycloak databases with exact runtimes and verified complete configuration fingerprints." diff --git a/scripts/test-keycloak-cohort-evidence.sh b/scripts/test-keycloak-cohort-evidence.sh new file mode 100755 index 0000000..9b5e5b1 --- /dev/null +++ b/scripts/test-keycloak-cohort-evidence.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +validator="${repo_root}/scripts/verify-keycloak-cohort-evidence.py" +work_dir=$(mktemp -d) +cleanup() { + find "${work_dir}" -mindepth 1 -delete + rmdir "${work_dir}" +} +trap cleanup EXIT + +head_sha=$(printf 'a%.0s' {1..40}) +release_sha=$(printf 'b%.0s' {1..40}) +canonical="${work_dir}/keycloak-cohort-restore-evidence.json" + +python3 - "${canonical}" "${head_sha}" "${release_sha}" <<'PY' +import hashlib +import json +import pathlib +import sys + +databases = { + "betacrew": "keycloak_betacrew", + "catwlk": "keycloak_catwlk", + "makepad": "keycloak_makepad", + "runtrace": "keycloak_runtrace", + "vestiaire": "keycloak_vestiaire", + "vif": "keycloak_vif", +} +payload = { + "schema": "makepad.keycloak-cohort-restore-evidence.v2", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/verify-keycloak-cohort-restores.yml", + "postgres_run_id": 101, + "postgres_run_attempt": 2, + "postgres_head_sha": sys.argv[2], + "postgres_ref": "refs/heads/main", + "keycloak_release_sha": sys.argv[3], + "keycloak_base_image": "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "catwlk_runtime_image_id": "sha256:" + "c" * 64, + "fingerprint_schema": "makepad.keycloak-config-fingerprint.v2", + "keycloak_upstream_version": "26.7.3", + "result": "restored-databases-compatible", + "instances": [], +} +categories = ["authentication", "clients", "components", "identity_providers", "realm", "required_actions", "roles"] +for index, (slug, database) in enumerate(sorted(databases.items())): + fingerprints = {category: format((index + 1) * 10 + offset, "064x") for offset, category in enumerate(categories)} + combined = hashlib.sha256("".join(f"{category}={fingerprints[category]}\n" for category in categories).encode()).hexdigest() + payload["instances"].append({ + "slug": slug, + "database": database, + "backup_sha256": format(index + 1, "064x"), + "runtime": "catwlk-custom-provider" if slug == "catwlk" else "keycloak-base", + "runtime_image": "sha256:" + "c" * 64 if slug == "catwlk" else "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "configuration_fingerprint": combined, + "configuration_fingerprints": fingerprints, + "restore": "passed", + "keycloak_startup": "passed", + "configuration_regression": "passed", + }) +pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY + +python3 "${validator}" "${canonical}" --run-id 101 --run-attempt 2 \ + --head-sha "${head_sha}" --keycloak-release-sha "${release_sha}" + +expect_failure() { + local name=$1 expression=$2 candidate + candidate="${work_dir}/${name}.json" + python3 - "${canonical}" "${candidate}" "${expression}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +exec(sys.argv[3], {"payload": payload}) +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY + if python3 "${validator}" "${candidate}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted ${name}." >&2 + exit 1 + fi +} + +expect_failure wrong-schema 'payload["schema"] = "makepad.keycloak-cohort-restore-evidence.v1"' +expect_failure extra-field 'payload["unexpected"] = True' +expect_failure extra-instance-field 'payload["instances"][0]["unexpected"] = True' +expect_failure missing-instance 'payload["instances"].pop()' +expect_failure duplicate-instance 'payload["instances"][-1] = payload["instances"][0]' +expect_failure unsorted-instance 'payload["instances"][0], payload["instances"][1] = payload["instances"][1], payload["instances"][0]' +expect_failure wrong-database 'payload["instances"][0]["database"] = "keycloak_vif"' +expect_failure bad-digest 'payload["instances"][0]["backup_sha256"] = "A" * 64' +expect_failure failed-restore 'payload["instances"][0]["restore"] = "failed"' +expect_failure wrong-image 'payload["keycloak_base_image"] = "dhi.io/keycloak:latest"' +expect_failure wrong-catwlk-runtime 'next(value for value in payload["instances"] if value["slug"] == "catwlk")["runtime"] = "keycloak-base"' +expect_failure missing-fingerprint-category 'payload["instances"][0]["configuration_fingerprints"].pop("roles")' +expect_failure tampered-fingerprint 'payload["instances"][0]["configuration_fingerprints"]["roles"] = "f" * 64' +expect_failure mutable-release 'payload["keycloak_release_sha"] = "main"' + +noncanonical="${work_dir}/noncanonical.json" +python3 - "${canonical}" "${noncanonical}" <<'PY' +import json +import pathlib +import sys +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, indent=2)) +PY +if python3 "${validator}" "${noncanonical}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted non-canonical JSON." >&2 + exit 1 +fi + +symlink="${work_dir}/symlink.json" +ln -s "${canonical}" "${symlink}" +if python3 "${validator}" "${symlink}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted a symlink." >&2 + exit 1 +fi + +oversized="${work_dir}/oversized.json" +dd if=/dev/zero of="${oversized}" bs=65537 count=1 status=none +if python3 "${validator}" "${oversized}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted an oversized file." >&2 + exit 1 +fi + +echo "Keycloak cohort evidence contract tests passed." diff --git a/scripts/test-keycloak-cohort-hardening.sh b/scripts/test-keycloak-cohort-hardening.sh new file mode 100755 index 0000000..bc2e999 --- /dev/null +++ b/scripts/test-keycloak-cohort-hardening.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +REPO_ROOT="${repo_root}" python3 - <<'PY' +import os +from pathlib import Path + +root=Path(os.environ["REPO_ROOT"]) +workflow=(root/".github/workflows/verify-keycloak-cohort-restores.yml").read_text() +dispatch=(root/"scripts/keycloak-cohort-capture-dispatch.sh").read_text() +installer=(root/"scripts/install-keycloak-cohort-capture-host.sh").read_text() +restore=(root/"scripts/restore-keycloak-cohort-backups.sh").read_text() +cleaner=(root/"scripts/clean-keycloak-cohort-resources.sh").read_text() +timer=(root/"host/systemd/makepad-keycloak-cohort-cleaner.timer").read_text() + +def require(condition, message): + if not condition: raise SystemExit(message) + +require("scp " not in workflow and "remote_script=" not in workflow, "cohort workflow must not copy or execute checked-out code on the database host") +probe=workflow.index('"probe ${helper_digest} ${cleaner_digest}"') +capture=workflow.index('"capture ${GITHUB_RUN_ID}') +require(probe < capture, "remote forced-command digest and TTL probe must precede capture") +for marker in ("SSH_ORIGINAL_COMMAND", "sha256sum", "systemctl is-enabled", "systemctl is-active", "--property=Result", "--property=ExecMainStatus", "probe)", "capture)", "fetch)", "cleanup)"): + require(marker in dispatch, f"forced-command dispatcher is missing {marker}") +require('sha256sum "${cleaner}"' in dispatch and "cleaner_digest" in workflow, "forced commands must bind the exact installed cleaner digest") +require('restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch"' in installer, "installer must bind the key to the forced command") +require("sshd -t" in installer and "DisableForwarding yes" in installer, "installer must validate and restrict sshd") +for marker in ("makepad.cleanup.contract", "makepad.cleanup.expires-epoch", "docker container ls -aq", "docker network ls -q"): + require(marker in cleaner, f"cohort resource cleaner is missing {marker}") +require("Persistent=true" in timer and "OnUnitActiveSec=15min" in timer, "cohort cleanup timer must survive downtime and recur") +for category in ("realm", "authentication", "roles", "clients", "identity_providers", "components", "required_actions"): + require(f"[{category}]" in restore, f"v2 fingerprint is missing {category}") +for table in ("realm_smtp_config", "authentication_execution", "role_attribute", "composite_role", "client_scope_role_mapping", "protocol_mapper_config", "identity_provider_config", "component_config", "required_action_provider"): + require(table in restore, f"v2 fingerprint is missing table {table}") +require("to_regclass" in restore and "to_jsonb(value)::text" in restore, "fingerprint must fail closed on schema drift and serialize deterministically") +require("POSTGRES_PASSWORD=${" not in restore and "KC_DB_PASSWORD=${" not in restore, "test secrets must not enter Docker Config.Env") +require("POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password" in restore and 'export KC_DB_PASSWORD="$(cat /run/secrets/postgres-password)"' in restore, "runtime passwords must originate in mounted files") +require("catwlk-custom-provider" in restore and "catwlk-keycloak-email-router.jar" in restore, "Catwlk restore must validate the checked-out custom runtime") +PY + +test_image=$(awk -F= '$1 == "BRIO_BACKUP_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +[[ "${test_image}" == *@sha256:* ]] +docker run --rm --network none --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" "${test_image}" \ + bash /repo/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh +docker run --rm --network none --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" "${test_image}" \ + bash /repo/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh + +echo "Keycloak cohort capture, runtime, fingerprint, and cleanup hardening contracts passed." diff --git a/scripts/verify-keycloak-cohort-evidence.py b/scripts/verify-keycloak-cohort-evidence.py new file mode 100755 index 0000000..a59a20f --- /dev/null +++ b/scripts/verify-keycloak-cohort-evidence.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Validate the immutable six-database Keycloak restore evidence contract.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + +SCHEMA = "makepad.keycloak-cohort-restore-evidence.v2" +WORKFLOW = ".github/workflows/verify-keycloak-cohort-restores.yml" +BASE_IMAGE = "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f" +CATEGORIES = { + "realm", + "authentication", + "roles", + "clients", + "identity_providers", + "components", + "required_actions", +} +DATABASES = { + "betacrew": "keycloak_betacrew", + "catwlk": "keycloak_catwlk", + "makepad": "keycloak_makepad", + "runtrace": "keycloak_runtrace", + "vestiaire": "keycloak_vestiaire", + "vif": "keycloak_vif", +} +TOP_KEYS = { + "schema", + "postgres_repository", + "postgres_workflow", + "postgres_run_id", + "postgres_run_attempt", + "postgres_head_sha", + "postgres_ref", + "keycloak_release_sha", + "keycloak_base_image", + "catwlk_runtime_image_id", + "fingerprint_schema", + "keycloak_upstream_version", + "result", + "instances", +} +INSTANCE_KEYS = { + "slug", + "database", + "backup_sha256", + "runtime", + "runtime_image", + "configuration_fingerprint", + "configuration_fingerprints", + "restore", + "keycloak_startup", + "configuration_regression", +} + + +def positive_integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{label} must be a positive integer") + return value + + +def exact_keys(value: object, expected: set[str], label: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != expected: + raise ValueError(f"{label} has unexpected fields") + return value + + +def validate( + path: Path, + *, + expected_run_id: int | None = None, + expected_run_attempt: int | None = None, + expected_head_sha: str | None = None, + expected_keycloak_release_sha: str | None = None, +) -> dict[str, object]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024: + raise ValueError("evidence must be a regular file no larger than 64 KiB") + raw = path.read_bytes() + if b"\x00" in raw: + raise ValueError("evidence contains a NUL byte") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("evidence is not valid UTF-8 JSON") from error + top = exact_keys(payload, TOP_KEYS, "evidence") + run_id = positive_integer(top["postgres_run_id"], "postgres_run_id") + attempt = positive_integer(top["postgres_run_attempt"], "postgres_run_attempt") + if top["schema"] != SCHEMA: + raise ValueError("unexpected evidence schema") + if top["postgres_repository"] != "Makepad-fr/postgres" or top["postgres_workflow"] != WORKFLOW: + raise ValueError("unexpected PostgreSQL producer identity") + if top["postgres_ref"] != "refs/heads/main": + raise ValueError("evidence did not originate from protected main") + for label in ("postgres_head_sha", "keycloak_release_sha"): + if not isinstance(top[label], str) or not re.fullmatch(r"[a-f0-9]{40}", top[label]): + raise ValueError(f"{label} is not a lowercase commit SHA") + if top["keycloak_base_image"] != BASE_IMAGE or top["keycloak_upstream_version"] != "26.7.3": + raise ValueError("evidence is not bound to the reviewed Keycloak runtime") + if not isinstance(top["catwlk_runtime_image_id"], str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", top["catwlk_runtime_image_id"]): + raise ValueError("Catwlk runtime image ID is not immutable") + if top["fingerprint_schema"] != "makepad.keycloak-config-fingerprint.v2": + raise ValueError("unexpected configuration fingerprint schema") + if top["result"] != "restored-databases-compatible": + raise ValueError("cohort compatibility did not pass") + instances = top["instances"] + if not isinstance(instances, list) or len(instances) != len(DATABASES): + raise ValueError("instances must contain the exact six-database cohort") + expected_slugs = sorted(DATABASES) + actual_slugs: list[str] = [] + for index, raw_instance in enumerate(instances): + instance = exact_keys(raw_instance, INSTANCE_KEYS, f"instances[{index}]") + slug = instance["slug"] + if not isinstance(slug, str): + raise ValueError("instance slug must be a string") + actual_slugs.append(slug) + if slug not in DATABASES or instance["database"] != DATABASES[slug]: + raise ValueError("instance slug/database mapping is invalid") + if not isinstance(instance["backup_sha256"], str) or not re.fullmatch(r"[a-f0-9]{64}", instance["backup_sha256"]): + raise ValueError("backup_sha256 must be a lowercase SHA-256 digest") + expected_runtime = "catwlk-custom-provider" if slug == "catwlk" else "keycloak-base" + expected_image = top["catwlk_runtime_image_id"] if slug == "catwlk" else BASE_IMAGE + if instance["runtime"] != expected_runtime or instance["runtime_image"] != expected_image: + raise ValueError(f"{slug} runtime does not match its reviewed exact image") + fingerprints = exact_keys(instance["configuration_fingerprints"], CATEGORIES, f"{slug} configuration fingerprints") + for category, digest in fingerprints.items(): + if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): + raise ValueError(f"{slug} {category} fingerprint is invalid") + canonical_fingerprints = "".join(f"{category}={fingerprints[category]}\n" for category in sorted(CATEGORIES)).encode() + combined = hashlib.sha256(canonical_fingerprints).hexdigest() + if instance["configuration_fingerprint"] != combined: + raise ValueError(f"{slug} combined configuration fingerprint is invalid") + for status in ("restore", "keycloak_startup", "configuration_regression"): + if instance[status] != "passed": + raise ValueError(f"{slug} {status} did not pass") + if actual_slugs != expected_slugs: + raise ValueError("instances are duplicated, missing, or not sorted by slug") + expected_values = ( + (expected_run_id, run_id, "run ID"), + (expected_run_attempt, attempt, "run attempt"), + (expected_head_sha, top["postgres_head_sha"], "PostgreSQL head SHA"), + (expected_keycloak_release_sha, top["keycloak_release_sha"], "Keycloak release SHA"), + ) + for expected, actual, label in expected_values: + if expected is not None and expected != actual: + raise ValueError(f"evidence {label} mismatch") + canonical = json.dumps(top, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + if raw not in (canonical, canonical + b"\n"): + raise ValueError("evidence JSON is not canonical") + return top + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("path", type=Path) + parser.add_argument("--run-id", type=int) + parser.add_argument("--run-attempt", type=int) + parser.add_argument("--head-sha") + parser.add_argument("--keycloak-release-sha") + arguments = parser.parse_args() + validate( + arguments.path, + expected_run_id=arguments.run_id, + expected_run_attempt=arguments.run_attempt, + expected_head_sha=arguments.head_sha, + expected_keycloak_release_sha=arguments.keycloak_release_sha, + ) + + +if __name__ == "__main__": + main() From e17f7ebf0d919b7c08c7423dc42c828ec7e2bc44 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 04:28:03 +0200 Subject: [PATCH 11/20] ci(postgres): harden self-hosted release controls --- .github/actionlint.yaml | 6 + .github/workflows/ci.yml | 75 +- .github/workflows/pr-ci-result.yml | 62 ++ host/systemd/postgres-ci-queue-alert.service | 12 + .../postgres-ci-queue-controller.service | 22 + scripts/ci-base-image.py | 32 + scripts/configure-postgres-ci-runner-group.sh | 281 +++++++ scripts/dispatch-ci-attestation.mjs | 35 + scripts/postgres-ci-queue-controller.mjs | 203 ++++++ scripts/publish-pr-ci-check.mjs | 196 +++++ ...econcile-github-environment-main-policy.py | 249 +++++++ scripts/run-ci.sh | 75 ++ scripts/run-postgres-ci-jit-vm.sh | 685 ++++++++++++++++++ scripts/run-postgres-ci-queue-controller.sh | 9 + .../test-github-environment-main-policy.py | 109 +++ scripts/test-postgres-ci-jit-result.sh | 77 ++ scripts/test-postgres-ci-queue-controller.mjs | 126 ++++ scripts/test-pr-ci-check.mjs | 152 ++++ scripts/verify-postgres-ci-jit-result.py | 165 +++++ 19 files changed, 2544 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/pr-ci-result.yml create mode 100644 host/systemd/postgres-ci-queue-alert.service create mode 100644 host/systemd/postgres-ci-queue-controller.service create mode 100755 scripts/ci-base-image.py create mode 100755 scripts/configure-postgres-ci-runner-group.sh create mode 100644 scripts/dispatch-ci-attestation.mjs create mode 100644 scripts/postgres-ci-queue-controller.mjs create mode 100644 scripts/publish-pr-ci-check.mjs create mode 100755 scripts/reconcile-github-environment-main-policy.py create mode 100755 scripts/run-ci.sh create mode 100755 scripts/run-postgres-ci-jit-vm.sh create mode 100755 scripts/run-postgres-ci-queue-controller.sh create mode 100755 scripts/test-github-environment-main-policy.py create mode 100755 scripts/test-postgres-ci-jit-result.sh create mode 100644 scripts/test-postgres-ci-queue-controller.mjs create mode 100644 scripts/test-pr-ci-check.mjs create mode 100755 scripts/verify-postgres-ci-jit-result.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 628281c..62d93da 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,3 +1,9 @@ self-hosted-runner: labels: - makepad + - makepad-postgres-ci + - makepad-postgres-deploy + - makepad-postgres-pr-ephemeral + - makepad-postgres-ci-attestor + - makepad-postgres-main-ci + - makepad-postgres-release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ac3a6c..2e383b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,42 +1,63 @@ name: CI on: - pull_request: push: branches: [main] + # Workflow code comes from protected main. Same-repository PR code executes + # only on a one-job disposable runner with no environment secrets. + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read jobs: - validate: + validate-pr: name: policy-and-integration - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: [self-hosted, linux, x64, makepad] + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.base.ref == 'main' && + github.event.pull_request.draft == false + runs-on: + group: org/Postgres PR Ephemeral + labels: [self-hosted, linux, x64, makepad-postgres-pr-ephemeral] + timeout-minutes: 45 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: persist-credentials: false - - name: Validate PostgreSQL deployment contract - run: ./scripts/validate-postgres-config.sh - - name: Check deployment shell scripts - run: >- - shellcheck - scripts/run-brio-encrypted-backup.sh - scripts/run-brio-encrypted-backup-loop.sh - scripts/deploy-postgres-stack.sh - scripts/verify-brio-encrypted-restore.sh - scripts/test-brio-bootstrap.sh - scripts/test-brio-encrypted-backup.sh - scripts/test-brio-encrypted-restore.sh - - name: Test idempotent Brio bootstraps - run: ./scripts/test-brio-bootstrap.sh - - name: Test Brio encrypted backup publication - run: ./scripts/test-brio-encrypted-backup.sh - - name: Test Brio encrypted restore safeguards - run: ./scripts/test-brio-encrypted-restore.sh - - name: Test backup contracts in the pinned runtime image + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 1 + - name: Pin checkout to exact internal PR head + shell: bash + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - backup_image=$(grep '^BRIO_BACKUP_IMAGE=' envs/canary/.env.db | cut -d= -f2-) - docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-backup.sh - docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-restore.sh + set -euo pipefail + [[ "${EXPECTED_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "$(git rev-parse HEAD)" == "${EXPECTED_HEAD_SHA}" ]] + - name: Run complete candidate suite in disposable isolation + run: ./scripts/run-ci.sh + + validate-main: + name: protected-main-policy-and-integration + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: + group: org/Postgres Main CI + labels: [self-hosted, linux, x64, makepad-postgres-main-ci] + timeout-minutes: 45 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + fetch-depth: 1 + - name: Pin protected main checkout + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" ]] + [[ "${GITHUB_REF}" == "refs/heads/main" ]] + [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + - name: Run complete protected-main suite + run: ./scripts/run-ci.sh diff --git a/.github/workflows/pr-ci-result.yml b/.github/workflows/pr-ci-result.yml new file mode 100644 index 0000000..a6d4545 --- /dev/null +++ b/.github/workflows/pr-ci-result.yml @@ -0,0 +1,62 @@ +name: Verify signed PR CI teardown + +on: + repository_dispatch: + types: [postgres-pr-ci-attestation] + +permissions: + actions: read + contents: read + pull-requests: read + +concurrency: + group: postgres-pr-ci-attestor-${{ github.event.client_payload.attestation.run.id }}-${{ github.event.client_payload.attestation.run.attempt }} + cancel-in-progress: false + +jobs: + publish: + if: >- + github.repository == 'Makepad-fr/postgres' && + github.ref == 'refs/heads/main' && + github.event.action == 'postgres-pr-ci-attestation' && + github.event.sender.type == 'Bot' && + github.event.sender.id == fromJSON(vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID) + # This persistent attestor is physically separate from the disposable host + # that executes PR code. It has no deployment credentials or Docker access. + runs-on: + group: org/Postgres PR Ephemeral + labels: [self-hosted, linux, x64, makepad-postgres-ci-attestor] + environment: postgres-ci-attestation + timeout-minutes: 5 + steps: + - name: Check out protected attestor source + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + ref: ${{ github.sha }} + fetch-depth: 1 + - name: Verify trusted attestor source and Launcher App sender + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == Makepad-fr/postgres ]] + [[ "${GITHUB_REF}" == refs/heads/main ]] + [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + [[ "${{ github.event.sender.id }}" == "${{ vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID }}" ]] + - name: Verify signed teardown and publish App-bound result + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + POSTGRES_CI_LAUNCHER_APP_SENDER_ID: ${{ vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID }} + POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256: ${{ vars.POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256 }} + POSTGRES_CI_ATTESTATION_PUBLIC_KEY: ${{ vars.POSTGRES_CI_ATTESTATION_PUBLIC_KEY }} + POSTGRES_PR_CHECK_APP_ID: ${{ vars.POSTGRES_PR_CHECK_APP_ID }} + POSTGRES_PR_CHECK_APP_PRIVATE_KEY: ${{ secrets.POSTGRES_PR_CHECK_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + : "${POSTGRES_PR_CHECK_APP_ID:?set protected postgres-ci-attestation App ID}" + : "${POSTGRES_PR_CHECK_APP_PRIVATE_KEY:?set protected postgres-ci-attestation App private key}" + : "${POSTGRES_CI_LAUNCHER_APP_SENDER_ID:?set the immutable Launcher App bot ID}" + : "${POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256:?set the reviewed base-image digest}" + : "${POSTGRES_CI_ATTESTATION_PUBLIC_KEY:?set the Ed25519 hypervisor public key}" + node scripts/publish-pr-ci-check.mjs diff --git a/host/systemd/postgres-ci-queue-alert.service b/host/systemd/postgres-ci-queue-alert.service new file mode 100644 index 0000000..92572be --- /dev/null +++ b/host/systemd/postgres-ci-queue-alert.service @@ -0,0 +1,12 @@ +[Unit] +Description=Alert on Postgres JIT queue-controller failure + +[Service] +Type=oneshot +User=root +Group=root +EnvironmentFile=/etc/makepad/postgres-ci/alert.env +ExecStart=/usr/local/libexec/makepad/send-postgres-host-alert postgres-ci-queue-controller +NoNewPrivileges=true +ProtectHome=true +ProtectSystem=strict diff --git a/host/systemd/postgres-ci-queue-controller.service b/host/systemd/postgres-ci-queue-controller.service new file mode 100644 index 0000000..227b4f1 --- /dev/null +++ b/host/systemd/postgres-ci-queue-controller.service @@ -0,0 +1,22 @@ +[Unit] +Description=Postgres supervised one-job JIT queue controller +After=network-online.target libvirtd.service +Wants=network-online.target +OnFailure=postgres-ci-queue-alert.service + +[Service] +Type=simple +User=root +Group=root +EnvironmentFile=/etc/makepad/postgres-ci/controller.env +ExecStart=/opt/makepad/postgres-ci/current/scripts/run-postgres-ci-queue-controller.sh +Restart=on-failure +RestartSec=15s +KillMode=control-group +NoNewPrivileges=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/run/lock /run /var/lib/makepad/postgres-ci /var/lib/libvirt + +[Install] +WantedBy=multi-user.target diff --git a/scripts/ci-base-image.py b/scripts/ci-base-image.py new file mode 100755 index 0000000..90c2993 --- /dev/null +++ b/scripts/ci-base-image.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + + +def file_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb", buffering=0) as source: + while block := source.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def assert_digest(path: Path, expected: str) -> None: + if file_digest(path) != expected: + raise ValueError("reviewed base-image digest changed") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("path", type=Path) + parser.add_argument("expected") + args = parser.parse_args() + assert_digest(args.path, args.expected) + print(args.expected) + + +if __name__ == "__main__": + main() diff --git a/scripts/configure-postgres-ci-runner-group.sh b/scripts/configure-postgres-ci-runner-group.sh new file mode 100755 index 0000000..a4100cf --- /dev/null +++ b/scripts/configure-postgres-ci-runner-group.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runner labels select a host; these selected-workflow organization groups are +# the policy boundary that prevents branch-authored workflow code from +# selecting the CI attestor. The JIT label is never persistent. + +readonly organization="Makepad-fr" +readonly repository="postgres" +readonly api_version="2022-11-28" + +die() { + printf '%s\n' "$*" >&2 + exit 1 +} + +[[ $# -eq 0 ]] || die "usage: configure-postgres-ci-runner-group.sh < GITHUB_ORG_RUNNER_CONTROLLER_TOKEN" +for command_name in gh python3 sort; do + command -v "${command_name}" >/dev/null || die "${command_name} is required" +done + +IFS= read -r controller_token || die "An organization runner-controller token is required on standard input" +[[ "${controller_token}" =~ ^(github_pat_|ghp_|ghs_|ghu_)[A-Za-z0-9_]+$ ]] || die "The controller token has an invalid format" +export GH_TOKEN="${controller_token}" +unset controller_token + +repository_id=$(gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "repos/${organization}/${repository}" --jq .id) +[[ "${repository_id}" =~ ^[1-9][0-9]*$ ]] || die "Could not resolve the Postgres repository ID" + +groups=( + 'Postgres PR Ephemeral|Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/pr-ci-result.yml@refs/heads/main|makepad-postgres-ci-attestor|makepad-postgres-pr-ephemeral' + 'Postgres Main CI|Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main|makepad-postgres-main-ci|' + 'Postgres Deploy|Makepad-fr/postgres/.github/workflows/manual-deploy.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/deploy-brio-identity-db.yml@refs/heads/main|makepad-postgres-deploy|' + 'Postgres Release|Makepad-fr/postgres/.github/workflows/release-brio-identity-db.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/verify-keycloak-cohort-restores.yml@refs/heads/main|makepad-postgres-release|' +) + +temporary_directory=$(mktemp -d) +chmod 0700 "${temporary_directory}" +cleanup() { + find "${temporary_directory}" -depth -mindepth 1 -delete + rmdir -- "${temporary_directory}" + unset GH_TOKEN +} +trap cleanup EXIT +all_configured_runner_ids="${temporary_directory}/configured-runner-ids" +: >"${all_configured_runner_ids}" +reconciled_groups="${temporary_directory}/reconciled-groups" +: >"${reconciled_groups}" + +# Reconcile every group before checking runner placement. This deliberately +# creates the fail-closed trust domain even when the attestor has not been +# registered yet; validation happens only after the desired state is applied. +for entry in "${groups[@]}"; do + IFS='|' read -r group_name selected_workflows required_labels forbidden_persistent_labels <<<"${entry}" + group_list="${temporary_directory}/groups.json" + gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups?per_page=100" >"${group_list}" + group_id=$(python3 - "${group_list}" "${group_name}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +groups = payload.get("runner_groups", []) +if payload.get("total_count", len(groups)) > len(groups): + raise SystemExit("more than 100 organization runner groups require explicit pagination") +matches = [group.get("id") for group in groups if group.get("name") == sys.argv[2]] +if len(matches) > 1: + raise SystemExit(f"duplicate runner groups named {sys.argv[2]}") +if matches: + print(matches[0]) +PY + ) + + payload=$(python3 - "${group_name}" "${selected_workflows}" "${repository_id}" <<'PY' +import json +import sys + +print(json.dumps({ + "name": sys.argv[1], + "visibility": "selected", + "allows_public_repositories": True, + "restricted_to_workflows": True, + "selected_workflows": sys.argv[2].split(","), + "selected_repository_ids": [int(sys.argv[3])], +}, separators=(",", ":"))) +PY + ) + + if [[ -z "${group_id}" ]]; then + created=$(printf '%s' "${payload}" | gh api --method POST \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups" --input -) + group_id=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' <<<"${created}") + else + update_payload=$(python3 - "${group_name}" "${selected_workflows}" <<'PY' +import json +import sys + +print(json.dumps({ + "name": sys.argv[1], + "visibility": "selected", + "allows_public_repositories": True, + "restricted_to_workflows": True, + "selected_workflows": sys.argv[2].split(","), +}, separators=(",", ":"))) +PY + ) + printf '%s' "${update_payload}" | gh api --method PATCH \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}" --input - >/dev/null + fi + [[ "${group_id}" =~ ^[1-9][0-9]*$ ]] || die "Invalid runner group ID for ${group_name}" + + gh api --method PUT --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}/repositories/${repository_id}" >/dev/null + + repositories="${temporary_directory}/repositories-${group_id}.json" + gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}/repositories?per_page=100" >"${repositories}" + while IFS= read -r unrelated_repository_id; do + [[ -z "${unrelated_repository_id}" ]] || gh api --method DELETE \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}/repositories/${unrelated_repository_id}" >/dev/null + done < <(python3 - "${repositories}" "${repository_id}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +repositories = payload.get("repositories", []) +if payload.get("total_count", len(repositories)) > len(repositories): + raise SystemExit("more than 100 selected repositories require explicit pagination") +expected = int(sys.argv[2]) +for repository in repositories: + repository_id = repository.get("id") + if isinstance(repository_id, int) and repository_id != expected: + print(repository_id) +PY + ) + + printf '%s|%s|%s|%s|%s\n' "${group_name}" "${group_id}" "${selected_workflows}" "${required_labels}" \ + "${forbidden_persistent_labels}" \ + >>"${reconciled_groups}" + printf 'Reconciled runner-group configuration %s (%s).\n' "${group_name}" "${group_id}" +done + +# Read back every group only after the complete reconciliation pass. A missing +# host can therefore fail bootstrap without preventing creation or repair of a +# later group. +while IFS='|' read -r group_name group_id selected_workflows required_labels forbidden_persistent_labels; do + observed_group="${temporary_directory}/group-${group_id}.json" + observed_repositories="${temporary_directory}/observed-repositories-${group_id}.json" + observed_runners="${temporary_directory}/runners-${group_id}.json" + gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}" >"${observed_group}" + gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}/repositories?per_page=100" >"${observed_repositories}" + gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${group_id}/runners?per_page=100" >"${observed_runners}" + python3 - "${observed_group}" "${observed_repositories}" "${observed_runners}" \ + "${group_name}" "${selected_workflows}" "${required_labels}" "${repository_id}" \ + "${all_configured_runner_ids}" "${forbidden_persistent_labels}" <<'PY' +import json +import pathlib +import sys + +group = json.loads(pathlib.Path(sys.argv[1]).read_text()) +repository_payload = json.loads(pathlib.Path(sys.argv[2]).read_text()) +runner_payload = json.loads(pathlib.Path(sys.argv[3]).read_text()) +repositories = repository_payload.get("repositories", []) +runners = runner_payload.get("runners", []) +expected = { + "name": sys.argv[4], + "visibility": "selected", + "allows_public_repositories": True, + "restricted_to_workflows": True, + "workflow_restrictions_read_only": False, +} +for key, value in expected.items(): + if group.get(key) != value: + raise SystemExit(f"runner group {sys.argv[4]} has unexpected {key}: {group.get(key)!r}") +if sorted(group.get("selected_workflows", [])) != sorted(sys.argv[5].split(",")): + raise SystemExit(f"runner group {sys.argv[4]} has unexpected selected_workflows") +if repository_payload.get("total_count", len(repositories)) > len(repositories): + raise SystemExit(f"runner group {sys.argv[4]} has more than 100 selected repositories") +if [repository.get("id") for repository in repositories] != [int(sys.argv[7])]: + raise SystemExit(f"runner group {sys.argv[4]} is not restricted to Postgres") +if runner_payload.get("total_count", len(runners)) > len(runners): + raise SystemExit(f"runner group {sys.argv[4]} has more than 100 runners") +available_labels = { + label.get("name").lower() + for runner in runners + for label in runner.get("labels", []) + if isinstance(label.get("name"), str) +} +missing = sorted(set(sys.argv[6].split(",")) - available_labels) +if missing: + raise SystemExit(f"runner group {sys.argv[4]} has no host for labels: {', '.join(missing)}") +required = sys.argv[6].split(",") +forbidden_persistent = {label for label in sys.argv[9].split(",") if label} +present_forbidden = sorted(forbidden_persistent & available_labels) +if present_forbidden: + raise SystemExit( + f"runner group {sys.argv[4]} has a persistent runner carrying JIT-only labels: " + f"{', '.join(present_forbidden)}" + ) +label_owners = { + label: { + runner.get("id") + for runner in runners + if label in { + str(item.get("name", "")).lower() + for item in runner.get("labels", []) + if isinstance(item.get("name"), str) + } + } + for label in required +} +default_labels = {"self-hosted", "linux", "x64", "makepad"} +for runner in runners: + labels = { + str(item.get("name", "")).lower() + for item in runner.get("labels", []) + if isinstance(item.get("name"), str) + } + owned = set(required) & labels + unexpected = labels - default_labels - set(required) - forbidden_persistent + if len(owned) != 1 or unexpected: + raise SystemExit( + f"runner group {sys.argv[4]} contains an unapproved runner/label set on " + f"{runner.get('name', runner.get('id'))}: {sorted(labels)}" + ) +for label, owners in label_owners.items(): + if len(owners) != 1: + raise SystemExit( + f"runner group {sys.argv[4]} must have exactly one persistent host for {label}" + ) +for index, label in enumerate(required): + for other in required[index + 1:]: + if label_owners[label] & label_owners[other]: + raise SystemExit( + f"runner group {sys.argv[4]} places mutually trusted labels " + f"{label} and {other} on the same host" + ) +with pathlib.Path(sys.argv[8]).open("a", encoding="utf-8") as destination: + for runner in runners: + runner_id = runner.get("id") + if isinstance(runner_id, int): + destination.write(f"{runner_id}\n") +PY + printf 'Validated runner group %s (%s).\n' "${group_name}" "${group_id}" +done <"${reconciled_groups}" + +# Repository-level or unrelated organization runners bypass these two groups. +# Refuse to declare bootstrap complete while any such runner is available. +accessible_runners="${temporary_directory}/repository-runners.json" +gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "repos/${organization}/${repository}/actions/runners?per_page=100" >"${accessible_runners}" +python3 - "${accessible_runners}" "${all_configured_runner_ids}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +runners = payload.get("runners", []) +if payload.get("total_count", len(runners)) > len(runners): + raise SystemExit("more than 100 Postgres-accessible runners require explicit pagination") +configured = { + int(value) + for value in pathlib.Path(sys.argv[2]).read_text().splitlines() + if value.strip() +} +unexpected = [runner for runner in runners if runner.get("id") not in configured] +if unexpected: + names = ", ".join(str(runner.get("name", runner.get("id"))) for runner in unexpected) + raise SystemExit(f"Postgres still exposes runners outside its restricted groups: {names}") +PY + +printf 'Postgres runner access is restricted to the four selected-workflow groups.\n' diff --git a/scripts/dispatch-ci-attestation.mjs b/scripts/dispatch-ci-attestation.mjs new file mode 100644 index 0000000..480c237 --- /dev/null +++ b/scripts/dispatch-ci-attestation.mjs @@ -0,0 +1,35 @@ +#!/usr/bin/env node +import { sign } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import process from "node:process"; +import { canonicalJSON } from "./publish-pr-ci-check.mjs"; + +const evidencePath = process.env.POSTGRES_CI_ATTESTATION_JSON_FILE; +const privateKeyPath = process.env.POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE; +if (!evidencePath?.startsWith("/") || !privateKeyPath?.startsWith("/")) throw new Error("absolute attestation and private-key paths are required"); +const [source, privateKey] = await Promise.all([readFile(evidencePath, "utf8"), readFile(privateKeyPath, "utf8")]); +const attestation = JSON.parse(source); +const canonical = canonicalJSON(attestation); +if (`${canonical}\n` !== source && canonical !== source) throw new Error("attestation file is not canonical JSON"); + +let token = ""; +for await (const chunk of process.stdin) token += chunk; +token = token.trim(); +if (!/^ghs_[A-Za-z0-9_]{20,}$/.test(token)) throw new Error("a dedicated Launcher App installation token is required on stdin"); +const signature = sign(null, Buffer.from(canonical), privateKey).toString("base64url"); +const response = await fetch("https://api.github.com/repos/Makepad-fr/postgres/dispatches", { + method: "POST", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "User-Agent": "makepad-postgres-ci-launcher", + "X-GitHub-Api-Version": "2022-11-28", + }, + body: JSON.stringify({event_type: "postgres-pr-ci-attestation", client_payload: {attestation, signature}}), + redirect: "error", + signal: AbortSignal.timeout(30_000), +}); +token = ""; +if (response.status !== 204) throw new Error(`Launcher App attestation dispatch failed with ${response.status}`); +process.stdout.write(`Dispatched signed teardown attestation for run ${attestation.run.id}, attempt ${attestation.run.attempt}.\n`); diff --git a/scripts/postgres-ci-queue-controller.mjs b/scripts/postgres-ci-queue-controller.mjs new file mode 100644 index 0000000..3899858 --- /dev/null +++ b/scripts/postgres-ci-queue-controller.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node +import crypto, { createSign } from "node:crypto"; +import { spawn } from "node:child_process"; +import { lstat, mkdir, readFile, realpath, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const REPOSITORY = "Makepad-fr/postgres"; +const WORKFLOW_PATH = ".github/workflows/ci.yml"; +const LABELS = ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]; + +const required = (name, env = process.env) => { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +}; + +const github = async ({ token, method = "GET", endpoint, body, fetchImpl = fetch }) => { + const response = await fetchImpl(`https://api.github.com${endpoint}`, { + method, + headers: {Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": "makepad-postgres-ci-controller", "X-GitHub-Api-Version": "2022-11-28"}, + body: body === undefined ? undefined : JSON.stringify(body), + redirect: "error", + signal: AbortSignal.timeout(30_000), + }); + const text = await response.text(); + let payload = {}; + if (text) payload = JSON.parse(text); + if (!response.ok) throw new Error(`GitHub ${method} ${endpoint} failed with ${response.status}`); + return payload; +}; + +const appJWT = ({appID, key, now = Date.now()}) => { + if (!/^[1-9]\d*$/.test(appID)) throw new Error("Launcher App ID must be numeric"); + const issued = Math.floor(now / 1000) - 60; + const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const unsigned = `${encode({alg: "RS256", typ: "JWT"})}.${encode({iat: issued, exp: issued + 540, iss: appID})}`; + const signer = createSign("RSA-SHA256"); + signer.update(unsigned); + signer.end(); + return `${unsigned}.${signer.sign(key, "base64url")}`; +}; + +export const selectAuthorizedJobs = ({ runs, jobsByRun, pullRequests, repositoryID }) => { + if (!Array.isArray(runs.workflow_runs) || runs.total_count !== runs.workflow_runs.length) throw new Error("workflow-run response is truncated"); + const selected = []; + for (const run of runs.workflow_runs) { + if (!Number.isSafeInteger(run.id) || run.id <= 0 || !Number.isSafeInteger(run.run_attempt) || run.run_attempt <= 0) continue; + const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; + if (run.name !== "CI" || run.path !== WORKFLOW_PATH || run.status !== "queued" || run.repository?.id !== repositoryID || !/^[a-f0-9]{40}$/.test(run.head_sha || "")) continue; + let sourceSHA; + let pullRequestNumber = null; + if (run.event === "pull_request_target") { + if (associations.length !== 1 || !Number.isSafeInteger(associations[0]?.number)) continue; + const association = associations[0]; + const pull = pullRequests.get(association.number); + if (association.head?.repo?.id !== repositoryID || association.base?.repo?.id !== repositoryID || association.base?.ref !== "main" || association.base?.sha !== run.head_sha || !/^[a-f0-9]{40}$/.test(association.head?.sha || "") || pull?.number !== association.number || pull?.head?.sha !== association.head?.sha || pull?.head?.repo?.id !== repositoryID || pull?.base?.repo?.id !== repositoryID || pull?.base?.ref !== "main" || pull?.base?.sha !== run.head_sha) continue; + sourceSHA = association.head.sha; + pullRequestNumber = association.number; + } else if (run.event === "push") { + if (run.head_branch !== "main") continue; + sourceSHA = run.head_sha; + } else { + continue; + } + const response = jobsByRun.get(`${run.id}:${run.run_attempt}`); + if (!response || !Array.isArray(response.jobs) || response.total_count !== response.jobs.length) throw new Error("workflow-job response is missing or truncated"); + for (const job of response.jobs) { + const labels = Array.isArray(job.labels) ? job.labels.map((value) => String(value).toLowerCase()).sort() : []; + if (Number.isSafeInteger(job.id) && job.id > 0 && job.name === "policy-and-integration" && job.status === "queued" && job.run_id === run.id && job.head_sha === run.head_sha && job.workflow_name === "CI" && labels.length === LABELS.length && labels.every((value, index) => value === [...LABELS].sort()[index])) { + selected.push({runID: run.id, attempt: run.run_attempt, jobID: job.id, event: run.event, sourceSHA, workflowSHA: run.head_sha, pullRequestNumber}); + } + } + } + return selected.sort((left, right) => left.jobID - right.jobID); +}; + +const atomicState = async (file, state) => { + const incoming = `${file}.incoming-${process.pid}-${crypto.randomBytes(8).toString("hex")}`; + await writeFile(incoming, `${JSON.stringify(state, null, 2)}\n`, {mode: 0o600, flag: "wx"}); + await rename(incoming, file); +}; + +const runLauncher = ({launcher, token, metadata, environment, arguments: launcherArguments = []}) => new Promise((resolve, reject) => { + const child = spawn(launcher, launcherArguments, { + env: {...environment, POSTGRES_CI_RUN_ID: String(metadata.runID || ""), POSTGRES_CI_RUN_ATTEMPT: String(metadata.attempt || ""), POSTGRES_CI_JOB_ID: String(metadata.jobID || ""), POSTGRES_CI_RUN_EVENT: metadata.event || "", POSTGRES_CI_HEAD_SHA: metadata.sourceSHA || "", POSTGRES_CI_WORKFLOW_SHA: metadata.workflowSHA || "", POSTGRES_CI_ATTESTATION_NONCE: metadata.nonce || "", POSTGRES_CI_LAUNCH_ID: metadata.launchID || ""}, + stdio: ["pipe", "inherit", "inherit"], + }); + child.stdin.end(`${token}\n`); + child.once("error", reject); + child.once("exit", (code, signal) => code === 0 && signal === null ? resolve() : reject(new Error(`launcher exited ${code ?? signal}`))); +}); + +export const reconcileIncompleteJobs = async ({state, persist, reconcile}) => { + for (const [jobID, record] of Object.entries(state.jobs).sort(([left], [right]) => Number(left) - Number(right))) { + if (!record || !["launching", "recovery-required"].includes(record.status)) continue; + if (!/^j[1-9][0-9]{0,15}-[a-f0-9]{16}$/.test(record.launchID || "")) { + throw new Error(`incomplete job ${jobID} has no safe deterministic resource manifest`); + } + try { + await reconcile(record); + record.status = "failed-recovered"; + record.failure = "controller restart reconciled an incomplete disposable launch"; + record.finishedAt = new Date().toISOString(); + await persist(); + } catch (error) { + record.status = "recovery-required"; + record.failure = error instanceof Error ? error.message.slice(0, 200) : "unknown reconciliation failure"; + await persist(); + throw error; + } + } +}; + +export const controller = async ({environment = process.env, fetchImpl = fetch, once = false} = {}) => { + if (process.getuid?.() !== 0) throw new Error("queue controller must run as root on the dedicated hypervisor"); + const repositoryID = Number(required("POSTGRES_CI_REPOSITORY_ID", environment)); + const appID = required("POSTGRES_CI_LAUNCHER_APP_ID", environment); + const installationID = required("POSTGRES_CI_LAUNCHER_APP_INSTALLATION_ID", environment); + const privateKeyFile = required("POSTGRES_CI_LAUNCHER_APP_PRIVATE_KEY_FILE", environment); + const stateDirectory = required("POSTGRES_CI_CONTROLLER_STATE_DIRECTORY", environment); + const launcher = required("POSTGRES_CI_LAUNCHER", environment); + if (!Number.isSafeInteger(repositoryID) || repositoryID <= 0 || !/^[1-9]\d*$/.test(installationID)) throw new Error("repository and installation IDs must be positive integers"); + if (!/^\/var\/lib\/makepad\/postgres-ci\/[A-Za-z0-9._/-]+$/.test(stateDirectory) || stateDirectory.includes("..") || path.normalize(stateDirectory) !== stateDirectory) throw new Error("controller state directory is outside the root-owned Postgres PR Ephemeral tree"); + for (const [file, expectedMode] of [[privateKeyFile, 0o400], [launcher, 0o755]]) { + if (!path.isAbsolute(file)) throw new Error(`controller file is not absolute: ${file}`); + const value = await lstat(file); + if (!value.isFile() || value.isSymbolicLink() || value.uid !== 0 || (value.mode & 0o777) !== expectedMode || await realpath(file) !== file) throw new Error(`insecure controller file: ${file}`); + } + const key = await readFile(privateKeyFile, "utf8"); + await mkdir(stateDirectory, {recursive: true, mode: 0o700}); + const directory = await lstat(stateDirectory); + if (!directory.isDirectory() || directory.isSymbolicLink() || directory.uid !== 0 || (directory.mode & 0o777) !== 0o700 || await realpath(stateDirectory) !== stateDirectory) throw new Error("controller state directory must be a root-only real path"); + const stateFile = path.join(stateDirectory, "jobs.json"); + let state = {version: 2, jobs: {}}; + try { state = JSON.parse(await readFile(stateFile, "utf8")); } catch (error) { if (error.code !== "ENOENT") throw error; } + if (state.version === 1 && state.jobs && typeof state.jobs === "object") { + if (Object.values(state.jobs).some((record) => record?.status === "launching")) throw new Error("legacy controller state contains an unreconciled launch; operator recovery is required"); + state = {version: 2, jobs: state.jobs}; + await atomicState(stateFile, state); + } + if (state.version !== 2 || !state.jobs || typeof state.jobs !== "object" || Array.isArray(state.jobs)) throw new Error("controller state is invalid"); + + do { + const jwt = appJWT({appID, key}); + const installation = await github({token: jwt, method: "POST", endpoint: `/app/installations/${installationID}/access_tokens`, body: {repositories: ["postgres"], permissions: {actions: "read", contents: "write", issues: "write", organization_self_hosted_runners: "write", pull_requests: "read"}}, fetchImpl}); + const token = installation.token; + if (typeof token !== "string" || !token.startsWith("ghs_")) throw new Error("Launcher App did not issue an installation token"); + await reconcileIncompleteJobs({ + state, + persist: () => atomicState(stateFile, state), + reconcile: (record) => runLauncher({launcher, token, metadata: record, environment, arguments: ["--reconcile", record.launchID]}), + }); + const runs = await github({token, endpoint: `/repos/${REPOSITORY}/actions/workflows/ci.yml/runs?status=queued&per_page=100`, fetchImpl}); + const jobsByRun = new Map(); + const pullRequests = new Map(); + for (const run of runs.workflow_runs || []) { + jobsByRun.set(`${run.id}:${run.run_attempt}`, await github({token, endpoint: `/repos/${REPOSITORY}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100`, fetchImpl})); + const number = run.pull_requests?.[0]?.number; + if (Number.isSafeInteger(number) && !pullRequests.has(number)) pullRequests.set(number, await github({token, endpoint: `/repos/${REPOSITORY}/pulls/${number}`, fetchImpl})); + } + const pending = selectAuthorizedJobs({runs, jobsByRun, pullRequests, repositoryID}).filter((job) => !state.jobs[String(job.jobID)]); + for (const job of pending) { + const nonce = crypto.randomBytes(32).toString("base64url"); + const launchID = `j${job.jobID}-${crypto.randomBytes(8).toString("hex")}`; + state.jobs[String(job.jobID)] = {...job, nonce, launchID, status: "launching", createdAt: new Date().toISOString()}; + await atomicState(stateFile, state); + try { + await runLauncher({launcher, token, metadata: {...job, nonce, launchID}, environment}); + state.jobs[String(job.jobID)].status = "completed"; + } catch (error) { + // Any nonzero launcher exit is cleanup-uncertain. Keep the deterministic + // launch identity eligible for mandatory startup reconciliation; the + // controller must never convert an uncertain launch into terminal state. + state.jobs[String(job.jobID)].status = "recovery-required"; + state.jobs[String(job.jobID)].failure = error instanceof Error ? error.message.slice(0, 200) : "unknown launcher failure"; + state.jobs[String(job.jobID)].recoveryRequiredAt = new Date().toISOString(); + // Persist the no-retry decision before any network alert. Exiting + // nonzero then activates the independent host OnFailure channel; the + // GitHub issue below is useful secondary evidence, not the sole alert. + await atomicState(stateFile, state); + const title = `Postgres JIT launcher failed for job ${job.jobID}`; + try { + await github({token, method: "POST", endpoint: `/repos/${REPOSITORY}/issues`, body: {title, body: `The supervised hypervisor controller could not complete run ${job.runID}, attempt ${job.attempt}, job ${job.jobID}. No success attestation was issued. Inspect the root-only hypervisor journal.`}, fetchImpl}); + } catch { + // The systemd OnFailure webhook remains independent of GitHub. + } + throw error; + } + state.jobs[String(job.jobID)].finishedAt = new Date().toISOString(); + await atomicState(stateFile, state); + } + if (once) break; + const pollSeconds = Math.min(120, Math.max(15, Number(environment.POSTGRES_CI_POLL_SECONDS || 30))); + await new Promise((resolve) => setTimeout(resolve, pollSeconds * 1000)); + } while (true); +}; + +const invokedAsCLI = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedAsCLI) controller({once: process.argv.includes("--once")}).catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/scripts/publish-pr-ci-check.mjs b/scripts/publish-pr-ci-check.mjs new file mode 100644 index 0000000..a5e7e9d --- /dev/null +++ b/scripts/publish-pr-ci-check.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +import { createSign, verify as verifySignature } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +const EXPECTED_REPOSITORY = "Makepad-fr/postgres"; +const EXPECTED_WORKFLOW_PATH = ".github/workflows/ci.yml"; +const EXPECTED_WORKFLOW_NAME = "CI"; +const EXPECTED_RUNNER_GROUP = "Postgres PR Ephemeral"; +const EXPECTED_RUNNER_LABELS = ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]; +const EXPECTED_SCHEMA = "makepad.postgres.ci-attestation.v1"; +const MAX_ATTESTATION_AGE_MS = 10 * 60 * 1000; +const MAX_FUTURE_SKEW_MS = 60 * 1000; +export const CHECK_NAMES = ["postgres-ci"]; + +const required = (name, environment = process.env) => { + const value = environment[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +}; + +const exactKeys = (value, keys, label) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`); + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) throw new Error(`${label} has unexpected fields`); +}; + +export const canonicalJSON = (value) => { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number" && Number.isSafeInteger(value)) return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJSON).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJSON(value[key])}`).join(",")}}`; + } + throw new Error("attestation contains a non-canonical JSON value"); +}; + +const base64url = (value) => Buffer.from(value).toString("base64url"); + +export const createAppJWT = ({ appID, privateKey, now = new Date() }) => { + if (!/^[1-9]\d*$/.test(appID)) throw new Error("GitHub App ID must be a positive integer"); + if (!/^-----BEGIN (?:RSA )?PRIVATE KEY-----/.test(privateKey.trim())) throw new Error("GitHub App private key must be a PEM private key"); + const issuedAt = Math.floor(now.getTime() / 1000) - 60; + const header = base64url(JSON.stringify({ alg: "RS256", typ: "JWT" })); + const payload = base64url(JSON.stringify({ iat: issuedAt, exp: issuedAt + 540, iss: appID })); + const unsigned = `${header}.${payload}`; + const signer = createSign("RSA-SHA256"); + signer.update(unsigned); + signer.end(); + return `${unsigned}.${signer.sign(privateKey, "base64url")}`; +}; + +const githubResponse = async ({ token, method = "GET", path, body, fetchImpl = fetch }) => { + const response = await fetchImpl(`https://api.github.com${path}`, { + method, + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28" + }, + body: body === undefined ? undefined : JSON.stringify(body), + redirect: "error", + signal: AbortSignal.timeout(30_000) + }); + const text = await response.text(); + let payload = {}; + if (text) { + try { payload = JSON.parse(text); } + catch { throw new Error(`GitHub ${method} ${path} returned non-JSON (${response.status})`); } + } + return { ok: response.ok, status: response.status, payload }; +}; + +const githubJSON = async (options) => { + const response = await githubResponse(options); + if (!response.ok) { + const message = typeof response.payload.message === "string" ? response.payload.message : "request failed"; + throw new Error(`GitHub ${options.method || "GET"} ${options.path} failed (${response.status}): ${message}`); + } + return response.payload; +}; + +export const verifySignedAttestation = ({ event, publicKey, approvedDigest, launcherSenderID, now = new Date() }) => { + if (!/^[1-9]\d*$/.test(String(launcherSenderID))) throw new Error("Launcher App sender ID must be a positive integer"); + if (event?.action !== "postgres-pr-ci-attestation") throw new Error("unexpected repository dispatch action"); + if (event?.repository?.full_name !== EXPECTED_REPOSITORY) throw new Error("attestation targets the wrong repository"); + if (event?.sender?.type !== "Bot" || String(event?.sender?.id) !== String(launcherSenderID)) throw new Error("attestation dispatch was not sent by the dedicated Launcher App"); + exactKeys(event.client_payload, ["attestation", "signature"], "dispatch payload"); + const attestation = event.client_payload.attestation; + const signature = event.client_payload.signature; + exactKeys(attestation, ["schema", "repository", "workflow", "ref", "run", "runner", "base_image_sha256", "nonce", "issued_at", "registration_absent", "teardown"], "attestation"); + exactKeys(attestation.workflow, ["name", "path"], "attestation workflow"); + exactKeys(attestation.run, ["id", "attempt", "job_id", "job_name", "event", "head_sha", "workflow_sha", "conclusion"], "attestation run"); + exactKeys(attestation.runner, ["id", "name", "group_id", "group_name", "labels"], "attestation runner"); + exactKeys(attestation.teardown, ["vm", "network", "firewall", "disk"], "attestation teardown"); + if (attestation.schema !== EXPECTED_SCHEMA || attestation.repository !== EXPECTED_REPOSITORY) throw new Error("attestation schema or repository mismatch"); + if (attestation.workflow.name !== EXPECTED_WORKFLOW_NAME || attestation.workflow.path !== EXPECTED_WORKFLOW_PATH || attestation.ref !== "refs/heads/main") throw new Error("attestation workflow or protected ref mismatch"); + for (const [label, value] of Object.entries({run_id: attestation.run.id, run_attempt: attestation.run.attempt, job_id: attestation.run.job_id, runner_id: attestation.runner.id, runner_group_id: attestation.runner.group_id})) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${label} must be a positive safe integer`); + } + if (attestation.run.job_name !== "policy-and-integration" || !["pull_request_target", "push"].includes(attestation.run.event) || !/^[a-f0-9]{40}$/.test(attestation.run.head_sha) || !/^[a-f0-9]{40}$/.test(attestation.run.workflow_sha) || (attestation.run.event === "push" && attestation.run.head_sha !== attestation.run.workflow_sha) || !["success", "failure"].includes(attestation.run.conclusion)) throw new Error("attested job identity or conclusion is invalid"); + if (!/^postgres-ci-jit-[a-z0-9-]{8,80}$/.test(attestation.runner.name) || attestation.runner.group_name !== EXPECTED_RUNNER_GROUP) throw new Error("attested runner identity is invalid"); + const labels = Array.isArray(attestation.runner.labels) ? attestation.runner.labels : []; + if (labels.length !== EXPECTED_RUNNER_LABELS.length || labels.some((label, index) => label !== EXPECTED_RUNNER_LABELS[index])) throw new Error("attested runner labels are not the exact JIT label set"); + if (attestation.base_image_sha256 !== approvedDigest || !/^[a-f0-9]{64}$/.test(approvedDigest)) throw new Error("attested base image digest is not approved"); + if (!/^[A-Za-z0-9_-]{43}$/.test(attestation.nonce)) throw new Error("attestation nonce is invalid"); + if (typeof attestation.issued_at !== "string" || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(attestation.issued_at)) throw new Error("attestation issued_at must be canonical UTC RFC 3339"); + const issuedAt = Date.parse(attestation.issued_at); + if (!Number.isFinite(issuedAt) || issuedAt < now.getTime() - MAX_ATTESTATION_AGE_MS || issuedAt > now.getTime() + MAX_FUTURE_SKEW_MS) throw new Error("attestation is stale or from the future"); + if (attestation.registration_absent !== true || Object.values(attestation.teardown).some((value) => value !== true)) throw new Error("runner registration or hypervisor teardown is incomplete"); + if (typeof signature !== "string" || !/^[A-Za-z0-9_-]{80,100}$/.test(signature)) throw new Error("attestation signature is invalid"); + let signatureValid = false; + try { signatureValid = verifySignature(null, Buffer.from(canonicalJSON(attestation)), publicKey, Buffer.from(signature, "base64url")); } + catch { signatureValid = false; } + if (!signatureValid) throw new Error("attestation signature verification failed"); + return attestation; +}; + +export const validateAuthoritativeEvidence = ({ attestation, run, jobs, job, pullRequest = null, runnerListStatus = 200, runnerLookupStatus }) => { + if (run.id !== attestation.run.id || run.run_attempt !== attestation.run.attempt || run.event !== attestation.run.event || run.head_sha !== attestation.run.workflow_sha || run.head_branch !== "main" || run.path !== EXPECTED_WORKFLOW_PATH || run.name !== EXPECTED_WORKFLOW_NAME || run.status !== "completed" || run.repository?.full_name !== EXPECTED_REPOSITORY || !Number.isSafeInteger(run.repository?.id)) throw new Error("authoritative workflow run does not match the attestation"); + if (jobs.total_count !== (jobs.jobs || []).length) throw new Error("authoritative job response is truncated"); + const matches = (jobs.jobs || []).filter((candidate) => candidate.id === attestation.run.job_id); + if (matches.length !== 1 || matches[0].id !== job.id) throw new Error("attested job is not unique in the authoritative run attempt"); + const labels = Array.isArray(job.labels) ? job.labels.map((label) => String(label).toLowerCase()).sort() : []; + const expectedLabels = [...EXPECTED_RUNNER_LABELS].sort(); + if (job.run_id !== run.id || job.head_sha !== attestation.run.workflow_sha || job.workflow_name !== EXPECTED_WORKFLOW_NAME || job.name !== attestation.run.job_name || job.status !== "completed" || job.runner_id !== attestation.runner.id || job.runner_name !== attestation.runner.name || job.runner_group_id !== attestation.runner.group_id || job.runner_group_name !== attestation.runner.group_name || labels.length !== expectedLabels.length || labels.some((label, index) => label !== expectedLabels[index])) throw new Error("authoritative job runner identity differs from the signed attestation"); + const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; + let pullRequestNumber = null; + if (attestation.run.event === "pull_request_target") { + if (associations.length !== 1) throw new Error("source run must identify exactly one pull request"); + const association = associations[0]; + if (association.head?.sha !== attestation.run.head_sha || association.head?.repo?.id !== run.repository?.id || association.base?.repo?.id !== run.repository?.id || association.base?.ref !== "main" || association.base?.sha !== attestation.run.workflow_sha || pullRequest?.number !== association.number || pullRequest.head?.sha !== attestation.run.head_sha || pullRequest.head?.repo?.full_name !== EXPECTED_REPOSITORY || pullRequest.base?.sha !== attestation.run.workflow_sha || pullRequest.base?.repo?.full_name !== EXPECTED_REPOSITORY || pullRequest.base?.ref !== "main") throw new Error("authoritative pull request differs from the signed head and base identities"); + pullRequestNumber = pullRequest.number; + } else if (attestation.run.head_sha !== attestation.run.workflow_sha) { + throw new Error("protected-main push source differs from its workflow SHA"); + } + const expectedConclusion = run.conclusion === "success" && job.conclusion === "success" ? "success" : "failure"; + if (attestation.run.conclusion !== expectedConclusion) throw new Error("signed conclusion differs from authoritative test result"); + if (runnerListStatus !== 200 || runnerLookupStatus !== 404) throw new Error("attested JIT runner is still registered or registration absence is uncertain"); + return { event: attestation.run.event, headSHA: attestation.run.head_sha, workflowSHA: attestation.run.workflow_sha, pullRequestNumber, conclusion: expectedConclusion, sourceRunID: run.id, sourceRunAttempt: run.run_attempt, detailsURL: run.html_url, nonce: attestation.nonce }; +}; + +export const assertNoAttestationReplay = ({existing, appID, prefix}) => { + if (!Number.isSafeInteger(existing.total_count) || existing.total_count !== (existing.check_runs || []).length) throw new Error("cannot prove Postgres PR Ephemeral replay protection"); + if ((existing.check_runs || []).some((check) => String(check.app?.id) === String(appID) && String(check.external_id || "").startsWith(prefix))) throw new Error("attestation replay detected for this run attempt"); +}; + +export const publishPRCheck = async ({ environment = process.env, fetchImpl = fetch, now = new Date() } = {}) => { + if (required("GITHUB_REPOSITORY", environment) !== EXPECTED_REPOSITORY || required("GITHUB_REF", environment) !== "refs/heads/main") throw new Error("PR attestation must run for Makepad-fr/postgres protected main"); + const event = JSON.parse(await readFile(required("GITHUB_EVENT_PATH", environment), "utf8")); + const attestation = verifySignedAttestation({ event, publicKey: required("POSTGRES_CI_ATTESTATION_PUBLIC_KEY", environment), approvedDigest: required("POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256", environment), launcherSenderID: required("POSTGRES_CI_LAUNCHER_APP_SENDER_ID", environment), now }); + const repositoryToken = required("GITHUB_TOKEN", environment); + const run = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/runs/${attestation.run.id}`, fetchImpl }); + const jobs = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/runs/${attestation.run.id}/attempts/${attestation.run.attempt}/jobs?per_page=100`, fetchImpl }); + const job = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/jobs/${attestation.run.job_id}`, fetchImpl }); + const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; + let pullRequest = null; + if (attestation.run.event === "pull_request_target") { + if (associations.length !== 1 || !Number.isSafeInteger(associations[0]?.number)) throw new Error("source run has no unique pull request association"); + pullRequest = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/pulls/${associations[0].number}`, fetchImpl }); + } + + const appID = required("POSTGRES_PR_CHECK_APP_ID", environment); + const appJWT = createAppJWT({ appID, privateKey: required("POSTGRES_PR_CHECK_APP_PRIVATE_KEY", environment), now }); + const installation = await githubJSON({ token: appJWT, path: `/repos/${EXPECTED_REPOSITORY}/installation`, fetchImpl }); + if (String(installation.app_id) !== appID || !Number.isSafeInteger(installation.id)) throw new Error("configured Checks App is not the Postgres installation"); + const installationToken = await githubJSON({ token: appJWT, method: "POST", path: `/app/installations/${installation.id}/access_tokens`, body: { repositories: ["postgres"], permissions: { checks: "write", organization_self_hosted_runners: "read" } }, fetchImpl }); + if (typeof installationToken.token !== "string" || !installationToken.token) throw new Error("Checks App installation did not issue a token"); + const runnerList = await githubResponse({ token: installationToken.token, path: "/orgs/Makepad-fr/actions/runners?per_page=1", fetchImpl }); + const runnerLookup = await githubResponse({ token: installationToken.token, path: `/orgs/Makepad-fr/actions/runners/${attestation.runner.id}`, fetchImpl }); + const verified = validateAuthoritativeEvidence({ attestation, run, jobs, job, pullRequest, runnerListStatus: runnerList.status, runnerLookupStatus: runnerLookup.status }); + + const externalID = `postgres-ci:${verified.event}:${verified.sourceRunID}:${verified.sourceRunAttempt}:${verified.nonce}`; + const checkRunIDs = {}; + for (const checkName of CHECK_NAMES) { + const existing = await githubJSON({ token: installationToken.token, path: `/repos/${EXPECTED_REPOSITORY}/commits/${verified.headSHA}/check-runs?check_name=${encodeURIComponent(checkName)}&filter=all&per_page=100`, fetchImpl }); + const prefix = `postgres-ci:${verified.event}:${verified.sourceRunID}:${verified.sourceRunAttempt}:`; + assertNoAttestationReplay({existing, appID, prefix}); + const scope = verified.event === "pull_request_target" ? `PR #${verified.pullRequestNumber}` : "protected-main push"; + const checkBody = { name: checkName, head_sha: verified.headSHA, status: "completed", conclusion: verified.conclusion, external_id: externalID, details_url: verified.detailsURL, completed_at: now.toISOString(), output: { title: verified.conclusion === "success" ? "Disposable CI and teardown verified" : "Disposable CI failed; teardown verified", summary: `Signed hypervisor evidence for ${scope}, run ${verified.sourceRunID}, attempt ${verified.sourceRunAttempt}.` } }; + const published = await githubJSON({ token: installationToken.token, method: "POST", path: `/repos/${EXPECTED_REPOSITORY}/check-runs`, body: checkBody, fetchImpl }); + if (published.name !== checkName || published.head_sha !== verified.headSHA || published.external_id !== externalID || published.conclusion !== verified.conclusion || String(published.app?.id) !== appID || !Number.isSafeInteger(published.id)) throw new Error(`published ${checkName} check does not match the verified signed result`); + checkRunIDs[checkName] = published.id; + } + return { ...verified, checkRunIDs, appID }; +}; + +const invokedAsCLI = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedAsCLI) { + publishPRCheck().then((result) => process.stdout.write(`Published ${CHECK_NAMES.join(",")}=${result.conclusion} for ${result.headSHA}.\n`)).catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/reconcile-github-environment-main-policy.py b/scripts/reconcile-github-environment-main-policy.py new file mode 100755 index 0000000..9da605c --- /dev/null +++ b/scripts/reconcile-github-environment-main-policy.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Audit or reconcile exact-main GitHub environment deployment policies.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from typing import Any +from urllib.parse import quote + + +REPOSITORY = "Makepad-fr/postgres" +REQUIRED_ENVIRONMENTS = ( + "canary", + "production", + "staging-brio-identity-db", + "release-brio-identity-db", + "keycloak-cohort-restore", + "postgres-ci-attestation", +) +MAX_POLICY_PAGES = 1000 + + +class PolicyError(RuntimeError): + """Raised when provider state cannot be proven safe and complete.""" + + +def _environment_path(environment: str) -> str: + if environment not in REQUIRED_ENVIRONMENTS: + raise PolicyError(f"Unsupported environment: {environment}") + return f"repos/{REPOSITORY}/environments/{quote(environment, safe='')}" + + +class GitHubClient: + """Minimal gh-backed client; authentication remains in gh's credential store.""" + + def request(self, method: str, path: str, payload: dict[str, Any] | None = None) -> Any: + command = [ + "gh", + "api", + "--method", + method, + "-H", + "Accept: application/vnd.github+json", + "-H", + "X-GitHub-Api-Version: 2022-11-28", + path, + ] + if payload is not None: + command.extend(("--input", "-")) + try: + result = subprocess.run( + command, + input=None if payload is None else json.dumps(payload, separators=(",", ":")), + check=True, + text=True, + stdout=subprocess.PIPE, + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PolicyError(f"GitHub API {method} failed for {path}") from error + try: + return json.loads(result.stdout) if result.stdout.strip() else None + except json.JSONDecodeError as error: + raise PolicyError(f"GitHub API returned invalid JSON for {path}") from error + + def get_environment(self, environment: str) -> dict[str, Any]: + response = self.request("GET", _environment_path(environment)) + if not isinstance(response, dict): + raise PolicyError(f"Invalid environment response for {environment}") + return response + + def put_environment(self, environment: str, payload: dict[str, Any]) -> None: + self.request("PUT", _environment_path(environment), payload) + + def list_policies(self, environment: str) -> list[dict[str, Any]]: + path = f"{_environment_path(environment)}/deployment-branch-policies" + policies: list[dict[str, Any]] = [] + seen_ids: set[int] = set() + expected_total: int | None = None + for page in range(1, MAX_POLICY_PAGES + 1): + response = self.request("GET", f"{path}?per_page=100&page={page}") + if not isinstance(response, dict): + raise PolicyError(f"Invalid branch-policy listing for {environment}") + total = response.get("total_count") + page_policies = response.get("branch_policies") + if not isinstance(total, int) or total < 0 or not isinstance(page_policies, list): + raise PolicyError(f"Incomplete branch-policy listing for {environment}") + if expected_total is None: + expected_total = total + elif total != expected_total: + raise PolicyError(f"Branch-policy listing changed during pagination for {environment}") + for policy in page_policies: + if not isinstance(policy, dict) or not isinstance(policy.get("id"), int): + raise PolicyError(f"Invalid branch policy for {environment}") + policy_id = policy["id"] + if policy_id in seen_ids: + raise PolicyError(f"Duplicate branch policy returned for {environment}") + seen_ids.add(policy_id) + policies.append(policy) + if len(policies) == expected_total: + return policies + if len(policies) > expected_total or not page_policies: + raise PolicyError(f"Truncated branch-policy listing for {environment}") + raise PolicyError(f"Branch-policy pagination exceeded its bound for {environment}") + + def create_main_policy(self, environment: str) -> None: + self.request( + "POST", + f"{_environment_path(environment)}/deployment-branch-policies", + {"name": "main", "type": "branch"}, + ) + + def delete_policy(self, environment: str, policy_id: int) -> None: + if not isinstance(policy_id, int) or policy_id <= 0: + raise PolicyError(f"Invalid branch-policy ID for {environment}") + self.request( + "DELETE", + f"{_environment_path(environment)}/deployment-branch-policies/{policy_id}", + ) + + +def has_custom_policy_mode(environment: dict[str, Any]) -> bool: + policy = environment.get("deployment_branch_policy") + return ( + isinstance(policy, dict) + and policy.get("protected_branches") is False + and policy.get("custom_branch_policies") is True + ) + + +def is_exact_main_policy(policies: list[dict[str, Any]]) -> bool: + return len(policies) == 1 and policies[0].get("name") == "main" and policies[0].get("type") == "branch" + + +def build_preserving_update(environment: dict[str, Any]) -> dict[str, Any]: + rules = environment.get("protection_rules") + if not isinstance(rules, list): + raise PolicyError("Environment protection rules are missing") + + wait_timer = 0 + reviewer_entries: list[dict[str, Any]] = [] + prevent_self_review = False + seen_rule_types: set[str] = set() + for rule in rules: + if not isinstance(rule, dict) or not isinstance(rule.get("type"), str): + raise PolicyError("Environment contains an invalid protection rule") + rule_type = rule["type"] + if rule_type in seen_rule_types: + raise PolicyError(f"Environment contains duplicate {rule_type} protection rules") + seen_rule_types.add(rule_type) + if rule_type == "branch_policy": + continue + if rule_type == "wait_timer": + candidate = rule.get("wait_timer") + if not isinstance(candidate, int) or not 0 <= candidate <= 43_200: + raise PolicyError("Environment wait timer is invalid") + wait_timer = candidate + continue + if rule_type != "required_reviewers": + raise PolicyError(f"Refusing to overwrite unsupported protection rule: {rule_type}") + + candidate_prevent = rule.get("prevent_self_review", False) + if not isinstance(candidate_prevent, bool): + raise PolicyError("Environment self-review setting is invalid") + prevent_self_review = candidate_prevent + reviewers = rule.get("reviewers") + if not isinstance(reviewers, list) or not 1 <= len(reviewers) <= 6: + raise PolicyError("Environment required reviewers are invalid") + for entry in reviewers: + reviewer = entry.get("reviewer") if isinstance(entry, dict) else None + reviewer_type = entry.get("type") if isinstance(entry, dict) else None + reviewer_id = reviewer.get("id") if isinstance(reviewer, dict) else None + if reviewer_type not in {"User", "Team"} or not isinstance(reviewer_id, int) or reviewer_id <= 0: + raise PolicyError("Environment required reviewer is invalid") + reviewer_entries.append({"type": reviewer_type, "id": reviewer_id}) + + return { + "wait_timer": wait_timer, + "prevent_self_review": prevent_self_review, + "reviewers": reviewer_entries, + "deployment_branch_policy": { + "protected_branches": False, + "custom_branch_policies": True, + }, + } + + +def audit_environment(client: GitHubClient, environment: str) -> None: + current = client.get_environment(environment) + if not has_custom_policy_mode(current): + raise PolicyError(f"{environment} does not use custom deployment branch policies") + policies = client.list_policies(environment) + if not is_exact_main_policy(policies): + raise PolicyError(f"{environment} is not restricted to the exact branch main") + + +def reconcile_environment(client: GitHubClient, environment: str) -> None: + current = client.get_environment(environment) + if not has_custom_policy_mode(current): + client.put_environment(environment, build_preserving_update(current)) + + policies = client.list_policies(environment) + exact_main = [policy for policy in policies if policy.get("name") == "main" and policy.get("type") == "branch"] + if not exact_main: + client.create_main_policy(environment) + policies = client.list_policies(environment) + exact_main = [policy for policy in policies if policy.get("name") == "main" and policy.get("type") == "branch"] + if not exact_main: + raise PolicyError(f"Unable to create exact main policy for {environment}") + + keep_id = exact_main[0]["id"] + for policy in policies: + if policy["id"] != keep_id: + client.delete_policy(environment, policy["id"]) + audit_environment(client, environment) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("audit", "apply")) + parser.add_argument("--environment", choices=REQUIRED_ENVIRONMENTS, action="append") + parser.add_argument("--confirm", default="") + args = parser.parse_args() + + environments = tuple(args.environment or REQUIRED_ENVIRONMENTS) + if args.mode == "apply": + if len(environments) != 1: + parser.error("apply requires exactly one --environment") + expected_confirmation = f"{REPOSITORY}:{environments[0]}:exact-main" + if args.confirm != expected_confirmation: + parser.error(f"apply requires --confirm {expected_confirmation}") + + client = GitHubClient() + try: + for environment in environments: + if args.mode == "apply": + reconcile_environment(client, environment) + else: + audit_environment(client, environment) + print(f"{environment}: exact custom branch policy main") + except PolicyError as error: + print(f"Environment policy verification failed: {error}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh new file mode 100755 index 0000000..6cdab10 --- /dev/null +++ b/scripts/run-ci.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "${repo_root}" + +./scripts/validate-postgres-config.sh +shellcheck \ + scripts/run-brio-encrypted-backup.sh \ + scripts/run-brio-encrypted-backup-loop.sh \ + scripts/deploy-brio-canary-postgres.sh \ + scripts/deploy-brio-identity-db-host.sh \ + scripts/deploy-postgres-stack.sh \ + scripts/brio-db-transaction.sh \ + scripts/ensure-brio-tmp-cleaner.sh \ + scripts/clean-keycloak-cohort-resources.sh \ + scripts/install-keycloak-cohort-cleaner.sh \ + scripts/keycloak-cohort-capture-dispatch.sh \ + scripts/install-keycloak-cohort-capture-host.sh \ + scripts/verify-brio-encrypted-restore.sh \ + scripts/test-brio-bootstrap.sh \ + scripts/test-brio-db-transaction.sh \ + scripts/test-brio-encrypted-backup.sh \ + scripts/test-brio-encrypted-restore.sh \ + scripts/test-brio-deploy-guards.sh \ + scripts/test-brio-deployment-contracts.sh \ + scripts/test-brio-deployment-failures.sh \ + scripts/test-brio-release-evidence.sh \ + scripts/test-keycloak-cohort-evidence.sh \ + scripts/test-keycloak-cohort-hardening.sh \ + scripts/test-postgres-ci-jit-result.sh \ + scripts/capture-keycloak-cohort-backups.sh \ + scripts/restore-keycloak-cohort-backups.sh \ + scripts/run-postgres-ci-jit-vm.sh \ + scripts/run-postgres-ci-queue-controller.sh \ + scripts/configure-postgres-ci-runner-group.sh \ + scripts/fixtures/brio-deployment-failure-fixture.sh \ + scripts/fixtures/keycloak-cohort-cleaner-fixture.sh \ + scripts/fixtures/keycloak-cohort-dispatch-fixture.sh +python3 - <<'PY' +import ast +from pathlib import Path + +for source in ( + "scripts/verify-brio-release-evidence.py", + "scripts/verify-keycloak-cohort-evidence.py", + "scripts/ci-base-image.py", + "scripts/verify-postgres-ci-jit-result.py", + "scripts/reconcile-github-environment-main-policy.py", + "scripts/test-github-environment-main-policy.py", +): + ast.parse(Path(source).read_text(), filename=source) +PY +PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-github-environment-main-policy.py +node --check scripts/publish-pr-ci-check.mjs +node --check scripts/postgres-ci-queue-controller.mjs +node --check scripts/dispatch-ci-attestation.mjs +node --test scripts/test-pr-ci-check.mjs scripts/test-postgres-ci-queue-controller.mjs +./scripts/test-postgres-ci-jit-result.sh +actionlint +git show --check --format= HEAD +git diff --check +./scripts/test-brio-deploy-guards.sh +./scripts/test-brio-deployment-contracts.sh +./scripts/test-brio-deployment-failures.sh +./scripts/test-brio-release-evidence.sh +./scripts/test-keycloak-cohort-evidence.sh +./scripts/test-keycloak-cohort-hardening.sh +./scripts/test-brio-bootstrap.sh +./scripts/test-brio-db-transaction.sh +./scripts/test-brio-encrypted-backup.sh +./scripts/test-brio-encrypted-restore.sh +backup_image=$(grep '^BRIO_BACKUP_IMAGE=' envs/canary/.env.db | cut -d= -f2-) +docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-backup.sh +docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-restore.sh diff --git a/scripts/run-postgres-ci-jit-vm.sh b/scripts/run-postgres-ci-jit-vm.sh new file mode 100755 index 0000000..da93d63 --- /dev/null +++ b/scripts/run-postgres-ci-jit-vm.sh @@ -0,0 +1,685 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +# Trusted-hypervisor launcher for one Postgres PR job. It obtains a GitHub JIT +# configuration, boots a fresh self-contained VM, and destroys the VM, disk, +# registration seed, network, firewall, and runner registration. Only then does +# it sign and dispatch per-run evidence with the dedicated Launcher App. + +readonly organization="Makepad-fr" +readonly runner_group="Postgres PR Ephemeral" +readonly runner_label="makepad-postgres-pr-ephemeral" +readonly api_version="2022-11-28" +script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +readonly script_directory + +die() { + printf '%s\n' "$*" >&2 + exit 1 +} + +[[ "$(id -u)" -eq 0 ]] || die "the JIT VM launcher must run as root on the dedicated CI hypervisor" +umask 077 +for trusted_helper in ci-base-image.py dispatch-ci-attestation.mjs verify-postgres-ci-jit-result.py; do + trusted_path="${script_directory}/${trusted_helper}" + [[ -f "${trusted_path}" && ! -L "${trusted_path}" && $(stat -c '%u' "${trusted_path}") == 0 ]] || \ + die "trusted launcher helper is missing, symlinked, or not root-owned: ${trusted_helper}" + trusted_mode=$(stat -c '%a' "${trusted_path}") + (( (8#${trusted_mode} & 8#022) == 0 )) || die "trusted launcher helper is writable outside root: ${trusted_helper}" +done + +job_root=${POSTGRES_CI_JOB_ROOT:-/var/lib/makepad/postgres-ci/jobs} +[[ "${job_root}" =~ ^/var/lib/makepad/postgres-ci/[A-Za-z0-9._/-]+$ && "${job_root}" != *..* ]] || die "POSTGRES_CI_JOB_ROOT is unsafe" + +if [[ $# -eq 2 && "$1" == --reconcile ]]; then + launch_id=$2 + [[ "${launch_id}" =~ ^j[1-9][0-9]{0,15}-[a-f0-9]{16}$ ]] || die "reconciliation launch ID is invalid" + for command_name in gh nft sha256sum virsh; do + command -v "${command_name}" >/dev/null || die "${command_name} is required for reconciliation" + done + IFS= read -r controller_token || die "a dedicated Launcher App installation token is required on standard input" + [[ "${controller_token}" =~ ^ghs_[A-Za-z0-9_]+$ ]] || die "the Launcher App token has an invalid format" + resource_hash=$(printf '%s' "${launch_id}" | sha256sum | cut -c1-10) + temporary_directory="${job_root}/postgres-ci-jit-${launch_id}" + vm_name="postgres-ci-${launch_id}" + runner_name="postgres-ci-jit-${launch_id}" + network_name="mdci-${launch_id}" + nft_table="mdci_${resource_hash}" + reconciliation_failed=false + if virsh dominfo "${vm_name}" >/dev/null 2>&1; then + virsh destroy "${vm_name}" >/dev/null 2>&1 || true + virsh undefine "${vm_name}" --nvram >/dev/null 2>&1 || virsh undefine "${vm_name}" >/dev/null 2>&1 || reconciliation_failed=true + fi + virsh dominfo "${vm_name}" >/dev/null 2>&1 && reconciliation_failed=true + if nft list table inet "${nft_table}" >/dev/null 2>&1; then nft delete table inet "${nft_table}" >/dev/null 2>&1 || reconciliation_failed=true; fi + nft list table inet "${nft_table}" >/dev/null 2>&1 && reconciliation_failed=true + if virsh net-info "${network_name}" >/dev/null 2>&1; then + virsh net-destroy "${network_name}" >/dev/null 2>&1 || true + virsh net-undefine "${network_name}" >/dev/null 2>&1 || reconciliation_failed=true + fi + virsh net-info "${network_name}" >/dev/null 2>&1 && reconciliation_failed=true + runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) || reconciliation_failed=true + while IFS= read -r runner_id; do + [[ -z "${runner_id}" ]] && continue + [[ "${runner_id}" =~ ^[1-9][0-9]*$ ]] && GH_TOKEN="${controller_token}" gh api --method DELETE \ + --header "X-GitHub-Api-Version: ${api_version}" "orgs/${organization}/actions/runners/${runner_id}" >/dev/null 2>&1 \ + || reconciliation_failed=true + done <<<"${runner_ids:-}" + remaining=$(GH_TOKEN="${controller_token}" gh api --paginate \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) || reconciliation_failed=true + [[ -z "${remaining:-}" ]] || reconciliation_failed=true + if [[ -e "${temporary_directory}" || -L "${temporary_directory}" ]]; then + [[ -d "${temporary_directory}" && ! -L "${temporary_directory}" && "$(stat -c '%u:%a' "${temporary_directory}")" == 0:700 ]] || die "reconciliation work directory is unsafe" + find "${temporary_directory}" -depth -mindepth 1 -delete + rmdir -- "${temporary_directory}" || reconciliation_failed=true + fi + unset controller_token + [[ "${reconciliation_failed}" == false ]] || die "incomplete JIT launch could not be fully reconciled" + printf 'Reconciled incomplete disposable launch %s.\n' "${launch_id}" + exit 0 +fi + +[[ $# -eq 0 ]] || die "usage: set exact POSTGRES_CI_RUN_* metadata and stream a Launcher App installation token on stdin" +[[ "$(uname -m)" == x86_64 ]] || die "the Postgres JIT base image and workflow require an x86_64 hypervisor" +[[ -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] || die "hardware-backed KVM is required for the one-job runner VM" +for command_name in cksum cloud-localds flock gh ip lsattr mktemp nft node python3 qemu-img seq sha256sum virsh virt-install; do + command -v "${command_name}" >/dev/null || die "${command_name} is required" +done + +base_image=${POSTGRES_CI_BASE_IMAGE:-} +expected_image_sha256=${POSTGRES_CI_BASE_IMAGE_SHA256:-} +public_dns=${POSTGRES_CI_PUBLIC_DNS_IPV4:-1.1.1.1} +run_id=${POSTGRES_CI_RUN_ID:-} +run_attempt=${POSTGRES_CI_RUN_ATTEMPT:-} +job_id=${POSTGRES_CI_JOB_ID:-} +run_event=${POSTGRES_CI_RUN_EVENT:-} +head_sha=${POSTGRES_CI_HEAD_SHA:-} +workflow_sha=${POSTGRES_CI_WORKFLOW_SHA:-} +attestation_nonce=${POSTGRES_CI_ATTESTATION_NONCE:-} +launch_id=${POSTGRES_CI_LAUNCH_ID:-} +attestation_private_key=${POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE:-} +result_poll_attempts=${POSTGRES_CI_RESULT_POLL_ATTEMPTS:-24} +result_poll_seconds=${POSTGRES_CI_RESULT_POLL_SECONDS:-5} +[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ && "${job_id}" =~ ^[1-9][0-9]*$ ]] || die "exact positive run, attempt, and job IDs are required" +[[ "${run_event}" == pull_request_target || "${run_event}" == push ]] || die "POSTGRES_CI_RUN_EVENT must be pull_request_target or push" +[[ "${head_sha}" =~ ^[a-f0-9]{40}$ ]] || die "POSTGRES_CI_HEAD_SHA must be the exact lowercase source SHA" +[[ "${workflow_sha}" =~ ^[a-f0-9]{40}$ ]] || die "POSTGRES_CI_WORKFLOW_SHA must be the protected workflow execution SHA" +if [[ "${run_event}" == push && "${head_sha}" != "${workflow_sha}" ]]; then + die "protected-main push source and workflow SHAs must be identical" +fi +[[ "${attestation_nonce}" =~ ^[A-Za-z0-9_-]{43}$ ]] || die "POSTGRES_CI_ATTESTATION_NONCE must be 32 random base64url bytes" +[[ "${launch_id}" =~ ^j[1-9][0-9]{0,15}-[a-f0-9]{16}$ && "${launch_id}" == "j${job_id}-"* ]] || die "POSTGRES_CI_LAUNCH_ID must bind the exact job to a deterministic resource set" +[[ "${result_poll_attempts}" =~ ^[1-9][0-9]*$ && "${result_poll_seconds}" =~ ^[0-9]+$ ]] || die "result polling controls must be non-negative integers" +((result_poll_attempts <= 60 && result_poll_seconds <= 30)) || die "result polling controls exceed the reviewed safety bound" +[[ "${attestation_private_key}" == /* && -f "${attestation_private_key}" && ! -L "${attestation_private_key}" ]] || die "a regular absolute Ed25519 attestation private-key file is required" +[[ "$(stat -c '%u:%a' "${attestation_private_key}")" == "0:400" ]] || die "the attestation private key must be root-owned mode 0400" +[[ "${base_image}" == /* && -f "${base_image}" && ! -L "${base_image}" ]] || die "POSTGRES_CI_BASE_IMAGE must be an absolute regular file" +[[ "${expected_image_sha256}" =~ ^[a-f0-9]{64}$ ]] || die "POSTGRES_CI_BASE_IMAGE_SHA256 must be a lowercase SHA-256 digest" +[[ "$(stat -c '%u' "${base_image}")" == 0 ]] || die "the base image must be owned by root" +base_mode=$(stat -c '%a' "${base_image}") +(( (8#${base_mode} & 8#022) == 0 )) || die "the base image must not be group- or world-writable" +python3 - "${base_image}" <<'PY' +import os +import pathlib +import stat +import sys + +path = pathlib.Path(sys.argv[1]) +for component in [pathlib.Path("/")] + list(reversed(path.parents[:-1])) + [path]: + value = os.lstat(component) + if stat.S_ISLNK(value.st_mode) or value.st_uid != 0 or value.st_mode & 0o022: + raise SystemExit(f"insecure base-image path component: {component}") +PY +attributes=$(lsattr -d -- "${base_image}" 2>/dev/null | awk '{print $1}') +[[ "${attributes}" == *i* ]] || die "the reviewed base image must have the filesystem immutable attribute" +python3 "${script_directory}/ci-base-image.py" "${base_image}" "${expected_image_sha256}" >/dev/null || die "the trusted base-image digest does not match" +qemu-img info --output=json "${base_image}" | python3 -c ' +import json, sys +payload = json.load(sys.stdin) +size = payload.get("virtual-size") +if payload.get("format") != "qcow2" or payload.get("backing-filename") or payload.get("data-file") or not isinstance(size, int) or not 8 * 1024**3 <= size <= 64 * 1024**3: + raise SystemExit("trusted base image must be qcow2 with an 8-64 GiB virtual disk") +' +python3 - "${public_dns}" <<'PY' +import ipaddress +import sys + +address = ipaddress.ip_address(sys.argv[1]) +if address.version != 4 or not address.is_global: + raise SystemExit("POSTGRES_CI_PUBLIC_DNS_IPV4 must be a globally routable IPv4 address") +PY + +IFS= read -r controller_token || die "a dedicated Launcher App installation token is required on standard input" +[[ "${controller_token}" =~ ^ghs_[A-Za-z0-9_]+$ ]] || die "the Launcher App token has an invalid format" + +install -d -m 0700 -o root -g root "${job_root}" +[[ -d "${job_root}" && ! -L "${job_root}" && "$(stat -c '%u:%a' "${job_root}")" == 0:700 ]] +temporary_directory="${job_root}/postgres-ci-jit-${launch_id}" +[[ ! -e "${temporary_directory}" && ! -L "${temporary_directory}" ]] || die "deterministic JIT resource directory already exists" +install -d -m 0700 -o root -g root "${temporary_directory}" +resource_hash=$(printf '%s' "${launch_id}" | sha256sum | cut -c1-10) +vm_name="postgres-ci-${launch_id}" +runner_name="postgres-ci-jit-${launch_id}" +network_name="mdci-${launch_id}" +bridge_name="md${resource_hash}" +nft_table="mdci_${resource_hash}" +overlay_path="${temporary_directory}/runner.qcow2" +seed_path="${temporary_directory}/seed.iso" +network_xml="${temporary_directory}/network.xml" +user_data="${temporary_directory}/user-data" +meta_data="${temporary_directory}/meta-data" +network_started=false +domain_defined=false +nft_created=false +jit_runner_id="" +attestation_eligible=false + +write_resource_manifest() { + local registered_id=${1:-} incoming="${temporary_directory}/.resources.json.incoming" + python3 - "${incoming}" "${launch_id}" "${job_id}" "${vm_name}" "${runner_name}" "${network_name}" "${bridge_name}" "${nft_table}" "${registered_id}" <<'PY' +import json, pathlib, sys +runner_id = int(sys.argv[9]) if sys.argv[9] else None +payload = {"version":1,"launch_id":sys.argv[2],"job_id":int(sys.argv[3]),"vm":sys.argv[4],"runner":sys.argv[5],"network":sys.argv[6],"bridge":sys.argv[7],"nft_table":sys.argv[8],"runner_id":runner_id} +pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n") +PY + chmod 0600 "${incoming}" + mv -fT "${incoming}" "${temporary_directory}/resources.json" + sync -f "${temporary_directory}" 2>/dev/null || sync +} +write_resource_manifest + +cleanup() { + local original_status=$? + local cleanup_failed=false + local retain_vm_files=false + local job_conclusion="" + trap - EXIT INT TERM HUP + unset encoded_jit_config + set +e + if [[ "${domain_defined}" == true ]]; then + virsh destroy "${vm_name}" >/dev/null 2>&1 + virsh undefine "${vm_name}" --nvram >/dev/null 2>&1 || virsh undefine "${vm_name}" >/dev/null 2>&1 + if virsh dominfo "${vm_name}" >/dev/null 2>&1; then + printf 'Failed to destroy and undefine ephemeral runner VM %s.\n' "${vm_name}" >&2 + cleanup_failed=true + retain_vm_files=true + fi + fi + if [[ "${nft_created}" == true ]]; then + nft delete table inet "${nft_table}" >/dev/null 2>&1 + if nft list table inet "${nft_table}" >/dev/null 2>&1; then + printf 'Failed to remove ephemeral runner firewall table %s.\n' "${nft_table}" >&2 + cleanup_failed=true + fi + fi + if [[ "${network_started}" == true ]]; then + virsh net-destroy "${network_name}" >/dev/null 2>&1 + virsh net-undefine "${network_name}" >/dev/null 2>&1 + if virsh net-info "${network_name}" >/dev/null 2>&1; then + printf 'Failed to remove ephemeral runner network %s.\n' "${network_name}" >&2 + cleanup_failed=true + fi + fi + if [[ -n "${controller_token:-}" && -n "${runner_name:-}" ]]; then + runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) + lookup_status=$? + if [[ "${lookup_status}" -ne 0 ]]; then + printf 'Failed to inspect the JIT runner registration during teardown.\n' >&2 + cleanup_failed=true + else + while IFS= read -r runner_id; do + [[ -z "${runner_id}" ]] && continue + if [[ ! "${runner_id}" =~ ^[1-9][0-9]*$ || ( -n "${jit_runner_id}" && "${runner_id}" != "${jit_runner_id}" ) ]] || \ + ! GH_TOKEN="${controller_token}" gh api --method DELETE \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners/${runner_id}" >/dev/null 2>&1; then + printf 'Failed to remove JIT runner registration %s.\n' "${runner_id}" >&2 + cleanup_failed=true + fi + done <<<"${runner_ids}" + remaining_runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners?per_page=100" \ + --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) + remaining_status=$? + if [[ "${remaining_status}" -ne 0 || -n "${remaining_runner_ids}" ]]; then + printf 'JIT runner registration removal could not be verified.\n' >&2 + cleanup_failed=true + fi + fi + fi + if [[ "${retain_vm_files}" == false ]]; then + if [[ -d "${temporary_directory}" && ! -L "${temporary_directory}" && "${temporary_directory}" == "${job_root}"/postgres-ci-jit-* ]]; then + find "${temporary_directory}" -depth -mindepth 1 -delete + rmdir -- "${temporary_directory}" + if [[ -e "${temporary_directory}" || -L "${temporary_directory}" ]]; then + cleanup_failed=true + fi + else + cleanup_failed=true + fi + fi + if [[ "${cleanup_failed}" == true ]]; then + printf 'Ephemeral CI teardown is incomplete; inspect the hypervisor alert immediately.\n' >&2 + if [[ "${retain_vm_files}" == true ]]; then + printf 'VM files are quarantined at %s until the domain is destroyed.\n' "${temporary_directory}" >&2 + fi + original_status=1 + fi + if [[ "${attestation_eligible}" == true && "${cleanup_failed}" == false ]]; then + # The VM has stopped and every hypervisor resource plus GitHub registration + # has been independently shown absent. Now bind the authoritative job result + # to that teardown before the hypervisor-only Ed25519 key signs anything. + run_payload_file=$(mktemp /run/postgres-ci-run-XXXXXXXX.json) + jobs_payload_file=$(mktemp /run/postgres-ci-jobs-XXXXXXXX.json) + chmod 0600 "${run_payload_file}" "${jobs_payload_file}" + completed_payload=false + for poll_attempt in $(seq 1 "${result_poll_attempts}"); do + if GH_TOKEN="${controller_token}" gh api \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "repos/${organization}/postgres/actions/runs/${run_id}" >"${run_payload_file}" 2>/dev/null && \ + GH_TOKEN="${controller_token}" gh api \ + --header "X-GitHub-Api-Version: ${api_version}" \ + "repos/${organization}/postgres/actions/runs/${run_id}/attempts/${run_attempt}/jobs?per_page=100" \ + >"${jobs_payload_file}" 2>/dev/null && \ + python3 - "${run_payload_file}" "${jobs_payload_file}" "${job_id}" <<'PY' +import json, pathlib, sys +run=json.loads(pathlib.Path(sys.argv[1]).read_text()); response=json.loads(pathlib.Path(sys.argv[2]).read_text()); jobs=response.get("jobs", []) +if not isinstance(jobs, list) or response.get("total_count") != len(jobs): raise SystemExit(1) +matches=[job for job in jobs if job.get("id") == int(sys.argv[3])] +raise SystemExit(0 if run.get("status") == "completed" and len(matches) == 1 and matches[0].get("status") == "completed" else 1) +PY + then + completed_payload=true + break + fi + if ((poll_attempt < result_poll_attempts && result_poll_seconds > 0)); then sleep "${result_poll_seconds}"; fi + done + if [[ "${completed_payload}" != true ]]; then + printf 'Authoritative workflow state did not converge before the bounded attestation deadline.\n' >&2 + cleanup_failed=true + original_status=1 + fi + if [[ "${cleanup_failed}" == false ]]; then + job_conclusion=$(python3 "${script_directory}/verify-postgres-ci-jit-result.py" \ + "${run_payload_file}" "${jobs_payload_file}" "${run_id}" "${run_attempt}" \ + "${job_id}" "${run_event}" "${head_sha}" "${workflow_sha}" "${jit_runner_id}" \ + "${runner_name}" "${runner_group_id}") + fi + rm -f -- "${run_payload_file}" "${jobs_payload_file}" || { + cleanup_failed=true + original_status=1 + } + if [[ "${job_conclusion}" != success && "${job_conclusion}" != failure ]]; then + printf 'Unable to bind authoritative job conclusion to teardown.\n' >&2 + cleanup_failed=true + original_status=1 + else + attestation_file=$(mktemp /run/postgres-ci-attestation-XXXXXXXX.json) + chmod 0600 "${attestation_file}" + python3 - "${attestation_file}" "${run_id}" "${run_attempt}" "${job_id}" "${run_event}" \ + "${head_sha}" "${workflow_sha}" "${job_conclusion}" "${jit_runner_id}" "${runner_name}" \ + "${runner_group_id}" "${runner_group}" "${runner_label}" "${expected_image_sha256}" \ + "${attestation_nonce}" <<'PY' +import datetime +import json +import pathlib +import sys + +payload = { + "base_image_sha256": sys.argv[14], + "issued_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "nonce": sys.argv[15], + "ref": "refs/heads/main", + "registration_absent": True, + "repository": "Makepad-fr/postgres", + "run": { + "attempt": int(sys.argv[3]), + "conclusion": sys.argv[8], + "event": sys.argv[5], + "head_sha": sys.argv[6], + "id": int(sys.argv[2]), + "job_id": int(sys.argv[4]), + "job_name": "policy-and-integration", + "workflow_sha": sys.argv[7], + }, + "runner": { + "group_id": int(sys.argv[11]), + "group_name": sys.argv[12], + "id": int(sys.argv[9]), + "labels": ["self-hosted", "linux", "x64", sys.argv[13]], + "name": sys.argv[10], + }, + "schema": "makepad.postgres.ci-attestation.v1", + "teardown": {"disk": True, "firewall": True, "network": True, "vm": True}, + "workflow": {"name": "CI", "path": ".github/workflows/ci.yml"}, +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY + if ! printf '%s\n' "${controller_token}" | \ + POSTGRES_CI_ATTESTATION_JSON_FILE="${attestation_file}" \ + POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE="${attestation_private_key}" \ + node "${script_directory}/dispatch-ci-attestation.mjs"; then + printf 'Signed teardown evidence could not be dispatched by the Launcher App.\n' >&2 + cleanup_failed=true + original_status=1 + fi + rm -f -- "${attestation_file}" || { + printf 'Could not remove transient attestation material.\n' >&2 + cleanup_failed=true + original_status=1 + } + fi + fi + unset controller_token + if [[ "${cleanup_failed}" == true ]]; then + printf 'The queue supervisor must raise an independent launcher failure alert.\n' >&2 + fi + exit "${original_status}" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +group_payload="${temporary_directory}/runner-groups.json" +GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups?per_page=100" >"${group_payload}" +runner_group_id=$(python3 - "${group_payload}" "${runner_group}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +groups = payload.get("runner_groups", []) +if payload.get("total_count", len(groups)) > len(groups): + raise SystemExit("more than 100 runner groups require explicit pagination") +matches = [item for item in groups if item.get("name") == sys.argv[2]] +if len(matches) != 1: + raise SystemExit("the exact Postgres PR Ephemeral runner group does not exist uniquely") +print(matches[0]["id"]) +PY +) +[[ "${runner_group_id}" =~ ^[1-9][0-9]*$ ]] || die "runner group ID is invalid" + +group_details="${temporary_directory}/runner-group.json" +group_repositories="${temporary_directory}/runner-group-repositories.json" +repository_payload="${temporary_directory}/repository.json" +GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${runner_group_id}" >"${group_details}" +GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runner-groups/${runner_group_id}/repositories?per_page=100" >"${group_repositories}" +GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ + "repos/${organization}/postgres" >"${repository_payload}" +python3 - "${group_details}" "${group_repositories}" "${repository_payload}" <<'PY' +import json +import pathlib +import sys + +group = json.loads(pathlib.Path(sys.argv[1]).read_text()) +repository_selection = json.loads(pathlib.Path(sys.argv[2]).read_text()) +repository = json.loads(pathlib.Path(sys.argv[3]).read_text()) +selected = repository_selection.get("repositories", []) +expected_workflows = { + "Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main", + "Makepad-fr/postgres/.github/workflows/pr-ci-result.yml@refs/heads/main", +} +if ( + group.get("name") != "Postgres PR Ephemeral" + or group.get("visibility") != "selected" + or group.get("allows_public_repositories") is not True + or group.get("restricted_to_workflows") is not True + or group.get("workflow_restrictions_read_only") is not False + or set(group.get("selected_workflows", [])) != expected_workflows +): + raise SystemExit("Postgres PR Ephemeral runner group is not restricted to the exact protected workflows") +if repository_selection.get("total_count", len(selected)) > len(selected): + raise SystemExit("runner-group repository selection is truncated") +if ( + repository.get("full_name") != "Makepad-fr/postgres" + or repository.get("private") is not False + or not isinstance(repository.get("id"), int) + or [item.get("id") for item in selected] != [repository["id"]] +): + raise SystemExit("Postgres PR Ephemeral runner group is not restricted to the public PostgreSQL repository") +PY + +jit_request="${temporary_directory}/jit-request.json" +jit_response="${temporary_directory}/jit-response.json" +python3 - "${runner_name}" "${runner_group_id}" "${runner_label}" >"${jit_request}" <<'PY' +import json +import sys + +print(json.dumps({ + "name": sys.argv[1], + "runner_group_id": int(sys.argv[2]), + "work_folder": "_work", + "labels": ["self-hosted", "Linux", "X64", sys.argv[3]], +}, separators=(",", ":"))) +PY +chmod 0600 "${jit_request}" +GH_TOKEN="${controller_token}" gh api --method POST --header "X-GitHub-Api-Version: ${api_version}" \ + "orgs/${organization}/actions/runners/generate-jitconfig" \ + --input "${jit_request}" >"${jit_response}" +jit_runner_id=$(python3 - "${jit_response}" "${runner_name}" "${runner_label}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +runner = payload.get("runner", {}) +runner_id = runner.get("id") +labels = { + str(item.get("name", "")).lower() + for item in runner.get("labels", []) + if isinstance(item, dict) +} +if ( + not isinstance(runner_id, int) + or runner_id <= 0 + or runner.get("name") != sys.argv[2] + or runner.get("status") != "offline" + or not {"self-hosted", "linux", "x64", sys.argv[3]}.issubset(labels) +): + raise SystemExit("GitHub returned an invalid JIT runner identity") +print(runner_id) +PY +) +write_resource_manifest "${jit_runner_id}" +encoded_jit_config=$(python3 - "${jit_response}" <<'PY' +import json +import pathlib +import re +import sys + +value = json.loads(pathlib.Path(sys.argv[1]).read_text()).get("encoded_jit_config", "") +if not re.fullmatch(r"[A-Za-z0-9_+/\-]{40,8192}={0,2}", value): + raise SystemExit("GitHub returned an invalid JIT configuration") +print(value) +PY +) + +# A fresh libvirt network is created for every VM. Its host-side nftables hook +# blocks private, WireGuard, link-local/metadata, multicast, every hypervisor +# address, and all egress except public DNS plus TLS. Guest root cannot remove +# these hypervisor rules. +exec 8>/run/lock/postgres-ci-network.lock +flock -x 8 +suffix_checksum=$(printf '%s' "${launch_id}" | cksum) +suffix_checksum=${suffix_checksum%% *} +subnet="" +for offset in $(seq 0 199); do + network_octet=$(((suffix_checksum + offset) % 200 + 20)) + candidate="172.31.${network_octet}" + if [[ -z "$(ip -4 route show exact "${candidate}.0/24")" ]]; then + subnet="${candidate}" + break + fi +done +[[ -n "${subnet}" ]] || die "no unused ephemeral runner subnet is available" +cat >"${network_xml}" < + ${network_name} + + + + + + + +EOF +chmod 0600 "${network_xml}" +virsh net-define "${network_xml}" >/dev/null +network_started=true +virsh net-start "${network_name}" >/dev/null +flock -u 8 + +nft_created=true +nft -f - </dev/null || die "base image mutated while the self-contained job disk was created" + +guest_script=$(cat <<'GUEST' +#!/usr/bin/env bash +set -euo pipefail +power_off() { + local status=$? + trap - EXIT + find /run/postgres-jit -depth -mindepth 1 -delete 2>/dev/null || true + rmdir /run/postgres-jit 2>/dev/null || true + systemctl poweroff --no-block + exit "${status}" +} +trap power_off EXIT +[[ -x /opt/actions-runner/run.sh ]] +[[ -f /run/postgres-jit/config && ! -L /run/postgres-jit/config ]] +jit_config=$("${meta_data}" +chmod 0600 "${user_data}" "${meta_data}" +cloud-localds "${seed_path}" "${user_data}" "${meta_data}" +chmod 0600 "${seed_path}" + +domain_defined=true +virt-install \ + --name "${vm_name}" \ + --virt-type kvm \ + --memory 6144 \ + --vcpus 4 \ + --import \ + --osinfo detect=on,require=off \ + --disk "path=${overlay_path},format=qcow2,bus=virtio,cache=none" \ + --disk "path=${seed_path},device=cdrom,readonly=on" \ + --network "network=${network_name},model=virtio" \ + --graphics none \ + --noautoconsole >/dev/null + +deadline=$((SECONDS + 2700)) +while (( SECONDS < deadline )); do + state=$(virsh domstate "${vm_name}" 2>/dev/null || true) + case "${state}" in + "shut off"|"crashed") break ;; + esac + sleep 5 +done +state=$(virsh domstate "${vm_name}" 2>/dev/null || true) +[[ "${state}" == "shut off" ]] || die "the one-job runner did not shut down within 45 minutes" +attestation_eligible=true +printf 'One-job JIT VM %s stopped; destroying all ephemeral state.\n' "${vm_name}" diff --git a/scripts/run-postgres-ci-queue-controller.sh b/scripts/run-postgres-ci-queue-controller.sh new file mode 100755 index 0000000..3aa0a2a --- /dev/null +++ b/scripts/run-postgres-ci-queue-controller.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +[[ "$(id -u)" -eq 0 ]] || { echo "controller supervisor must run as root" >&2; exit 1; } +exec 9>/run/lock/postgres-ci-queue-controller.lock +flock -n 9 || { echo "another Postgres queue controller owns the hypervisor" >&2; exit 1; } +script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +exec node "${script_directory}/postgres-ci-queue-controller.mjs" "$@" diff --git a/scripts/test-github-environment-main-policy.py b/scripts/test-github-environment-main-policy.py new file mode 100755 index 0000000..c42eaac --- /dev/null +++ b/scripts/test-github-environment-main-policy.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Behavioral tests for exact-main GitHub environment reconciliation.""" + +from __future__ import annotations + +import copy +import runpy +from pathlib import Path + + +module = runpy.run_path( + str(Path(__file__).with_name("reconcile-github-environment-main-policy.py")), + run_name="postgres_environment_policy", +) +PolicyError = module["PolicyError"] +REQUIRED_ENVIRONMENTS = module["REQUIRED_ENVIRONMENTS"] +audit_environment = module["audit_environment"] +build_preserving_update = module["build_preserving_update"] +reconcile_environment = module["reconcile_environment"] + + +class FakeClient: + def __init__(self, environment, policies): + self.environment = copy.deepcopy(environment) + self.policies = copy.deepcopy(policies) + self.calls = [] + self.next_id = 100 + + def get_environment(self, environment): + self.calls.append(("get", environment)) + return copy.deepcopy(self.environment) + + def put_environment(self, environment, payload): + self.calls.append(("put", environment, copy.deepcopy(payload))) + self.environment["deployment_branch_policy"] = copy.deepcopy(payload["deployment_branch_policy"]) + + def list_policies(self, environment): + self.calls.append(("list", environment)) + return copy.deepcopy(self.policies) + + def create_main_policy(self, environment): + self.calls.append(("create", environment, "main", "branch")) + self.policies.append({"id": self.next_id, "name": "main", "type": "branch"}) + self.next_id += 1 + + def delete_policy(self, environment, policy_id): + self.calls.append(("delete", environment, policy_id)) + self.policies = [policy for policy in self.policies if policy["id"] != policy_id] + + +protected_environment = { + "deployment_branch_policy": {"protected_branches": True, "custom_branch_policies": False}, + "protection_rules": [ + {"type": "branch_policy"}, + {"type": "wait_timer", "wait_timer": 15}, + { + "type": "required_reviewers", + "prevent_self_review": True, + "reviewers": [{"type": "Team", "reviewer": {"id": 42}}], + }, + ], +} + +assert "production" in REQUIRED_ENVIRONMENTS +preserved = build_preserving_update(protected_environment) +assert preserved == { + "wait_timer": 15, + "prevent_self_review": True, + "reviewers": [{"type": "Team", "id": 42}], + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}, +} + +client = FakeClient(protected_environment, [{"id": 7, "name": "release/*", "type": "branch"}]) +try: + audit_environment(client, "production") +except PolicyError: + pass +else: + raise AssertionError("generic protected-branch policy was accepted") + +client.calls.clear() +reconcile_environment(client, "production") +audit_environment(client, "production") +assert client.policies == [{"id": 100, "name": "main", "type": "branch"}] +assert next(index for index, call in enumerate(client.calls) if call[0] == "create") < next( + index for index, call in enumerate(client.calls) if call[0] == "delete" +) +put_call = next(call for call in client.calls if call[0] == "put") +assert put_call[2]["reviewers"] == [{"type": "Team", "id": 42}] +assert put_call[2]["wait_timer"] == 15 + +client = FakeClient( + { + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}, + "protection_rules": [{"type": "branch_policy"}], + }, + [ + {"id": 9, "name": "main", "type": "branch"}, + {"id": 10, "name": "main", "type": "tag"}, + ], +) +try: + audit_environment(client, "production") +except PolicyError: + pass +else: + raise AssertionError("additional tag policy was accepted") + +print("GitHub environment exact-main policy tests passed.") diff --git a/scripts/test-postgres-ci-jit-result.sh b/scripts/test-postgres-ci-jit-result.sh new file mode 100755 index 0000000..c5f6d50 --- /dev/null +++ b/scripts/test-postgres-ci-jit-result.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +work_dir=$(mktemp -d) +cleanup() { + find "${work_dir}" -mindepth 1 -delete + rmdir "${work_dir}" +} +trap cleanup EXIT + +source_sha=$(printf 'a%.0s' {1..40}) +workflow_sha=$(printf 'b%.0s' {1..40}) +run_file="${work_dir}/run.json" +jobs_file="${work_dir}/jobs.json" +python3 - "${run_file}" "${jobs_file}" "${source_sha}" "${workflow_sha}" <<'PY' +import json +import pathlib +import sys + +run = { + "id": 101, + "run_attempt": 2, + "event": "pull_request_target", + "head_sha": sys.argv[4], + "head_branch": "main", + "name": "CI", + "path": ".github/workflows/ci.yml", + "status": "completed", + "conclusion": "success", + "repository": {"id": 77, "full_name": "Makepad-fr/postgres"}, + "pull_requests": [{ + "number": 9, + "head": {"sha": sys.argv[3], "repo": {"id": 77}}, + "base": {"ref": "main", "sha": sys.argv[4], "repo": {"id": 77}}, + }], +} +job = { + "id": 202, + "run_id": 101, + "head_sha": sys.argv[4], + "workflow_name": "CI", + "runner_id": 303, + "runner_name": "postgres-ci-jit-j202-1111111111111111", + "runner_group_id": 404, + "runner_group_name": "Postgres PR Ephemeral", + "name": "policy-and-integration", + "status": "completed", + "conclusion": "success", + "labels": ["self-hosted", "Linux", "X64", "makepad-postgres-pr-ephemeral"], +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(run)) +pathlib.Path(sys.argv[2]).write_text(json.dumps({"total_count": 1, "jobs": [job]})) +PY + +result=$(python3 "${script_dir}/verify-postgres-ci-jit-result.py" \ + "${run_file}" "${jobs_file}" 101 2 202 pull_request_target \ + "${source_sha}" "${workflow_sha}" 303 postgres-ci-jit-j202-1111111111111111 404) +[[ "${result}" == success ]] + +python3 - "${run_file}" <<'PY' +import json +import pathlib +import sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["pull_requests"][0]["base"]["sha"] = "c" * 40 +path.write_text(json.dumps(value)) +PY +if python3 "${script_dir}/verify-postgres-ci-jit-result.py" \ + "${run_file}" "${jobs_file}" 101 2 202 pull_request_target \ + "${source_sha}" "${workflow_sha}" 303 postgres-ci-jit-j202-1111111111111111 404 >/dev/null 2>&1; then + echo "JIT result verifier accepted a PR association with the wrong base SHA." >&2 + exit 1 +fi + +echo "Postgres JIT authoritative-result tests passed." diff --git a/scripts/test-postgres-ci-queue-controller.mjs b/scripts/test-postgres-ci-queue-controller.mjs new file mode 100644 index 0000000..97f2500 --- /dev/null +++ b/scripts/test-postgres-ci-queue-controller.mjs @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import {readFile} from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import {fileURLToPath, pathToFileURL} from "node:url"; + +const candidateRoot = path.resolve(process.env.POSTGRES_CANDIDATE_ROOT || fileURLToPath(new URL("..", import.meta.url))); +const controllerURL = pathToFileURL(path.join(candidateRoot, "scripts/postgres-ci-queue-controller.mjs")); +const {reconcileIncompleteJobs, selectAuthorizedJobs} = await import(controllerURL.href); +const launcherURL = pathToFileURL(path.join(candidateRoot, "scripts/run-postgres-ci-jit-vm.sh")); + +const repositoryID = 77; +const prBase = () => { + const association = {number: 9, head: {sha: "a".repeat(40), repo: {id: repositoryID}}, base: {ref: "main", sha: "b".repeat(40), repo: {id: repositoryID}}}; + const run = {id: 101, run_attempt: 2, name: "CI", path: ".github/workflows/ci.yml", event: "pull_request_target", status: "queued", head_sha: "b".repeat(40), repository: {id: repositoryID}, pull_requests: [association]}; + const job = {id: 202, run_id: 101, head_sha: "b".repeat(40), workflow_name: "CI", name: "policy-and-integration", status: "queued", labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]}; + return { + runs: {total_count: 1, workflow_runs: [run]}, + jobsByRun: new Map([["101:2", {total_count: 1, jobs: [job]}]]), + pullRequests: new Map([[9, {number: 9, head: {sha: "a".repeat(40), repo: {id: repositoryID}}, base: {ref: "main", sha: "b".repeat(40), repo: {id: repositoryID}}}]]), + }; +}; + +test("selects the exact queued protected-base same-repository PR job without relying on a nonexistent job attempt field", () => { + assert.deepEqual(selectAuthorizedJobs({...prBase(), repositoryID}), [{runID: 101, attempt: 2, jobID: 202, event: "pull_request_target", sourceSHA: "a".repeat(40), workflowSHA: "b".repeat(40), pullRequestNumber: 9}]); +}); + +test("selects an exact protected-main push job so release CI cannot remain queued", () => { + const value = prBase(); + const run = value.runs.workflow_runs[0]; + run.event = "push"; + run.head_branch = "main"; + run.pull_requests = []; + value.pullRequests.clear(); + assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), [{runID: 101, attempt: 2, jobID: 202, event: "push", sourceSHA: "b".repeat(40), workflowSHA: "b".repeat(40), pullRequestNumber: null}]); +}); + +test("rejects fork, wrong workflow, extra-label, moved-head, and non-queued jobs", () => { + for (const mutate of [ + (value) => { value.runs.workflow_runs[0].pull_requests[0].head.repo.id = 999; }, + (value) => { value.runs.workflow_runs[0].path = ".github/workflows/evil.yml"; }, + (value) => { value.jobsByRun.get("101:2").jobs[0].labels.push("persistent"); }, + (value) => { value.pullRequests.get(9).head.sha = "b".repeat(40); }, + (value) => { value.runs.workflow_runs[0].pull_requests[0].base.sha = "d".repeat(40); }, + (value) => { value.jobsByRun.get("101:2").jobs[0].status = "in_progress"; }, + ]) { + const value = prBase(); + mutate(value); + assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), []); + } +}); + +test("rejects a non-main push, mismatched job head, and wrong job workflow", () => { + for (const mutate of [ + (value) => { value.runs.workflow_runs[0].head_branch = "feature"; }, + (value) => { value.jobsByRun.get("101:2").jobs[0].head_sha = "c".repeat(40); }, + (value) => { value.jobsByRun.get("101:2").jobs[0].workflow_name = "Other"; }, + ]) { + const value = prBase(); + value.runs.workflow_runs[0].event = "push"; + value.runs.workflow_runs[0].head_branch = "main"; + value.runs.workflow_runs[0].pull_requests = []; + mutate(value); + assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), []); + } +}); + +test("the durable controller records deterministic resources before launch and never selects recorded IDs again", async () => { + const source = await readFile(controllerURL, "utf8"); + assert.match(source, /state\.jobs\[String\(job\.jobID\)\] = \{\.\.\.job, nonce, launchID, status: "launching"/); + assert.match(source, /filter\(\(job\) => !state\.jobs\[String\(job\.jobID\)\]\)/); + assert.match(source, /await runLauncher/); + assert.match(source, /issues/); + const launcherFailure = source.indexOf("// Any nonzero launcher exit is cleanup-uncertain."); + const failed = source.indexOf('status = "recovery-required"', launcherFailure); + const persisted = source.indexOf("await atomicState(stateFile, state);", failed); + const issue = source.indexOf("/issues", failed); + const rethrow = source.indexOf("throw error;", failed); + assert.ok(launcherFailure > 0 && launcherFailure < failed && failed < persisted && persisted < issue && issue < rethrow); + assert.match(source, /systemd OnFailure webhook remains independent of GitHub/); + assert.match(source, /pull_requests: "read"/); + assert.match(source, /await reconcileIncompleteJobs/); + assert.doesNotMatch(source, /status = "failed"/); +}); + +test("startup reconciliation marks every incomplete launch failed-recovered and persists each transition", async () => { + const state = {version: 2, jobs: { + "202": {status: "launching", launchID: "j202-1111111111111111"}, + "203": {status: "recovery-required", launchID: "j203-2222222222222222"}, + "204": {status: "completed", launchID: "j204-3333333333333333"}, + }}; + const reconciled = []; + let persisted = 0; + await reconcileIncompleteJobs({state, persist: async () => { persisted += 1; }, reconcile: async (record) => { reconciled.push(record.launchID); }}); + assert.deepEqual(reconciled, ["j202-1111111111111111", "j203-2222222222222222"]); + assert.equal(persisted, 2); + assert.equal(state.jobs["202"].status, "failed-recovered"); + assert.equal(state.jobs["203"].status, "failed-recovered"); +}); + +test("failed startup reconciliation stays recovery-required and blocks polling", async () => { + const state = {version: 2, jobs: {"202": {status: "launching", launchID: "j202-1111111111111111"}}}; + await assert.rejects(reconcileIncompleteJobs({state, persist: async () => {}, reconcile: async () => { throw new Error("still present"); }}), /still present/); + assert.equal(state.jobs["202"].status, "recovery-required"); +}); + +test("the hypervisor signs only after every disposable resource and registration is proven absent", async () => { + const source = await readFile(launcherURL, "utf8"); + const vmRemoval = source.indexOf('virsh undefine "${vm_name}"'); + const firewallRemoval = source.indexOf('nft delete table inet "${nft_table}"'); + const networkRemoval = source.indexOf('virsh net-undefine "${network_name}"'); + const registrationRemoval = source.indexOf('actions/runners/${runner_id}'); + const absenceCheck = source.indexOf('remaining_runner_ids='); + const teardownGate = source.indexOf('"${attestation_eligible}" == true && "${cleanup_failed}" == false'); + const signingDispatch = source.indexOf('node "${script_directory}/dispatch-ci-attestation.mjs"', teardownGate); + assert.ok(vmRemoval > 0 && firewallRemoval > vmRemoval && networkRemoval > firewallRemoval); + assert.ok(registrationRemoval > 0 && absenceCheck > registrationRemoval); + assert.ok(teardownGate > absenceCheck && signingDispatch > teardownGate); + assert.equal((source.match(/generate-jitconfig/g) || []).length, 2); // API endpoint and explanatory comment. + assert.match(source, /run\.sh --jitconfig/); + assert.match(source, /resources\.json/); + assert.match(source, /--reconcile/); + assert.match(source, /POSTGRES_CI_RESULT_POLL_ATTEMPTS/); + assert.match(source, /repository\.get\("private"\) is not False/); + assert.match(source, /group\.get\("allows_public_repositories"\) is not True/); +}); diff --git a/scripts/test-pr-ci-check.mjs b/scripts/test-pr-ci-check.mjs new file mode 100644 index 0000000..ee8d275 --- /dev/null +++ b/scripts/test-pr-ci-check.mjs @@ -0,0 +1,152 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; +import path from "node:path"; +import test from "node:test"; +import {fileURLToPath, pathToFileURL} from "node:url"; + +const candidateRoot = path.resolve(process.env.POSTGRES_CANDIDATE_ROOT || fileURLToPath(new URL("..", import.meta.url))); +const { + assertNoAttestationReplay, + canonicalJSON, + validateAuthoritativeEvidence, + verifySignedAttestation, +} = await import(pathToFileURL(path.join(candidateRoot, "scripts/publish-pr-ci-check.mjs")).href); + +const now = new Date("2026-09-05T10:00:00Z"); +const digest = "a".repeat(64); +const {privateKey, publicKey} = generateKeyPairSync("ed25519"); + +const baseAttestation = () => ({ + base_image_sha256: digest, + issued_at: now.toISOString().replace(".000Z", "Z"), + nonce: "A".repeat(43), + ref: "refs/heads/main", + registration_absent: true, + repository: "Makepad-fr/postgres", + run: {attempt: 2, conclusion: "success", event: "pull_request_target", head_sha: "b".repeat(40), workflow_sha: "c".repeat(40), id: 1234, job_id: 5678, job_name: "policy-and-integration"}, + runner: {group_id: 12, group_name: "Postgres PR Ephemeral", id: 44, labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"], name: "postgres-ci-jit-20260905100000-deadbeef"}, + schema: "makepad.postgres.ci-attestation.v1", + teardown: {disk: true, firewall: true, network: true, vm: true}, + workflow: {name: "CI", path: ".github/workflows/ci.yml"}, +}); + +const signedEvent = (attestation = baseAttestation(), senderID = 9001) => ({ + action: "postgres-pr-ci-attestation", + repository: {full_name: "Makepad-fr/postgres"}, + sender: {id: senderID, type: "Bot"}, + client_payload: { + attestation, + signature: sign(null, Buffer.from(canonicalJSON(attestation)), privateKey).toString("base64url"), + }, +}); + +const verify = (event, overrides = {}) => verifySignedAttestation({event, publicKey, approvedDigest: digest, launcherSenderID: "9001", now, ...overrides}); + +const authoritative = (attestation = baseAttestation()) => { + const job = { + id: 5678, + run_id: 1234, + head_sha: "c".repeat(40), + workflow_name: "CI", + name: "policy-and-integration", + status: "completed", + conclusion: attestation.run.conclusion, + runner_id: 44, + runner_name: "postgres-ci-jit-20260905100000-deadbeef", + runner_group_id: 12, + runner_group_name: "Postgres PR Ephemeral", + labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"], + }; + const association = {number: 7, head: {sha: "b".repeat(40), repo: {id: 88}}, base: {ref: "main", sha: "c".repeat(40), repo: {id: 88}}}; + return { + attestation, + run: {id: 1234, run_attempt: 2, event: attestation.run.event, head_sha: attestation.run.workflow_sha, head_branch: "main", path: ".github/workflows/ci.yml", name: "CI", status: "completed", conclusion: attestation.run.conclusion, repository: {id: 88, full_name: "Makepad-fr/postgres"}, pull_requests: [association], html_url: "https://github.example/run/1234"}, + jobs: {total_count: 1, jobs: [job]}, + job, + pullRequest: {number: 7, head: {sha: "b".repeat(40), repo: {full_name: "Makepad-fr/postgres"}}, base: {ref: "main", sha: "c".repeat(40), repo: {full_name: "Makepad-fr/postgres"}}}, + runnerLookupStatus: 404, + }; +}; + +test("accepts fresh hypervisor-signed teardown evidence from immutable Launcher App sender", () => { + assert.equal(verify(signedEvent()).run.job_id, 5678); +}); + +test("rejects forged evidence", () => { + const event = signedEvent(); + event.client_payload.attestation.run.head_sha = "c".repeat(40); + assert.throws(() => verify(event), /signature verification failed/); +}); + +test("rejects stale evidence", () => { + const attestation = baseAttestation(); + attestation.issued_at = "2026-09-05T09:40:00Z"; + assert.throws(() => verify(signedEvent(attestation)), /stale or from the future/); +}); + +test("rejects an unapproved base image digest", () => { + assert.throws(() => verify(signedEvent(), {approvedDigest: "c".repeat(64)}), /not approved/); +}); + +test("rejects incomplete hypervisor teardown", () => { + const attestation = baseAttestation(); + attestation.teardown.network = false; + assert.throws(() => verify(signedEvent(attestation)), /teardown is incomplete/); +}); + +test("rejects mutable sender-name forgery with the wrong numeric App sender ID", () => { + assert.throws(() => verify(signedEvent(baseAttestation(), 9002)), /dedicated Launcher App/); +}); + +test("rejects authoritative runner mismatch and a still-registered runner", () => { + const mismatch = authoritative(); + mismatch.job.runner_id = 45; + assert.throws(() => validateAuthoritativeEvidence(mismatch), /runner identity differs/); + const registered = authoritative(); + registered.runnerLookupStatus = 200; + assert.throws(() => validateAuthoritativeEvidence(registered), /still registered/); + const noListAuthority = authoritative(); + noListAuthority.runnerListStatus = 403; + assert.throws(() => validateAuthoritativeEvidence(noListAuthority), /absence is uncertain/); +}); + +test("accepts a failing test result only as a failing check after verified teardown", () => { + const attestation = baseAttestation(); + attestation.run.conclusion = "failure"; + const verified = validateAuthoritativeEvidence(authoritative(attestation)); + assert.equal(verified.conclusion, "failure"); +}); + +test("accepts protected-main push evidence only when source and workflow SHAs match", () => { + const attestation = baseAttestation(); + attestation.run.event = "push"; + attestation.run.head_sha = attestation.run.workflow_sha; + const evidence = authoritative(attestation); + evidence.run.pull_requests = []; + evidence.pullRequest = null; + const verified = validateAuthoritativeEvidence(evidence); + assert.equal(verified.event, "push"); + const mismatch = baseAttestation(); + mismatch.run.event = "push"; + assert.throws(() => verify(signedEvent(mismatch)), /identity or conclusion is invalid/); +}); + +test("rejects an authoritative workflow execution SHA mismatch", () => { + const evidence = authoritative(); + evidence.job.head_sha = "d".repeat(40); + assert.throws(() => validateAuthoritativeEvidence(evidence), /runner identity differs/); +}); + +test("rejects a pull association whose exact base SHA differs from the workflow SHA", () => { + const evidence = authoritative(); + evidence.run.pull_requests[0].base.sha = "d".repeat(40); + assert.throws(() => validateAuthoritativeEvidence(evidence), /head and base identities/); +}); + +test("rejects replay for the same run attempt and Checks App", () => { + assert.throws(() => assertNoAttestationReplay({ + appID: "500", + prefix: "postgres-ci:pull_request_target:1234:2:", + existing: {total_count: 1, check_runs: [{app: {id: 500}, external_id: `postgres-ci:pull_request_target:1234:2:${"A".repeat(43)}`}]}, + }), /replay detected/); +}); diff --git a/scripts/verify-postgres-ci-jit-result.py b/scripts/verify-postgres-ci-jit-result.py new file mode 100755 index 0000000..388fe85 --- /dev/null +++ b/scripts/verify-postgres-ci-jit-result.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Bind a disposed JIT runner to its exact authoritative GitHub result.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +EXPECTED_REPOSITORY = "Makepad-fr/postgres" +EXPECTED_LABELS = {"self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"} + + +def positive(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{label} must be a positive integer") + return value + + +def load_object(path: Path, label: str) -> dict[str, object]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 2 * 1024 * 1024: + raise ValueError(f"{label} must be a small regular file") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def validate( + run: dict[str, object], + response: dict[str, object], + *, + run_id: int, + attempt: int, + job_id: int, + event: str, + source_sha: str, + workflow_sha: str, + runner_id: int, + runner_name: str, + runner_group_id: int, +) -> str: + for value, label in ( + (run_id, "run ID"), + (attempt, "run attempt"), + (job_id, "job ID"), + (runner_id, "runner ID"), + (runner_group_id, "runner group ID"), + ): + positive(value, label) + if event not in {"pull_request_target", "push"}: + raise ValueError("unsupported workflow event") + if not re.fullmatch(r"[a-f0-9]{40}", source_sha) or not re.fullmatch(r"[a-f0-9]{40}", workflow_sha): + raise ValueError("source and workflow SHAs must be lowercase commit IDs") + if not re.fullmatch(r"postgres-ci-jit-j[1-9][0-9]{0,15}-[a-f0-9]{16}", runner_name): + raise ValueError("runner name is outside the deterministic JIT namespace") + + jobs = response.get("jobs") + total = response.get("total_count") + if not isinstance(jobs, list) or isinstance(total, bool) or total != len(jobs): + raise ValueError("authoritative attempt-job response is truncated") + matches = [value for value in jobs if isinstance(value, dict) and value.get("id") == job_id] + if len(matches) != 1: + raise ValueError("exact job is not unique in the authoritative run attempt") + job = matches[0] + raw_labels = job.get("labels") + if not isinstance(raw_labels, list) or not all(isinstance(value, str) for value in raw_labels): + raise ValueError("authoritative job labels are invalid") + actual_labels = [value.lower() for value in raw_labels] + if len(actual_labels) != len(EXPECTED_LABELS) or set(actual_labels) != EXPECTED_LABELS: + raise ValueError("authoritative job identity does not match this hypervisor execution") + + repository = run.get("repository") + if not isinstance(repository, dict): + raise ValueError("authoritative repository identity is missing") + repository_id = positive(repository.get("id"), "repository ID") + if ( + run.get("id") != run_id + or run.get("run_attempt") != attempt + or run.get("event") != event + or run.get("head_sha") != workflow_sha + or run.get("head_branch") != "main" + or run.get("name") != "CI" + or run.get("path") != ".github/workflows/ci.yml" + or run.get("status") != "completed" + or repository.get("full_name") != EXPECTED_REPOSITORY + or job.get("run_id") != run_id + or job.get("id") != job_id + or job.get("head_sha") != workflow_sha + or job.get("workflow_name") != "CI" + or job.get("runner_id") != runner_id + or job.get("runner_name") != runner_name + or job.get("runner_group_id") != runner_group_id + or job.get("runner_group_name") != "Postgres PR Ephemeral" + or job.get("name") != "policy-and-integration" + or job.get("status") != "completed" + ): + raise ValueError("authoritative job identity does not match this hypervisor execution") + + if event == "pull_request_target": + associations = run.get("pull_requests") + if not isinstance(associations, list) or len(associations) != 1 or not isinstance(associations[0], dict): + raise ValueError("authoritative pull request association differs from the requested source") + association = associations[0] + head = association.get("head") + base = association.get("base") + if not isinstance(head, dict) or not isinstance(base, dict): + raise ValueError("authoritative pull request association differs from the requested source") + head_repository = head.get("repo") + base_repository = base.get("repo") + if ( + not isinstance(head_repository, dict) + or not isinstance(base_repository, dict) + or head.get("sha") != source_sha + or head_repository.get("id") != repository_id + or base_repository.get("id") != repository_id + or base.get("ref") != "main" + or base.get("sha") != workflow_sha + ): + raise ValueError("authoritative pull request association differs from the requested source") + elif source_sha != workflow_sha: + raise ValueError("protected-main push source differs from its workflow SHA") + + run_conclusion = run.get("conclusion") + job_conclusion = job.get("conclusion") + if run_conclusion != job_conclusion or run_conclusion not in {"success", "failure"}: + raise ValueError("authoritative run and job conclusions are not an exact supported result") + return run_conclusion + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("run_file", type=Path) + parser.add_argument("jobs_file", type=Path) + parser.add_argument("run_id", type=int) + parser.add_argument("attempt", type=int) + parser.add_argument("job_id", type=int) + parser.add_argument("event") + parser.add_argument("source_sha") + parser.add_argument("workflow_sha") + parser.add_argument("runner_id", type=int) + parser.add_argument("runner_name") + parser.add_argument("runner_group_id", type=int) + arguments = parser.parse_args() + print( + validate( + load_object(arguments.run_file, "run response"), + load_object(arguments.jobs_file, "jobs response"), + run_id=arguments.run_id, + attempt=arguments.attempt, + job_id=arguments.job_id, + event=arguments.event, + source_sha=arguments.source_sha, + workflow_sha=arguments.workflow_sha, + runner_id=arguments.runner_id, + runner_name=arguments.runner_name, + runner_group_id=arguments.runner_group_id, + ) + ) + + +if __name__ == "__main__": + main() From 62496a7aa4849d8c1016570a12fafb40292efcb5 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 04:28:12 +0200 Subject: [PATCH 12/20] test(postgres): codify hardened release operations --- README.md | 452 +++++++++++++++++++++-- scripts/validate-postgres-config.sh | 551 ++++++++++++++++++++++++++-- 2 files changed, 947 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index cd5dee1..d8086e3 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,18 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database - `bootstrap/brio-staging-app.sql`: idempotent SQL bootstrap for the Brio staging application database - `bootstrap/keycloak-brio-staging.sql`: targeted idempotent bootstrap for Brio's Keycloak database +- `bootstrap/vif-app.sql`: VIF application bootstrap that reads its password only through `\getenv` - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets - `scripts/run-brio-encrypted-backup.sh`: streaming CMS-encrypted backup for one allowlisted Brio database - `scripts/verify-brio-encrypted-restore.sh`: destructive two-database Brio restore verification - `scripts/deploy-postgres-stack.sh`: checked-in remote Swarm preflight, deployment, convergence, and database-provisioning entrypoint +- `scripts/deploy-brio-canary-postgres.sh`: prevalidated, snapshot-backed canary host/Swarm transaction, Brio bootstrap, access probes, and backup verification +- `scripts/deploy-brio-identity-db-host.sh`: guarded standalone DB-VM snapshot, durable recovery evidence, rollback, HBA update, Brio identity bootstrap, access probes, and backup verification +- `scripts/ensure-brio-tmp-cleaner.sh`: persistent host guard that expires abandoned Brio deployment directories after three hours while preserving recovery-marked runs +- `scripts/install-keycloak-cohort-capture-host.sh`: idempotent root installer for the digest-bound cohort SSH forced command and persistent file/Docker-resource cleaner +- `scripts/clean-keycloak-cohort-resources.sh`: fail-closed cleanup of expired labeled cohort containers, networks, and dump directories +- `scripts/test-brio-deployment-failures.sh`: isolated behavioral fault tests for promotion, stack, signal, rollback, evidence-retention, and symlink failures ## Networks @@ -62,12 +69,36 @@ docker node update --label-add infra.makepad.postgres=true ## Deployment -Use the manual GitHub Actions workflow in this repository. +Use the manual GitHub Actions workflow in this repository for Swarm canary and +ordinary shared-stack deployments. Brio's topology is deliberately split: +`brio_staging` belongs to the canary application Swarm, while +`keycloak_brio_staging` is provisioned only by the separate `Deploy Brio +Identity Database` workflow on the standalone database VM. The production +Swarm override contains no Brio identity backup service and must never be used +to bootstrap or back up the Brio Keycloak database. + +Both deployment workflows require the protected `Postgres Deploy` runner group +and repository-scoped `makepad-postgres-deploy` label. Protected-main CI uses +the separate `Postgres Main CI` group and `makepad-postgres-main-ci` label. +Pull-request code is never executed on either persistent host; the disposable +PR boundary is documented below. A generic Makepad runner cannot execute these +jobs. Both deployment workflows also reject every Git ref except `main`; +configure the GitHub environments with the same deployment-branch restriction +and required reviewers. + +Swarm deployments share one target-wide concurrency group across canary and +production. Every run uploads to a unique +`/.deploy/postgres--` bundle; no workflow writes +a shared remote `stack.yml`. Before credentials are uploaded, both deployment +paths install or validate a restricted, restartable host cleaner. It removes +only abandoned top-level `/tmp/postgres-brio-*` directories older than three +hours, covering runner interruption or loss of the SSH session in addition to +the workflows' immediate `always()` cleanup. The dedicated database VM currently runs standalone Docker Compose rather than -joining the application Swarm. On that host, deploy the same TLS and backup -policy with `compose.host.yml` after provisioning the certificate, key, CA, -password files, backup directory, and committed HBA policy: +joining the application Swarm. Do not run the following recovery-level command +for the ordinary Brio identity rollout; the protected workflow supplies and +validates the reviewed inputs: ```bash : "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set to the DB certificate SAN hostname}" @@ -77,8 +108,33 @@ docker compose --env-file envs/production/.env.db -f compose.host.yml up -d --pu The standalone DB-VM deployment additionally requires `MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST` to be exported as the exact DB -VM hostname present in the PostgreSQL server certificate SAN. The encrypted +VM host or IP present in the PostgreSQL server certificate SAN. The encrypted identity backup refuses any connection mode other than `verify-full`. +The current host private key contract is numeric owner/group `70:70`, mode +`0400`; the deploy preflight requires that exact verified live form without +copying the key into CI. +The workflow also refuses a Swarm node, pins Compose project `postgres`, and +requires the existing `postgres-postgres-1` container to have the exact Compose +project/service labels, host networking, pinned image, healthy state, and +`/var/lib/makepad/postgres:/var/lib/postgresql/data` read-write bind. It requires an exact restart +acknowledgement and confirmation of a current successful encrypted restore +test, and renders the Keycloak application HBA rule to one verified egress +`/32`. Candidate Compose, HBA, SQL, and scripts are staged and prevalidated. +Immediately before the first managed-file mutation, the deploy snapshots the +exact prior Compose inputs, HBA, scripts, affected backup credentials, and the +complete standalone Brio identity backup directory. Any +error or `HUP`/`INT`/`TERM` after that boundary restores the snapshot, recreates +the prior Compose project, and requires the exact PostgreSQL target to return +healthy. The candidate promotion recreates the fixed Compose project so the +updated bind-mounted HBA is loaded, then verifies TLS identity, plaintext rejection, +cross-database rejection, read-only backup access, and a newly published CMS +backup. The rollback is disarmed only after all those local probes and the fresh +encrypted backup pass. Local DB-VM probes and the host-network backup container set +`PGHOSTADDR=127.0.0.1` while retaining the certificate host in `PGHOST`, so +transport is deterministically local while `verify-full` still checks the +declared DNS or IP SAN. The identity backup HBA rule is correspondingly limited +to `127.0.0.1/32`; only the Keycloak application role receives the separately +rendered egress `/32` rule. The host deployment preserves the existing host-network endpoint used by Keycloak while requiring TLS and SCRAM for `runtrace`, `keycloak_runtrace`, @@ -96,6 +152,7 @@ Required environment secrets: - `DEPLOY_SSH_PORT` - `DEPLOY_SSH_USER` - `DEPLOY_SSH_PRIVATE_KEY` +- `DEPLOY_SSH_KNOWN_HOSTS` - `DEPLOY_REMOTE_DIR` - `DEPLOY_STACK_NAME` - `DEPLOY_CATWLK_DB_NETWORK` @@ -106,6 +163,318 @@ Canary additionally requires: - `DEPLOY_BRIO_STAGING_DB_NETWORK` set exactly to `makepad_brio_staging_db`; the workflow rejects alternate names so Brio and PostgreSQL cannot drift onto disconnected look-alike networks +- `POSTGRES_CANARY_SUPERUSER_PASSWORD` +- `POSTGRES_CA_PEM` +- `POSTGRES_SERVER_CERT_PEM`, whose SAN includes + `makepad-postgres-brio-staging` +- `POSTGRES_SERVER_KEY_PEM`, matching that certificate +- `BRIO_STAGING_DB_PASSWORD` +- `BRIO_STAGING_BACKUP_DB_PASSWORD` +- `BRIO_BACKUP_RECIPIENT_CERT_PEM`, containing only the public recovery + certificate + +The canary workflow materializes these only in mode-0700 job directories with +mode-0600 files, transfers them to one job-scoped remote `/tmp` directory, +and removes local and remote job material in `always()` cleanup steps. Before +any mutation it validates exact managed destinations (including every existing +path component), immutable object digests, all network contracts, the current +stack identity, and both Compose and Swarm renderings. It atomically publishes +a root-owned transaction journal under +`/var/lib/makepad/postgres-recovery/brio-canary/-` containing the +exact managed-file, database, ACL/default-privilege, backup-directory, and +Swarm service pre-state. Replacements are staged beside their destinations and +promoted with same-filesystem renames. Any later error or signal restores the +database and host snapshot, rolls back each changed pre-existing service once +and verifies its exact prior Spec hash, and removes only objects labeled with +the current deployment ID. A SIGKILL leaves the journal for mandatory recovery +at the beginning of the next deployment. A failed compensation leaves +`RECOVERY_REQUIRED`; normal workflow and TTL cleanup preserve it. Passwords +are read from mounted files inside short-lived containers and never placed in +Docker or `psql` command arguments. + +Canary acceptance uses a hardened one-shot backup container and does not force +a second update of the PostgreSQL or backup Swarm service. Because `docker +stack deploy` does not prune a service removed from a Compose file, a legacy +`${stack}_keycloak_brio_staging_backup` service causes deployment to fail +closed. Inventory its exact Spec and retire that exact service only through a +separately reviewed operation after the standalone `keycloak_brio_staging` +backup has been accepted; never use a broad stack prune for this migration. + +The protected `staging-brio-identity-db` GitHub environment requires: + +- secrets `BRIO_IDENTITY_DB_DEPLOY_SSH_HOST`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_PORT`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_USER`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY`, and + `BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS` +- secrets `KEYCLOAK_BRIO_STAGING_DB_PASSWORD`, + `KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD`, and + `BRIO_BACKUP_RECIPIENT_CERT_PEM` +- variable `BRIO_IDENTITY_DB_HOSTNAME`, equal to the server-certificate SAN + used by Keycloak for the standalone DB endpoint; the current value is + `65.21.134.125`, which is present as an IP SAN in the deployed certificate +- variable `BRIO_KEYCLOAK_DB_SOURCE_CIDR`, equal to Keycloak's verified single + database egress `/32` (currently `88.99.209.165/32`); the Brio WireGuard + tunnel is for Keycloak-to-MailDev traffic and is not a database route + +Protect that environment with required reviewers. Dispatch `Deploy Brio +Identity Database` from `main`, type +`restart-standalone-postgres-for-brio-staging`, and confirm the restore gate +only after recording a successful restore of the current encrypted backup. +The SSH account must be non-root but has Docker access, which is root-equivalent +and therefore restricted to this reviewed deployment path. + +All Brio workflow-supplied deployment, release, and CI credentials have one +canonical Proton Pass item and are copied only into the named, protected +GitHub environment. They are not repository or organization secrets, runner +configuration, or files on a persistent runner. Existing shared PostgreSQL +production credentials keep their established Proton items; this rollout does +not rename or duplicate them. Non-secret deployment constants remain protected +GitHub environment variables. The exact Brio inventory is: + +| Canonical Proton Pass item | Protected GitHub environment | Exact mirrored fields | +| --- | --- | --- | +| `Hetzner Database Server makepad` | `canary`, `production`, `staging-brio-identity-db`, and `keycloak-cohort-restore` | canonical SSH fields `DEPLOY_SSH_HOST`, `DEPLOY_SSH_PORT`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_PRIVATE_KEY`, `DEPLOY_SSH_KNOWN_HOSTS`; mirror the same reviewed values under the workflow aliases `BRIO_IDENTITY_DB_DEPLOY_SSH_HOST`, `BRIO_IDENTITY_DB_DEPLOY_SSH_PORT`, `BRIO_IDENTITY_DB_DEPLOY_SSH_USER`, `BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY`, `BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, `KEYCLOAK_COHORT_DB_SSH_USER`, `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, and `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS` only in their named environments | +| `Brio Staging - PostgreSQL` | `canary` and `staging-brio-identity-db` | secrets `POSTGRES_CANARY_SUPERUSER_PASSWORD`, `BRIO_STAGING_DB_PASSWORD`, `BRIO_STAGING_BACKUP_DB_PASSWORD`, `KEYCLOAK_BRIO_STAGING_DB_PASSWORD`, and `KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD` | +| `Brio Staging - PKI and Backup Keys` | `canary` and `staging-brio-identity-db` | secrets `POSTGRES_CA_PEM`, `POSTGRES_SERVER_CERT_PEM`, `POSTGRES_SERVER_KEY_PEM`, and public recipient certificate `BRIO_BACKUP_RECIPIENT_CERT_PEM` | +| `PostgreSQL · Brio identity release orchestrator` | `release-brio-identity-db` | secret `KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN` | +| `PostgreSQL · Keycloak cohort source reader` | `keycloak-cohort-restore` | secret `KEYCLOAK_COHORT_SOURCE_TOKEN` | +| `Makepad Docker Hardened Images` | `keycloak-cohort-restore` | canonical fields `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT`, mirrored as secrets `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD` | +| `PostgreSQL · PR Checks App` | `postgres-ci-attestation` | variable `POSTGRES_PR_CHECK_APP_ID` and secret `POSTGRES_PR_CHECK_APP_PRIVATE_KEY` | +| `PostgreSQL · JIT Launcher App` | `postgres-ci-attestation` | public variable `POSTGRES_CI_LAUNCHER_APP_SENDER_ID`; private App fields remain on the controller host only | +| `PostgreSQL · JIT hypervisor attestation` | `postgres-ci-attestation` | public variables `POSTGRES_CI_ATTESTATION_PUBLIC_KEY` and `POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256`; the signing key remains on the hypervisor only | + +The `canary`, `production`, `staging-brio-identity-db`, and +`keycloak-cohort-restore` environments also hold reviewed non-secret constants +such as `DEPLOY_REMOTE_DIR`, stack/network names, +`BRIO_IDENTITY_DB_HOSTNAME`, and `BRIO_KEYCLOAK_DB_SOURCE_CIDR`. Store them as +environment variables, not duplicated password-vault secrets. + +Use `pass-cli` from an approved administrator workstation and stream secret +values over standard input: + +```bash +pass-cli item view --item-title '' --field '' \ + | gh secret set '' --env '' --repo 'Makepad-fr/postgres' +``` + +Never place values in command arguments, temporary files, shell history, +Actions logs, or issue text. Mirror non-secret variables with the same reviewed +reconciliation session, compare their GitHub read-back, and record only item +IDs, field names, timestamps, and non-secret fingerprints in the deployment +change record. Every listed environment, including `production`, must have +exactly one custom branch deployment policy whose type is `branch` and whose +name is exactly `main`; GitHub's generic "protected branches" option is not an +equivalent restriction. A release is blocked if an item or field is missing, +if that exact policy or required reviewers are absent, or if GitHub differs +from the reviewed Proton version. + +Audit all six policies without changing provider state: + +```bash +python3 scripts/reconcile-github-environment-main-policy.py audit +``` + +Reconcile one environment only after reviewing its current protection rules. +The helper preserves the current wait timer and required reviewers, creates +the exact `main` branch rule before removing broader custom rules, uses bounded +fail-closed pagination, and verifies the provider read-back. Applying requires +an explicit repository/environment confirmation; for production use: + +```bash +python3 scripts/reconcile-github-environment-main-policy.py apply \ + --environment production \ + --confirm Makepad-fr/postgres:production:exact-main +``` + +Run this only from an administrator workstation whose `gh` session has +environment-administration permission. The helper never reads or writes +environment secrets. + +Host-only JIT Launcher, attestation-signing, runner-controller, and alert +credentials are also canonical in the Proton items documented below, but are +intentionally never copied into Actions; only their public identities and +reviewed digests are mirrored to `postgres-ci-attestation`. + +If automatic standalone rollback cannot re-establish the exact healthy target, +the deploy script first deletes all incoming job credentials, then retains a +root-owned mode-0700 recovery bundle under +`/var/lib/makepad/postgres-recovery/brio-identity/-` and leaves a +non-secret `RECOVERY_REQUIRED` marker in the run-scoped `/tmp` directory. The +workflow and three-hour cleaner deliberately skip that marked directory. +Operators must inspect and resolve the retained evidence from the DB VM, then +remove both exact run directories only after recovery is complete. The bundle +may contain the former managed backup credential and must never be copied into +Actions artifacts or ordinary logs. + +The database-VM workflow's local probes do not prove the public route from the +Keycloak host. Release is deliberately two-phase. A successful `Deploy Brio +Identity Database` run stops after uploading exactly one 35-day artifact named +`brio-db-deployment-evidence--` containing the single canonical +`brio-db-deployment-evidence.json` file. That artifact says only that the +standalone host deployment is ready; it never claims Keycloak-path success. + +A reviewer then dispatches the separate protected `Release Brio Identity +Database` workflow with that exact PostgreSQL run ID and attempt. The protected +`release-brio-identity-db` environment supplies the dedicated +`KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN`. Its canonical credential must first be +stored in Proton Pass item `PostgreSQL · Brio identity release orchestrator`, +then mirrored only to that protected environment. Use a dedicated fine-grained +token or short-lived installation-token broker restricted to exactly +`Makepad-fr/postgres` and `Makepad-fr/keycloak`, with Metadata read, Contents +read, and Actions read/write; it must have no Administration, Environments, +Secrets, Members, Packages, Deployments, or organization-runner permission and +must never be an ambient maintainer PAT. Record the App installation or token +owner and a non-secret fingerprint in the same Proton item. The orchestrator independently requires the +PostgreSQL run to be completed successfully on `main`, fully paginates and validates the exact +artifact name, size, one-file ZIP shape, schema, commit, run, and attempt, +resolves the current exact Keycloak `main` SHA, and dispatches +`Makepad-fr/keycloak/.github/workflows/verify-brio-database.yml`. It accepts +only the exact completed verifier run and its single +`brio-db-path-attestation--` artifact; it never creates, +copies, or synthesizes an attestation itself. +The bearer credential is supplied to each API request through curl's stdin-only +configuration and is never materialized in the persistent runner filesystem. + +The protected `Verify Brio Identity Database Path` workflow has the run name +`Verify Brio DB path for PostgreSQL run ` and publishes the +`brio-db-path-ok` evidence. Its dedicated Keycloak runner verifies +`sslmode=verify-full` to `65.21.134.125`, the certificate IP SAN, exact +database/role, TLS 1.2 or 1.3, and server-observed source `88.99.209.165`; no +Keycloak database credential is granted to the PostgreSQL runner. + +Pull requests use protected-base `pull_request_target` workflow code and reject +forks before checking out the exact same-repository head. The public repository +does not have a persistent PR runner. A dedicated root-only hypervisor queue +controller authorizes the exact queued run, attempt, job, PR head, PR base SHA, +protected workflow SHA, group, and label through GitHub's APIs. It durably +records a deterministic launch/resource manifest before launch, obtains a +one-job JIT configuration, and boots a fresh +self-contained qcow2 VM with the exclusive +`makepad-postgres-pr-ephemeral` label. The hypervisor firewall denies private, +WireGuard, link-local, metadata, multicast, IPv6, and hypervisor destinations; +only public DNS and TLS egress are allowed. The guest contains no repository, +deployment, Proton Pass, App, SSH, cloud, or service credential. + +After the job stops, the hypervisor destroys and proves absent the VM, disk, +cloud-init seed, network, firewall table, and GitHub runner registration. Only +then may it sign canonical `makepad.postgres.ci-attestation.v1` evidence with +its root-only Ed25519 key and dispatch it with the dedicated Launcher App. A +physically separate `makepad-postgres-ci-attestor` host in the selected-workflow +`org/Postgres PR Ephemeral` group runs only protected +`pr-ci-result.yml`; it has no Docker, deployment, or Launcher credentials. It +verifies the immutable numeric Launcher-App sender ID, signature, freshness, +nonce replay, reviewed base-image digest, exact authoritative run/job/runner +identity and conclusion, all teardown flags, and an independent 404 lookup for +the removed runner before the Checks-only App can publish the required +`postgres-ci` result. Failed or uncertain cleanup never produces a successful +check. Main pushes run independently on `org/Postgres Main CI`. The systemd +service uses control-group termination. On every controller start, all +`launching` or `recovery-required` records are reconciled before queue polling: +the exact VM, network, nftables table, job directory, and named runner +registration must all be proven absent. Recovered jobs are never executed or +attested again. Authoritative run/job completion is polled for a bounded period +after teardown to tolerate API propagation without rerunning untrusted code. + +Reconcile the four exact selected-workflow groups with +`scripts/configure-postgres-ci-runner-group.sh`, streaming its organization +runner-controller credential on stdin. Because `Makepad-fr/postgres` is public, +the groups explicitly allow public repositories but select only this exact +repository and protected-main workflow files. Repository-level runners and +runners exposed by unrelated groups are rejected. No persistent runner may +carry the JIT-only label. Install and supervise +`host/systemd/postgres-ci-queue-controller.service`; an abnormal launcher exit +must trigger the independent host alert service and no blind retry occurs. + +Long-lived CI controller material is canonical in Proton Pass before it is +installed at its narrow runtime boundary: + +| Proton Pass item | Exact runtime fields and authority | +| --- | --- | +| `PostgreSQL · PR Checks App` | protected `postgres-ci-attestation` environment variable `POSTGRES_PR_CHECK_APP_ID` and secret `POSTGRES_PR_CHECK_APP_PRIVATE_KEY`; App installed only on this repository with Metadata read, Actions read, Checks write, and organization self-hosted-runners read | +| `PostgreSQL · JIT Launcher App` | root-only hypervisor `POSTGRES_CI_LAUNCHER_APP_ID`, `POSTGRES_CI_LAUNCHER_APP_INSTALLATION_ID`, and mode-0400 `POSTGRES_CI_LAUNCHER_APP_PRIVATE_KEY_FILE`; repository variable `POSTGRES_CI_LAUNCHER_APP_SENDER_ID`; App installed only on this repository with Metadata read, Actions read, Contents write for repository dispatch, Issues write for secondary alerts, Pull requests read, and organization self-hosted-runners write | +| `PostgreSQL · JIT hypervisor attestation` | root-only mode-0400 `POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE`; repository variable `POSTGRES_CI_ATTESTATION_PUBLIC_KEY`; reviewed repository variable and root-only value `POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256`/`POSTGRES_CI_BASE_IMAGE_SHA256` | +| `PostgreSQL · runner-group controller` | administrator workstation input streamed to `scripts/configure-postgres-ci-runner-group.sh`; organization runner-group write and repository Metadata read only, never installed on a runner or hypervisor | +| `PostgreSQL · CI hypervisor alert` | root-only host alert URL file consumed only by the systemd `OnFailure` handler; never mirrored to GitHub Actions | + +The Launcher and Checks Apps are different installations and keys. Store their +numeric IDs, installation IDs, public-key fingerprints, approved base-image +digest, and rotation history beside the Proton items so reconciliation can +compare identities without printing secrets. The hypervisor's immutable base +image is root-owned, non-writable, has no backing/data chain, and is verified by +the reviewed SHA-256 both before and after each full per-job copy. + +## Keycloak 26.7.3 cohort restore evidence + +Before the six-realm Keycloak release, dispatch protected workflow `Verify +Keycloak Cohort Restore Compatibility` with the exact lowercase current +Keycloak protected-main SHA. There is no mutable rollout repository variable. +The workflow resolves `Makepad-fr/keycloak` main independently, checks out that +exact release, and verifies its pinned +`dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f` +runtime and upstream version `26.7.3`. + +The protected release runner captures fresh custom-format, no-owner, +no-privilege dumps of exactly `keycloak_betacrew`, `keycloak_catwlk`, +`keycloak_makepad`, `keycloak_runtrace`, `keycloak_vestiaire`, and +`keycloak_vif` from the exact healthy production Compose container. Every dump +is structurally inspected. Each is then restored into a fresh internal Docker +network and disposable PostgreSQL instance. Catwlk uses the custom DHI-derived +provider image built from the exact checked-out Keycloak release; the other five +instances use the pinned base image. Each runtime must become ready and the v2 +secret-safe fingerprints for realm settings/themes/SMTP, authentication flows, +roles/composites, clients/scopes/mappers, identity providers, components, and +required actions must remain stable. Canonical sorted rows are streamed directly +into SHA-256; raw configuration values are never written or logged. The check +fails when a required persistence table disappears. Because Keycloak's schema is +internal, a reviewed schema change requires coordinated fingerprint/evidence +schema revisions rather than a silent fallback. +No dump is uploaded as an Actions artifact, and remote and local copies are +deleted in the always-cleanup step. + +Success uploads exactly one artifact named +`keycloak-cohort-restore-evidence--` containing only canonical +`keycloak-cohort-restore-evidence.json`, schema +`makepad.keycloak-cohort-restore-evidence.v2`. It binds the exact PostgreSQL +workflow/run/attempt/main SHA, exact Keycloak release SHA/base image/version, +the immutable locally built Catwlk image ID, fingerprint schema, and the sorted +six-instance list. Each entry contains its slug, database, fresh backup SHA-256, +exact runtime identity, category and combined hashes, and `passed` restore, +Keycloak-startup, and configuration-regression statuses. The Keycloak deployment +consumer must resolve that exact completed +successful main-branch workflow run and artifact; operator assertions are not +evidence. + +The `keycloak-cohort-restore` environment is protected to `main` with required +reviewers. Provision its long-lived fields in Proton Pass first, then mirror +only to that environment: + +| Proton Pass item | Protected environment fields | +| --- | --- | +| `PostgreSQL · Keycloak cohort source reader` | `KEYCLOAK_COHORT_SOURCE_TOKEN`, dedicated token/App broker restricted to `Makepad-fr/keycloak` with Metadata and Contents read only | +| `Hetzner Database Server makepad` | canonical `DEPLOY_SSH_*` fields mirrored to `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, and `KEYCLOAK_COHORT_DB_SSH_USER`; dedicated non-root Docker-capable DB capture account only | +| `Makepad Docker Hardened Images` | canonical `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT` fields mirrored to `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD`, with read-only pull access to the exact reviewed Keycloak image | + +The source token has no Actions write, Checks, Administration, Environments, +Secrets, Deployments, or organization permissions. The DB capture account has +no interactive command. Before adding its GitHub secret, an operator runs +`scripts/install-keycloak-cohort-capture-host.sh +` as root on the database host. The installer writes +a root-owned `authorized_keys` forced command, disables forwarding and TTYs, +validates sshd, installs the reviewed capture helper, and enables the persistent +cohort cleanup timer. The workflow first proves the installed helper and +cleaner's exact SHA-256 digests, active timer, and successful last cleaner result, +then may issue only validated +`probe`/`capture`/`fetch`/`cleanup` commands. Checked-out code is never copied to +or executed on the database host. + +Run `scripts/install-keycloak-cohort-cleaner.sh` as root on the protected release +runner host as well. The persistent timer removes expired exactly labeled cohort +containers and networks plus `/tmp/postgres-keycloak-cohort-*` directories after +three hours, including after host downtime. Disposable database and Keycloak +passwords are mode-0400 mounted files rather than Docker configuration +environment values. The ordinary `always()` step still removes local and remote +material immediately after a run. Production additionally requires: @@ -113,12 +482,28 @@ Production additionally requires: - `DEPLOY_VIF_DB_PASSWORD` Production can override the VIF database and role names with `DEPLOY_VIF_DB_NAME` and `DEPLOY_VIF_DB_USER`; both default to `vif`. +The VIF password is written only to a mode-0600 file inside a mode-0700, +job-scoped runtime directory. It is never persisted in `.env.deploy` or passed +through a `psql -v` argument; the mounted bootstrap reads it with `\getenv`. `DEPLOY_SSH_USER` must be a non-root deployment account with the Docker permissions needed to create overlay networks and deploy the stack. The workflow rejects `DEPLOY_SSH_USER=root`. -Before the first deployment, provision the PostgreSQL superuser password as a non-empty root-owned file on the database node. The production default path is `/etc/makepad/secrets/postgres-superuser-password`; canary uses `/etc/makepad/secrets/postgres-canary-superuser-password`. Keep the file outside the repository, set mode `0600`, and override `MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH` only when the host secret manager materializes it elsewhere. PostgreSQL receives the value through `POSTGRES_PASSWORD_FILE`, and deployment helpers mount the same file read-only instead of placing the password in command arguments or tracked environment files. - -Provision a private-CA-issued PostgreSQL server certificate before deployment. Its SANs must include every hostname clients verify, including `makepad-postgres`, `makepad-postgres-brio-staging`, and the DB VM hostname used by Keycloak. Keep the unencrypted private key outside git and create versioned Swarm objects on the database manager. Canary intentionally requires new `v2` objects so the older certificate cannot be reused without the Brio alias: +Before an ordinary non-Brio deployment, provision the PostgreSQL superuser +password as a non-empty root-owned mode-0600 file on the database node. The +production default is `/etc/makepad/secrets/postgres-superuser-password`. +Canary's `/etc/makepad/secrets/postgres-canary-superuser-password` is instead +provisioned by its protected workflow environment. PostgreSQL receives the +value through `POSTGRES_PASSWORD_FILE`; helpers mount it read-only instead of +placing it in command arguments or tracked environment files. + +Provision a private-CA-issued PostgreSQL server certificate before deployment. +Its SANs must include every hostname clients verify, including +`makepad-postgres`, `makepad-postgres-brio-staging`, and the DB VM hostname used +by Keycloak. Keep the unencrypted private key outside git. Canary automatically +creates and content-labels its required `v2` Swarm objects from protected +environment secrets, or rejects an existing name whose label/content differs. +The following commands describe the equivalent recovery operation and the +production object: ```sh docker config create makepad_postgres_tls_cert_v1 /secure/path/server.crt @@ -128,18 +513,20 @@ docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-serv ``` The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy preserves the source-restricted Fresko and BetaCrew rules described above, rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging`, and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. - -The workflow copies the checked-in remote deployment entrypoint with the deployment bundle and deploys only the PostgreSQL stack. Before deployment it validates -the password and CA files, certificate chain, seven-day expiry margin, and—for -canary—the exact `makepad-postgres-brio-staging` SAN. After the stack update it -performs a real `sslmode=verify-full` query over Brio's isolated network using -that alias; a certificate/key mismatch prevents PostgreSQL from becoming ready -and causes the canary service update to roll back. If one of the configured -database networks does not exist yet, it is created as an encrypted overlay on -the manager before deployment. After `docker stack deploy`, the workflow waits -until PostgreSQL and the environment's Brio backup service are running the exact -pinned image and the Swarm update has completed; only then does it run the TLS -database probe. +The Brio HBA policy uses fresh immutable `makepad_postgres_canary_runtrace_hba_v3` +and `makepad_postgres_runtrace_hba_v3` object names; deployed `v2` objects are +historical and must never be replaced or relabelled in place. + +The workflow copies checked-in deployment entrypoints and deploys only the +PostgreSQL stack. Canary first validates the password and CA files, certificate +chain, certificate/key match, seven-day expiry margin, and exact +`makepad-postgres-brio-staging` SAN. A mismatch fails before stack deployment. +If a configured database network is absent it is created as an encrypted +overlay; existing network drift fails closed. After exact-image convergence, +the canary entrypoint runs the idempotent app bootstrap, proves the alias and +`sslmode=verify-full` connection, proves plaintext and cross-database access are +rejected, proves the backup role is read-only, then requires a new checksummed +CMS-encrypted backup before the workflow succeeds. ## Runtrace Backup And Restore @@ -175,11 +562,11 @@ and Keycloak database live on different PostgreSQL deployments: - Canary `brio_staging_backup` attaches only to `makepad_brio_staging_db`, connects as the read-only `brio_staging_backup` role, and dumps only `brio_staging` through `makepad-postgres-brio-staging`. -- The production/DB-VM `keycloak_brio_staging_backup` dumps only +- The standalone DB-VM `keycloak_brio_staging_backup` dumps only `keycloak_brio_staging` as the read-only - `keycloak_brio_staging_backup` role. The Swarm form attaches only to the - database network; the standalone DB-VM form uses the certificate-SAN hostname - configured in `MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST`. + `keycloak_brio_staging_backup` role and uses the certificate-SAN hostname + configured in `MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST`. There is no + Swarm form of this identity backup service. Both services use the pinned official `postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825` @@ -214,11 +601,12 @@ sudo install -o root -g root -m 0444 /secure/path/brio-recipient.crt \ /etc/makepad/tls/backups/brio-recipient.crt ``` -The Swarm deploy preflight copies the tracked backup scripts, rejects symlinked -inputs, requires each backup directory to be owned by uid 999 with mode 0700, -requires each database credential to be owned by uid 999 with mode 0400, and -requires a root-owned, non-writable public recipient certificate that remains -valid for at least seven days and can create a CMS AES-256-GCM envelope. +The canary Swarm and standalone DB-VM deploy preflights copy the tracked backup +scripts, reject symlinked inputs, require each backup directory to be owned by +uid 999 with mode 0700, require each database credential to be owned by uid 999 +with mode 0400, and require a root-owned, non-writable public recipient +certificate that remains valid for at least seven days and can create a CMS +AES-256-GCM envelope. Each mounted credential is the password for its database-specific backup role; never place a PostgreSQL superuser or application-owner password in either backup credential file. The backup command refuses any role other than the @@ -400,6 +788,9 @@ Run the static deployment checks and the disposable PostgreSQL 16 bootstrap test ./scripts/test-brio-bootstrap.sh ./scripts/test-brio-encrypted-backup.sh ./scripts/test-brio-encrypted-restore.sh +./scripts/test-brio-deploy-guards.sh +./scripts/test-brio-deployment-contracts.sh +./scripts/test-brio-deployment-failures.sh ``` Run the local static checks before opening a deployment PR: @@ -410,4 +801,7 @@ bash scripts/test-runtrace-tls-policy.sh bash scripts/test-runtrace-backup.sh bash scripts/test-brio-encrypted-backup.sh bash scripts/test-brio-encrypted-restore.sh +bash scripts/test-brio-deploy-guards.sh +bash scripts/test-brio-deployment-contracts.sh +bash scripts/test-brio-deployment-failures.sh ``` diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 5cecb9a..bebc427 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -62,6 +62,7 @@ keycloak_runtrace_sql = read_required_text(repo_root / "bootstrap/keycloak-runtr openpanel_sql = read_required_text(repo_root / "bootstrap/openpanel-app.sql", "OpenPanel app SQL bootstrap") brio_sql = read_required_text(repo_root / "bootstrap/brio-staging-app.sql", "Brio staging app SQL bootstrap") keycloak_brio_sql = read_required_text(repo_root / "bootstrap/keycloak-brio-staging.sql", "targeted Brio Keycloak SQL bootstrap") +vif_sql = read_required_text(repo_root / "bootstrap/vif-app.sql", "VIF application SQL bootstrap") readme = read_required_text(repo_root / "README.md", "README") base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") host_compose = read_required_text(repo_root / "compose.host.yml", "host Compose file") @@ -87,10 +88,71 @@ production_env = read_required_text(repo_root / "envs/production/.env.db", "prod manual_deploy_workflow = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") remote_deploy_path = repo_root / "scripts/deploy-postgres-stack.sh" remote_deploy = read_required_text(remote_deploy_path, "remote deploy script") -manual_deploy = manual_deploy_workflow + "\n" + remote_deploy +canary_deploy_path = repo_root / "scripts/deploy-brio-canary-postgres.sh" +canary_deploy = read_required_text(canary_deploy_path, "Brio canary deploy script") +identity_deploy_path = repo_root / "scripts/deploy-brio-identity-db-host.sh" +identity_deploy = read_required_text(identity_deploy_path, "Brio identity DB-VM deploy script") +identity_workflow = read_required_text(repo_root / ".github/workflows/deploy-brio-identity-db.yml", "Brio identity DB-VM workflow") +release_workflow = read_required_text(repo_root / ".github/workflows/release-brio-identity-db.yml", "Brio identity database release orchestrator") +cohort_workflow = read_required_text(repo_root / ".github/workflows/verify-keycloak-cohort-restores.yml", "Keycloak cohort restore workflow") +pr_finalizer_workflow = read_required_text(repo_root / ".github/workflows/pr-ci-result.yml", "PR CI finalizer") +pr_check_publisher = read_required_text(repo_root / "scripts/publish-pr-ci-check.mjs", "PR CI check publisher") +pr_queue_controller = read_required_text(repo_root / "scripts/postgres-ci-queue-controller.mjs", "PR JIT queue controller") +pr_jit_launcher = read_required_text(repo_root / "scripts/run-postgres-ci-jit-vm.sh", "PR JIT VM launcher") +pr_jit_result_validator_path = repo_root / "scripts/verify-postgres-ci-jit-result.py" +pr_jit_result_validator = read_required_text(pr_jit_result_validator_path, "PR JIT authoritative-result validator") +pr_runner_policy = read_required_text(repo_root / "scripts/configure-postgres-ci-runner-group.sh", "runner-group policy reconciler") +environment_policy_reconciler = read_required_text(repo_root / "scripts/reconcile-github-environment-main-policy.py", "GitHub environment policy reconciler") +environment_policy_test = read_required_text(repo_root / "scripts/test-github-environment-main-policy.py", "GitHub environment policy test") +release_evidence_validator = read_required_text(repo_root / "scripts/verify-brio-release-evidence.py", "Brio release evidence validator") +cohort_evidence_validator = read_required_text(repo_root / "scripts/verify-keycloak-cohort-evidence.py", "Keycloak cohort evidence validator") +cohort_capture = read_required_text(repo_root / "scripts/capture-keycloak-cohort-backups.sh", "Keycloak cohort backup capture") +cohort_restore = read_required_text(repo_root / "scripts/restore-keycloak-cohort-backups.sh", "Keycloak cohort restore verifier") +cohort_dispatch_path = repo_root / "scripts/keycloak-cohort-capture-dispatch.sh" +cohort_dispatch = read_required_text(cohort_dispatch_path, "Keycloak cohort forced-command dispatcher") +cohort_host_installer_path = repo_root / "scripts/install-keycloak-cohort-capture-host.sh" +cohort_host_installer = read_required_text(cohort_host_installer_path, "Keycloak cohort capture-host installer") +cohort_cleaner_path = repo_root / "scripts/clean-keycloak-cohort-resources.sh" +cohort_cleaner = read_required_text(cohort_cleaner_path, "Keycloak cohort Docker-resource cleaner") +cohort_cleaner_installer_path = repo_root / "scripts/install-keycloak-cohort-cleaner.sh" +cohort_cleaner_installer = read_required_text(cohort_cleaner_installer_path, "Keycloak cohort cleaner installer") +tmp_cleaner_path = repo_root / "scripts/ensure-brio-tmp-cleaner.sh" +tmp_cleaner = read_required_text(tmp_cleaner_path, "Brio abandoned-material cleaner") +deploy_guard_test = read_required_text(repo_root / "scripts/test-brio-deploy-guards.sh", "Brio deployment guard test") +deployment_failure_test_path = repo_root / "scripts/test-brio-deployment-failures.sh" +deployment_failure_fixture_path = repo_root / "scripts/fixtures/brio-deployment-failure-fixture.sh" +deployment_failure_test = read_required_text(deployment_failure_test_path, "Brio deployment failure-injection test") +deployment_failure_fixture = read_required_text(deployment_failure_fixture_path, "Brio deployment failure-injection fixture") +manual_deploy = manual_deploy_workflow + "\n" + remote_deploy + "\n" + canary_deploy ci_workflow = read_required_text(repo_root / ".github/workflows/ci.yml", "CI workflow") +ci_runner = read_required_text(repo_root / "scripts/run-ci.sh", "CI suite runner") normalized_readme = re.sub(r"\s+", " ", readme) +for environment in ( + "canary", + "production", + "staging-brio-identity-db", + "release-brio-identity-db", + "keycloak-cohort-restore", + "postgres-ci-attestation", +): + require(f'"{environment}"' in environment_policy_reconciler, f"Environment policy reconciler must include {environment}.") +for required in ( + '"protected_branches": False', + '"custom_branch_policies": True', + '{"name": "main", "type": "branch"}', + "MAX_POLICY_PAGES = 1000", + "build_preserving_update", + "audit_environment(client, environment)", +): + require(required in environment_policy_reconciler, f"Environment policy reconciler is missing: {required}") +require('assert "production" in REQUIRED_ENVIRONMENTS' in environment_policy_test, "Environment policy test must cover production explicitly.") +require("python3 scripts/test-github-environment-main-policy.py" in ci_runner, "CI must run the environment policy behavioral test.") +require( + "exactly one custom branch deployment policy whose type is `branch` and whose name is exactly `main`" in normalized_readme, + "README must require exact-main custom deployment policies.", +) + require("docker network create" not in sql, "SQL bootstrap must not manage Docker networks.") require("${POSTGRES_ADMIN_URL:?" in readme, "README bootstrap command must fail fast for POSTGRES_ADMIN_URL.") require("PostgreSQL superuser connection URI" in normalized_readme, "README must define POSTGRES_ADMIN_URL as a PostgreSQL superuser connection URI.") @@ -149,8 +211,8 @@ require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy work require(remote_deploy_path.stat().st_mode & 0o111, "Remote deploy script must be executable.") for required in ( 'cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh"', - 'scp "${scp_opts[@]}" "${bundle_root}/scripts/deploy-postgres-stack.sh"', - 'printf -v remote_script_q %q "${REMOTE_DIR}/scripts/deploy-postgres-stack.sh"', + 'remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + 'remote_script="${remote_bundle}/scripts/deploy-postgres-stack.sh"', ): require(required in manual_deploy_workflow, f"Manual deploy workflow must bundle and invoke the remote deploy script: {required}") require("<<'EOF'" not in manual_deploy_workflow, "Manual deploy workflow must not embed the oversized remote deployment heredoc.") @@ -233,8 +295,17 @@ for database, roles in ( ): require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") for role in roles: - require(re.search(rf"^hostssl\s+{database}\s+{role}\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must allow TLS access to {database} for {role}.") + address = "127.0.0.1/32" if role == "keycloak_brio_staging_backup" else "all" + require(re.search(rf"^hostssl\s+{database}\s+{role}\s+{re.escape(address)}\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must allow TLS access to {database} for {role} from {address}.") require(re.search(rf"^host\s+all\s+{role}\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject {role} from every non-target database.") +for allow_record, reject_record in ( + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_app", "all", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_app", "all", "reject")), + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_backup", "127.0.0.1/32", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_backup", "all", "reject")), +): + require(hba_records.index(allow_record) < hba_records.index(reject_record), "Brio identity HBA allows must precede their target-wide role rejection.") +require("MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v3" in canary_env, "Canary must use the fresh immutable Brio HBA v3 object.") +require("MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v3" in production_env, "Production must use the fresh immutable Brio HBA v3 object.") +require("runtrace_hba_v2" not in canary_env + production_env, "Active environments must never drift an already deployed HBA v2 object.") require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose Brio's certificate-matching database alias.") require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach Brio's isolated database network.") require("ensure_internal_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate Brio's internal encrypted database network.") @@ -351,14 +422,19 @@ require("wait_for_service_convergence" in manual_deploy, "Manual deploy must wai require("docker service ps --no-trunc --filter desired-state=running" in manual_deploy, "Manual deploy must inspect running task images rather than stale replica counts.") require('wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}"' in manual_deploy, "Manual deploy must converge PostgreSQL before probing it.") require('wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}"' in manual_deploy, "Canary deploy must converge the Brio application backup task.") -require('wait_for_service_convergence "${stack_name}_keycloak_brio_staging_backup" "${brio_backup_image}"' in manual_deploy, "Production deploy must converge the Brio identity backup task.") +require("keycloak_brio_staging_backup" not in production_compose, "The Brio identity backup must never be routed through the production Swarm override.") require("Postgres did not become reachable via makepad-postgres-vif" in manual_deploy, "Manual deploy workflow must fail clearly when VIF readiness times out.") require( - not re.search(r"\S\\gexec", manual_deploy), - "Manual deploy workflow must separate every VIF provisioning \\gexec command from SQL text by whitespace.", + not re.search(r"\S\\gexec", vif_sql), + "VIF bootstrap must separate every \\gexec command from SQL text by whitespace.", ) -require("ALTER ROLE %I LOGIN PASSWORD %L" in manual_deploy, "Manual deploy workflow must always refresh the VIF role password.") -require("ALTER DATABASE %I OWNER TO %I" in manual_deploy, "Manual deploy workflow must repair VIF database ownership drift.") +require("ALTER ROLE %I LOGIN PASSWORD %L" in vif_sql, "VIF bootstrap must always refresh the VIF role password.") +require("ALTER DATABASE %I OWNER TO %I" in vif_sql, "VIF bootstrap must repair VIF database ownership drift.") +require("\\getenv vif_password VIF_PASSWORD" in vif_sql, "VIF bootstrap must read its password from the mounted-file environment only.") +require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual_deploy_workflow + remote_deploy, "VIF password must never be persisted in the deployment environment file.") +require('-v vif_password=' not in remote_deploy, "VIF password must never be placed in psql command arguments.") +for required in ("postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", "vif-db-password", "bootstrap/vif-app.sql"): + require(required in manual_deploy, f"VIF deployment is missing file-only credential control: {required}") require( sql.count("DO $$") == len(expected_instances), "SQL bootstrap must use one DO block for each expected role.", @@ -479,21 +555,6 @@ for required in ( require(required in canary_backup_service, f"Canary Brio backup service is missing: {required}") require("- db" not in canary_backup_service, "Canary Brio backup must attach only to Brio's isolated database network.") -require(" keycloak_brio_staging_backup:" in production_compose, "Production Compose must run the Brio identity backup service.") -production_backup_service = production_compose.split(" keycloak_brio_staging_backup:", 1)[1].split("\nnetworks:", 1)[0] -for required in ( - "BRIO_BACKUP_DATABASE: keycloak_brio_staging", - "PGHOST: makepad-postgres", - "PGUSER: keycloak_brio_staging_backup", - "PGSSLMODE: verify-full", - "BRIO_BACKUP_RETENTION_DAYS", - "BRIO_BACKUP_RECIPIENT_CERT", - "user: \"999:999\"", - "read_only: true", - "no-new-privileges:true", - "- db", -): - require(required in production_backup_service, f"Production Brio identity backup service is missing: {required}") require("BRIO_RESTORE_RECIPIENT_KEY" not in canary_compose + production_compose + host_compose, "Backup services must never mount the Brio recovery private key.") for required in ( "keycloak_brio_staging_backup:", @@ -525,7 +586,7 @@ for required in ( require(required in brio_backup, f"Brio encrypted backup script is missing: {required}") require("--file=" not in brio_backup, "Brio backup must stream pg_dump instead of writing a plaintext dump file.") require("BRIO_RESTORE_RECIPIENT_KEY" not in brio_backup, "Brio backup service must not receive the recovery private key.") -require("PGUSER=postgres" not in canary_backup_service + production_backup_service, "Brio backup services must never run as the PostgreSQL superuser.") +require("PGUSER=postgres" not in canary_backup_service + host_compose, "Brio backup services must never run as the PostgreSQL superuser.") require("healthcheck" in brio_backup_loop and "interval_seconds * 2" in brio_backup_loop, "Brio backup health check must enforce freshness.") for required in ( "replace-nonproduction-brio-restore-targets", @@ -548,7 +609,7 @@ for required in ( require("run-brio-encrypted-backup.sh" in brio_backup_test, "Brio backup contract test must execute the real backup script.") require("verify-brio-encrypted-restore.sh" in brio_restore_test, "Brio restore contract test must execute the real restore verifier.") for required in ("test-brio-bootstrap.sh", "test-brio-encrypted-backup.sh", "test-brio-encrypted-restore.sh"): - require(required in ci_workflow, f"CI must run the Brio PostgreSQL contract: {required}") + require(required in ci_runner, f"CI must run the Brio PostgreSQL contract: {required}") for required in ( "independently administered off-host storage", "successful recorded restore of both databases remain external release gates", @@ -590,13 +651,112 @@ for forbidden in ('${HOME}/.ssh', "$HOME/.ssh", "~/.ssh", "add-ssh-host-key-acti for workflow_name, workflow_text in ( ("CI", ci_workflow), ("manual deploy", manual_deploy_workflow), + ("identity DB-VM deploy", identity_workflow), + ("identity database release", release_workflow), + ("Keycloak cohort restore", cohort_workflow), + ("PR CI finalizer", pr_finalizer_workflow), ): - checkout_count = workflow_text.count("uses: actions/checkout@v5") + checkout_ref = "uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5" + checkout_count = workflow_text.count(checkout_ref) require(checkout_count > 0, f"{workflow_name} workflow must check out the repository.") require( workflow_text.count("persist-credentials: false") == checkout_count, f"Every self-hosted checkout in the {workflow_name} workflow must disable persisted Git credentials.", ) + +# The repository is public. Every Actions job must therefore use one of the +# explicitly selected self-hosted runner groups; adding a hosted runner (or a +# string-form, ungrouped self-hosted label) is a release-policy violation. +workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorted( + (repo_root / ".github/workflows").glob("*.yaml") +) +require(workflow_paths, "At least one GitHub Actions workflow must exist.") +for workflow_path in workflow_paths: + workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") + runs_on_count = len(re.findall(r"(?m)^ runs-on:\s*$", workflow_text)) + grouped_count = len(re.findall(r"(?m)^ runs-on:\s*\n group: [^\n]+\n labels: \[[^\n]*self-hosted[^\n]*\]$", workflow_text)) + require(runs_on_count > 0, f"Workflow {workflow_path.name} must define at least one job runner.") + require( + runs_on_count == grouped_count, + f"Every job in {workflow_path.name} must use a selected group and explicit self-hosted labels.", + ) + require( + not re.search(r"(?i)(ubuntu|windows|macos)-(latest|[0-9]+)", workflow_text), + f"Workflow {workflow_path.name} must not use a GitHub-hosted runner image.", + ) + +# Credential material is canonical in Proton Pass and may be mirrored only to +# the protected environment that consumes it. Validate each complete table row +# so a field cannot silently drift into a different environment or item. +credential_inventory = { + "Hetzner Database Server makepad": ( + ("canary", "production", "staging-brio-identity-db", "keycloak-cohort-restore"), + ( + "DEPLOY_SSH_HOST", "DEPLOY_SSH_PORT", "DEPLOY_SSH_USER", + "DEPLOY_SSH_PRIVATE_KEY", "DEPLOY_SSH_KNOWN_HOSTS", + "BRIO_IDENTITY_DB_DEPLOY_SSH_HOST", "BRIO_IDENTITY_DB_DEPLOY_SSH_PORT", + "BRIO_IDENTITY_DB_DEPLOY_SSH_USER", "BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY", + "BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS", "KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY", + "KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS", "KEYCLOAK_COHORT_DB_SSH_HOST", + "KEYCLOAK_COHORT_DB_SSH_PORT", "KEYCLOAK_COHORT_DB_SSH_USER", + ), + ), + "Brio Staging - PostgreSQL": ( + ("canary", "staging-brio-identity-db"), + ( + "POSTGRES_CANARY_SUPERUSER_PASSWORD", "BRIO_STAGING_DB_PASSWORD", + "BRIO_STAGING_BACKUP_DB_PASSWORD", "KEYCLOAK_BRIO_STAGING_DB_PASSWORD", + "KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD", + ), + ), + "Brio Staging - PKI and Backup Keys": ( + ("canary", "staging-brio-identity-db"), + ( + "POSTGRES_CA_PEM", "POSTGRES_SERVER_CERT_PEM", "POSTGRES_SERVER_KEY_PEM", + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + ), + ), + "PostgreSQL · Brio identity release orchestrator": ( + ("release-brio-identity-db",), + ("KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN",), + ), + "PostgreSQL · Keycloak cohort source reader": ( + ("keycloak-cohort-restore",), + ("KEYCLOAK_COHORT_SOURCE_TOKEN",), + ), + "Makepad Docker Hardened Images": ( + ("keycloak-cohort-restore",), + ("DOCKERHUB_USERNAME", "DOCKERHUB_PRO_PAT", "DHI_REGISTRY_USERNAME", "DHI_REGISTRY_PASSWORD"), + ), + "PostgreSQL · PR Checks App": ( + ("postgres-ci-attestation",), + ( + "POSTGRES_PR_CHECK_APP_ID", "POSTGRES_PR_CHECK_APP_PRIVATE_KEY", + ), + ), + "PostgreSQL · JIT Launcher App": ( + ("postgres-ci-attestation",), + ("POSTGRES_CI_LAUNCHER_APP_SENDER_ID",), + ), + "PostgreSQL · JIT hypervisor attestation": ( + ("postgres-ci-attestation",), + ("POSTGRES_CI_ATTESTATION_PUBLIC_KEY", "POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256"), + ), +} +readme_lines = readme.splitlines() +for item, (environments, fields) in credential_inventory.items(): + candidate_rows = [line for line in readme_lines if line.startswith("|") and f"`{item}`" in line] + require(candidate_rows, f"README credential inventory is missing canonical Proton item {item}.") + require( + any(all(value in row for value in (*environments, *fields)) for row in candidate_rows), + f"README must map every field for {item} to its exact protected GitHub environment.", + ) +require("pass-cli item view --item-title '' --field ''" in readme, "README must document stdin-only pass-cli credential synchronization.") +require("| gh secret set '' --env '' --repo 'Makepad-fr/postgres'" in normalized_readme, "README must mirror workflow secrets only into protected GitHub environments.") +for workflow_path in workflow_paths: + workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") + for field in set(re.findall(r"(?:secrets|vars)\.([A-Z][A-Z0-9_]*)", workflow_text)): + require(field in readme, f"Workflow field {field} in {workflow_path.name} is absent from the credential inventory.") for policy in ( "hostnossl brio_staging", "hostnossl keycloak_brio_staging", @@ -604,4 +764,341 @@ for policy in ( "hostssl keycloak_brio_staging", ): require(policy in runtrace_hba, f"PostgreSQL HBA policy is missing {policy}.") + +for path in ( + canary_deploy_path, + identity_deploy_path, + tmp_cleaner_path, + repo_root / "scripts/test-brio-deploy-guards.sh", + deployment_failure_test_path, + deployment_failure_fixture_path, + repo_root / "scripts/capture-keycloak-cohort-backups.sh", + repo_root / "scripts/restore-keycloak-cohort-backups.sh", + repo_root / "scripts/test-keycloak-cohort-evidence.sh", + repo_root / "scripts/test-keycloak-cohort-hardening.sh", + cohort_dispatch_path, + cohort_host_installer_path, + cohort_cleaner_path, + cohort_cleaner_installer_path, + repo_root / "scripts/run-postgres-ci-jit-vm.sh", + pr_jit_result_validator_path, + repo_root / "scripts/test-postgres-ci-jit-result.sh", + repo_root / "scripts/run-postgres-ci-queue-controller.sh", + repo_root / "scripts/configure-postgres-ci-runner-group.sh", +): + require(os.access(path, os.X_OK), f"Brio deployment script must be executable: {path}") + +for required in ( + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + "BRIO_STAGING_BACKUP_DB_PASSWORD", + "BRIO_STAGING_DB_PASSWORD", + "POSTGRES_CANARY_SUPERUSER_PASSWORD", + "POSTGRES_CA_PEM", + "POSTGRES_SERVER_CERT_PEM", + "POSTGRES_SERVER_KEY_PEM", + "Materialize job-scoped Brio canary inputs", + "postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "install -d -m 0700", + "chmod 0600", + "bootstrap/brio-staging-app.sql", + "deploy-brio-canary-postgres.sh", + "Remove remote job-scoped deployment material", +): + require(required in manual_deploy_workflow, f"Canary workflow is missing secure Brio input/deploy control: {required}") +require("makepad-postgres-deploy" in manual_deploy_workflow, "Manual deployment must use the repository-scoped deploy runner label.") +require("group: Postgres Deploy" in manual_deploy_workflow, "Manual deployment must use the protected Postgres Deploy runner group.") +require('[[ "${GITHUB_REF}" == "refs/heads/main" ]]' in manual_deploy_workflow, "Manual deployment must refuse unreviewed refs.") +require("pull_request_target:" in ci_workflow, "PR CI must execute protected-base workflow code.") +require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks.") +require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact candidate head.") +require("group: org/Postgres PR Ephemeral" in ci_workflow, "PR CI must use the selected-workflow ephemeral runner group.") +require("group: org/Postgres Main CI" in ci_workflow, "Main CI must use its protected selected-workflow runner group.") +require("repository_dispatch:" in pr_finalizer_workflow and "types: [postgres-pr-ci-attestation]" in pr_finalizer_workflow and "environment: postgres-ci-attestation" in pr_finalizer_workflow, "PR result publication must accept only protected signed teardown dispatches.") +require("POSTGRES_PR_CHECK_APP_PRIVATE_KEY" in pr_finalizer_workflow and 'CHECK_NAMES = ["postgres-ci"]' in pr_check_publisher, "The required PR result must be published by its dedicated Checks App.") +for required in ( + "makepad.postgres.ci-attestation.v1", + "verifySignature", + "registration_absent", + "runnerLookupStatus !== 404", + "POSTGRES_CI_ATTESTATION_PUBLIC_KEY", + "POSTGRES_CI_LAUNCHER_APP_SENDER_ID", +): + require(required in pr_check_publisher + pr_finalizer_workflow, f"Signed JIT teardown finalization is missing: {required}") +for required in ( + "generate-jitconfig", + "--jitconfig", + "qemu-img convert", + "virsh undefine", + "nft delete table", + "registration_absent", + "dispatch-ci-attestation.mjs", + "makepad-postgres-pr-ephemeral", + "resources.json", + "--reconcile", + "POSTGRES_CI_RESULT_POLL_ATTEMPTS", + "verify-postgres-ci-jit-result.py", +): + require(required in pr_jit_launcher, f"Disposable PR VM launcher is missing: {required}") +require('base.get("sha") != workflow_sha' in pr_jit_result_validator, "The final JIT attestation verifier must bind the PR base SHA to the workflow SHA.") +require("test-postgres-ci-jit-result.sh" in ci_runner, "CI must run the executable final JIT base-SHA regression test.") +for required in ( + 'job.name === "policy-and-integration"', + "state.jobs[String(job.jobID)]", + "await atomicState(stateFile, state)", + "await runLauncher", + "await reconcileIncompleteJobs", + "launchID", + 'organization_self_hosted_runners: "write"', +): + require(required in pr_queue_controller, f"Supervised JIT queue controller is missing: {required}") +require('"allows_public_repositories": True' in pr_runner_policy, "The selected-workflow runner policy must explicitly support the public PostgreSQL repository.") +require("makepad-postgres-ci-attestor" in pr_runner_policy and "makepad-postgres-pr-ephemeral" in pr_runner_policy, "Runner policy must separate the persistent attestor from the JIT-only label.") +require('association.head?.repo?.id !== run.repository?.id' in pr_check_publisher, "The PR Checks publisher must independently reject fork runs.") +require('association.base?.sha !== attestation.run.workflow_sha' in pr_check_publisher, "The PR Checks publisher must bind the exact PR base SHA.") +require('association.base?.sha !== run.head_sha' in pr_queue_controller, "The queue controller must bind the exact PR base SHA before launch.") +require('"${RUNNER_TEMP}"/postgres-deploy-*|"${RUNNER_TEMP}"/postgres-brio-canary-runtime-*|"${RUNNER_TEMP}"/postgres-brio-vif-runtime-*' in manual_deploy_workflow, "Cleanup must allow only the exact job-scoped deployment directory prefixes.") +require("for cleanup_target in" in manual_deploy_workflow, "Manual workflow cleanup must use a narrowly named cleanup target variable.") +require("group: postgres-shared-swarm-target" in manual_deploy_workflow, "Canary and production must share one target-wide Swarm concurrency group.") +require("postgres-swarm-${{ inputs.environment }}" not in manual_deploy_workflow, "Swarm concurrency must not split by environment on the shared target.") +require("${REMOTE_DIR}/stack.yml" not in manual_deploy_workflow + remote_deploy, "Deployment must never write the shared remote stack.yml path.") +require('stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml"' in remote_deploy, "Generated stack configuration must stay inside the unique run bundle.") + +for required in ( + "postgres-server-cert.pem", + "postgres-server-key.pem", + "PostgreSQL TLS certificate and private key do not match", + "-checkhost makepad-postgres-brio-staging", + "prevalidate_swarm_config", + "content-sha256", + "prevalidate_swarm_secret", + "bootstrap/brio-staging-app.sql", + "\\getenv brio_staging_app_password", + "PGSSLMODE=verify-full", + "Plaintext access to brio_staging was unexpectedly accepted", + "Brio application role was unexpectedly accepted by a non-target database", + "show default_transaction_read_only", + "makepad-postgres-brio-staging", + "last-success.json", + "sha256sum --check --status", + "openssl cms -cmsout", +): + require(required in canary_deploy, f"Canary deployment orchestrator is missing: {required}") +require('-e PGPASSWORD=' not in canary_deploy, "Canary deployment must not put database passwords in Docker command arguments.") +shared_network_validation = canary_deploy.find('prevalidate_network "${db_network}" false') +incomplete_recovery = canary_deploy.find("recover_incomplete_journals", shared_network_validation) +database_journal = canary_deploy.find('run_db_transaction prepare "${journal_stage}"', incomplete_recovery) +require(-1 not in (shared_network_validation, incomplete_recovery, database_journal) and shared_network_validation < incomplete_recovery < database_journal, "The validated shared database network must precede recovery and first-deployment journal capture.") +for required in ( + "assert_no_symlink_components", + "candidate-stack.yml", + "docker stack config", + 'tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar"', + "rollback_canary", + "prior-service-spec-hashes.list", + "mv -fT \"$super_stage\"", + "rollback_armed=0", +): + require(required in canary_deploy, f"Canary atomic deployment contract is missing: {required}") + +for required in ( + "environment: staging-brio-identity-db", + 'refs/heads/main', + "restart-standalone-postgres-for-brio-staging", + "backup_restore_confirmed", + "BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY", + "BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS", + "BRIO_IDENTITY_DB_DEPLOY_SSH_HOST", + "BRIO_IDENTITY_DB_DEPLOY_SSH_USER", + "KEYCLOAK_BRIO_STAGING_DB_PASSWORD", + "KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD", + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + "BRIO_IDENTITY_DB_HOSTNAME", + "BRIO_KEYCLOAK_DB_SOURCE_CIDR", + "deploy-brio-identity-db-host.sh", + "Remove remote job-scoped identity secrets", + "postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "ensure-brio-tmp-cleaner.sh", + "brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "brio-db-deployment-evidence.json", + "makepad.brio-db-deployment-evidence.v1", + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", + "makepad-postgres-deploy", + "group: Postgres Deploy", +): + require(required in identity_workflow, f"Standalone identity DB workflow is missing: {required}") +for required in ( + "environment: release-brio-identity-db", + "KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN", + "verify-brio-database.yml/dispatches", + "verify-brio-release-evidence.py postgres-run", + "verify-brio-release-evidence.py postgres-evidence", + "verify-brio-release-evidence.py verifier-run", + "verify-brio-release-evidence.py attestation", + "brio-db-path-attestation", + "fetch_complete_listing", + '--config -', + 'release_token=${RELEASE_ORCHESTRATOR_TOKEN}', + 'unset RELEASE_ORCHESTRATOR_TOKEN', +): + require(required in release_workflow + release_evidence_validator, f"Protected two-phase database release orchestrator is missing: {required}") +for forbidden in ("brio-db-path-attestation.json\" <<", "actions/upload-artifact"): + require(forbidden not in release_workflow, "The release orchestrator must never synthesize or republish Keycloak attestation evidence.") +for required in ( + "name: Verify Keycloak Cohort Restore Compatibility", + "workflow_dispatch:", + "keycloak_release_sha:", + "environment: keycloak-cohort-restore", + "KEYCLOAK_COHORT_SOURCE_TOKEN", + "repos/Makepad-fr/keycloak/git/ref/heads/main", + "keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "keycloak-cohort-restore-evidence.json", + "makepad.keycloak-cohort-restore-evidence.v2", + "restored-databases-compatible", + "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "KEYCLOAK_UPSTREAM_VERSION=26.7.3", + "restore-keycloak-cohort-backups.sh", + "verify-keycloak-cohort-evidence.py", +): + require(required in cohort_workflow + cohort_evidence_validator, f"Six-database Keycloak cohort producer is missing: {required}") +require("vars." not in cohort_workflow, "The cohort producer must not accept a mutable repository variable as release evidence.") +for slug, database in ( + ("betacrew", "keycloak_betacrew"), + ("catwlk", "keycloak_catwlk"), + ("makepad", "keycloak_makepad"), + ("runtrace", "keycloak_runtrace"), + ("vestiaire", "keycloak_vestiaire"), + ("vif", "keycloak_vif"), +): + require(slug in cohort_evidence_validator and database in cohort_capture + cohort_restore, f"Cohort contract is missing {slug}/{database}.") +for required in ("pg_dump", "--no-owner", "--no-privileges", "pg_restore --list", "postgres-postgres-1"): + require(required in cohort_capture, f"Live cohort capture is missing: {required}") +for required in ( + "pg_restore", "start-dev", "/health/ready", "realm_smtp_config", + "authentication_execution", "role_attribute", "composite_role", "client_scope_role_mapping", "protocol_mapper_config", + "identity_provider_config", "component_config", "required_action_provider", + "configuration_regression", "catwlk-custom-provider", "POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password", +): + require(required in cohort_restore, f"Disposable cohort restore/startup verifier is missing: {required}") +require("scp " not in cohort_workflow and "remote_script=" not in cohort_workflow, "Cohort workflow must not execute checked-out code on the database host.") +for required in ( + "SSH_ORIGINAL_COMMAND", "sha256sum", 'sha256sum "${cleaner}"', "systemctl is-enabled", "systemctl is-active", + "--property=Result", "--property=ExecMainStatus", + "probe)", "capture)", "fetch)", "cleanup)", +): + require(required in cohort_dispatch, f"Cohort forced-command dispatcher is missing: {required}") +require('restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch"' in cohort_host_installer, "Capture key must be bound to the exact forced command.") +for required in ("makepad.cleanup.contract", "makepad.cleanup.expires-epoch", "docker container ls -aq", "docker network ls -q"): + require(required in cohort_cleaner, f"Cohort resource cleaner is missing: {required}") +require("install-keycloak-cohort-cleaner.sh" in cohort_host_installer and "makepad-keycloak-cohort-cleaner.timer" in cohort_cleaner_installer, "Capture host must install the persistent cohort resource cleaner.") +require("makepad.keycloak-config-fingerprint.v2" in cohort_evidence_validator, "Cohort evidence must bind the v2 fingerprint schema.") +require("test-keycloak-cohort-hardening.sh" in ci_runner, "CI must run the cohort hardening contract test.") +require("POSTGRES_HOST_COMPOSE_PROJECT" not in identity_workflow + readme, "The standalone Compose project must be fixed in code, not selected by a workflow variable.") + +for required in ( + "Swarm.LocalNodeState", + '[[ "${swarm_state}" == "inactive" ]]', + "/srv/makepad/postgres", + "compose_project=postgres", + "expected_container_name=postgres-postgres-1", + "com.docker.compose.project", + "com.docker.compose.service", + "com.docker.compose.oneoff", + "bind|/var/lib/makepad/postgres|true", + '"${network_mode}" == "host"', + "keycloak-db-source-cidr", + "-checkip", + "127.0.0.1/32", + "65.21.134.125", + "88.99.209.165/32", + "Failed to render ordered, exact source-restricted Keycloak Brio HBA rules", + "--force-recreate", + "--project-name", + "bootstrap/keycloak-brio-staging.sql", + "\\getenv keycloak_brio_staging_app_password", + "PGHOSTADDR=127.0.0.1", + "PGSSLMODE=verify-full", + "Plaintext Keycloak Brio database access was unexpectedly accepted", + "Keycloak Brio role was unexpectedly accepted by a non-target database", + "show default_transaction_read_only", + "last-success.json", + "sha256sum --check --status", + "openssl cms -cmsout", + "restore_snapshot", + "rollback_deployment", + "rollback_armed=1", + "trap handle_exit EXIT", + "trap 'exit 129' HUP", + "trap 'exit 130' INT", + "trap 'exit 143' TERM", + 'tar --numeric-owner -cpf "$stage/rollback/managed.tar"', + 'identity-backups.tar', + 'identity-backup-absent', + "up -d --remove-orphans --wait --force-recreate", + "preserve_recovery_evidence", + "postgres-recovery/brio-identity", + "RECOVERY_REQUIRED", + "recovery_id=${identifier}", +): + require(required in identity_deploy, f"Standalone identity DB orchestrator is missing: {required}") +require("docker stack" not in identity_deploy and "docker swarm" not in identity_deploy, "Standalone identity DB deployment must not invoke Swarm deployment commands.") +require('"${key_uid}:${key_gid}:${key_mode}" == "70:70:400"' in identity_deploy, "Standalone DB-VM preflight must preserve the exact live server-key owner/group/mode contract.") +require('-e PGPASSWORD=' not in identity_deploy, "Identity DB deployment must not put database passwords in Docker command arguments.") +snapshot_index = identity_deploy.find('tar --numeric-owner -cpf "$stage/rollback/managed.tar"') +arm_index = identity_deploy.find("rollback_armed=1", snapshot_index) +first_install_index = identity_deploy.find('install_host_path "${candidate_compose}"', arm_index) +fresh_backup_index = identity_deploy.find('[[ "${backup_verified}" == "1" ]]', first_install_index) +disarm_index = identity_deploy.find("rollback_armed=0", fresh_backup_index) +require(-1 not in (snapshot_index, arm_index, first_install_index, fresh_backup_index, disarm_index), "Standalone rollback boundary markers are incomplete.") +require(snapshot_index < arm_index < first_install_index < fresh_backup_index < disarm_index, "Rollback must arm after snapshot and disarm only after probes and fresh backup verification.") +for required in ( + "MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH=", + "MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH=", + "MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH=", + "MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH=", +): + require(required in production_env, f"Production host environment is missing explicit standalone input: {required}") +require("test-brio-deploy-guards.sh" in ci_runner, "CI must run the deployment guard contract test.") +require("test-brio-deployment-failures.sh" in ci_runner, "CI must run executable Brio deployment failure-injection tests.") +for required in ( + "after-managed-file-promotion", + "term-after-managed-file-promotion", + "after-stack-deploy", + "rollback-restore", + "rollback-recreate", + "RECOVERY_REQUIRED", + ".State.Restarting", + "unexpected cleaner command", + "cleaner-running-output", + "symlink component", +): + require(required in deployment_failure_fixture, f"Failure-injection fixture is missing behavioral case: {required}") +require("PGHOSTADDR: 127.0.0.1" in host_compose, "Standalone identity backup must use a deterministic local transport address while verifying the configured certificate host.") +for required in ( + "makepad-postgres-brio-tmp-cleaner", + "--restart unless-stopped", + "--read-only", + "--cap-drop ALL", + "--security-opt no-new-privileges", + "type=bind,src=/tmp,dst=/host-tmp", + "-name 'postgres-brio-*'", + "-mmin +180", + "sleep 900", + "RECOVERY_REQUIRED", + "observed_command", + "verify_running", + ".State.Restarting", +): + require(required in tmp_cleaner, f"Host TTL cleaner is missing its restricted contract: {required}") +for workflow in (manual_deploy_workflow, identity_workflow): + require("ensure-brio-tmp-cleaner.sh" in workflow, "Every Brio deployment target must install the host-enforced TTL cleaner.") + require(workflow.find("ensure-brio-tmp-cleaner.sh") < workflow.find('scp "${scp_opts[@]}" "${runtime_dir}'), "TTL cleaner must be installed before job secrets are transferred.") +for required in ( + "Verify Brio Identity Database Path", + "Verify Brio DB path for PostgreSQL run ", + "brio-db-path-ok", + "65.21.134.125", + "88.99.209.165", + "no Keycloak database credential is granted to the PostgreSQL runner", +): + require(required in normalized_readme, f"README is missing the Keycloak-origin database release gate: {required}") PY From fe88df9e2559efbb4dcc913bb395c56a13b86bad Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 14:50:58 +0200 Subject: [PATCH 13/20] revert(postgres): remove unrelated database policies --- README.md | 12 ++------ config/runtrace-pg_hba.conf | 23 ++------------- scripts/validate-postgres-config.sh | 44 ----------------------------- 3 files changed, 5 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index d8086e3..5a6c113 100644 --- a/README.md +++ b/README.md @@ -137,14 +137,8 @@ to `127.0.0.1/32`; only the Keycloak application role receives the separately rendered egress `/32` rule. The host deployment preserves the existing host-network endpoint used by -Keycloak while requiring TLS and SCRAM for `runtrace`, `keycloak_runtrace`, -`fresko_production`, `betacrew`, and `keycloak_betacrew`. Fresko's runtime, -schema-owner, and importer roles and the BetaCrew application role are limited -to the private WireGuard source `10.80.0.1/32`; the BetaCrew Keycloak role is -limited to `88.99.209.165/32`, with local maintenance access limited to -`127.0.0.1/32`. The committed HBA policy rejects every other source or plaintext -connection for those databases before reaching the shared fallback. Other -databases keep their existing SCRAM transport policy. +Keycloak while requiring TLS and SCRAM for `runtrace` and +`keycloak_runtrace`. Other databases keep their existing SCRAM transport policy. Required environment secrets: @@ -512,7 +506,7 @@ docker config create makepad_postgres_canary_tls_cert_v2 /secure/path/canary-ser docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-server.key ``` -The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy preserves the source-restricted Fresko and BetaCrew rules described above, rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging`, and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. +The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging` and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. The Brio HBA policy uses fresh immutable `makepad_postgres_canary_runtrace_hba_v3` and `makepad_postgres_runtrace_hba_v3` object names; deployed `v2` objects are historical and must never be replaced or relabelled in place. diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index a6d5750..a1d759e 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -3,29 +3,10 @@ local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject -hostssl runtrace all all scram-sha-256 -hostssl keycloak_runtrace all all scram-sha-256 -# Fresko can reach PostgreSQL only over the Makepad private WireGuard route. -# The application, migration, and importer roles have separate passwords; any -# other source or TLS mode is explicitly rejected before the shared fallback. -hostnossl fresko_production all all reject -hostssl fresko_production fresko_runtime 10.80.0.1/32 scram-sha-256 -hostssl fresko_production fresko_schema_owner 10.80.0.1/32 scram-sha-256 -hostssl fresko_production fresko_importer 10.80.0.1/32 scram-sha-256 -hostssl fresko_production all all reject -# BetaCrew app and identity traffic require TLS and exact source roles. -hostnossl betacrew all all reject -hostnossl keycloak_betacrew all all reject -hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256 -hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256 -hostssl betacrew postgres 127.0.0.1/32 scram-sha-256 -hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256 -hostssl betacrew all all reject -hostssl keycloak_betacrew all all reject -# Brio staging application and identity traffic require TLS and cannot access -# any other database through the shared fallback. hostnossl brio_staging all all reject hostnossl keycloak_brio_staging all all reject +hostssl runtrace all all scram-sha-256 +hostssl keycloak_runtrace all all scram-sha-256 hostssl brio_staging brio_staging_app all scram-sha-256 hostssl brio_staging brio_staging_backup all scram-sha-256 hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index bebc427..deecf94 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -245,50 +245,6 @@ for required in ( for database in ("runtrace", "keycloak_runtrace"): require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") require(re.search(rf"^hostssl\s+{database}\s+all\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must require TLS and SCRAM for {database}.") -hba_records = [ - tuple(line.split()) - for line in runtrace_hba.splitlines() - if line.strip() and not line.lstrip().startswith("#") -] -fresko_betacrew_records = [ - record - for record in hba_records - if len(record) >= 2 and record[1] in {"fresko_production", "betacrew", "keycloak_betacrew"} -] -require( - fresko_betacrew_records - == [ - ("hostnossl", "fresko_production", "all", "all", "reject"), - ("hostssl", "fresko_production", "fresko_runtime", "10.80.0.1/32", "scram-sha-256"), - ("hostssl", "fresko_production", "fresko_schema_owner", "10.80.0.1/32", "scram-sha-256"), - ("hostssl", "fresko_production", "fresko_importer", "10.80.0.1/32", "scram-sha-256"), - ("hostssl", "fresko_production", "all", "all", "reject"), - ("hostnossl", "betacrew", "all", "all", "reject"), - ("hostnossl", "keycloak_betacrew", "all", "all", "reject"), - ("hostssl", "betacrew", "betacrew_app", "10.80.0.1/32", "scram-sha-256"), - ("hostssl", "keycloak_betacrew", "keycloak_betacrew_app", "88.99.209.165/32", "scram-sha-256"), - ("hostssl", "betacrew", "postgres", "127.0.0.1/32", "scram-sha-256"), - ("hostssl", "keycloak_betacrew", "postgres", "127.0.0.1/32", "scram-sha-256"), - ("hostssl", "betacrew", "all", "all", "reject"), - ("hostssl", "keycloak_betacrew", "all", "all", "reject"), - ], - "HBA must preserve the exact live Fresko and BetaCrew TLS, source, role, and rejection policy.", -) -for required in ( - "`fresko_production`", - "`betacrew`", - "`keycloak_betacrew`", - "`10.80.0.1/32`", - "`88.99.209.165/32`", - "`127.0.0.1/32`", -): - require(required in readme, f"README must document the preserved Fresko/BetaCrew HBA policy: {required}") -shared_fallback = ("host", "all", "all", "all", "scram-sha-256") -require(shared_fallback in hba_records, "HBA must retain the shared SCRAM fallback.") -require( - max(hba_records.index(record) for record in fresko_betacrew_records) < hba_records.index(shared_fallback), - "Every Fresko and BetaCrew restriction must precede the shared HBA fallback.", -) for database, roles in ( ("brio_staging", ("brio_staging_app", "brio_staging_backup")), ("keycloak_brio_staging", ("keycloak_brio_staging_app", "keycloak_brio_staging_backup")), From 6e925a4e345e40c9d411763cc181b50486ef4c47 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 14:55:44 +0200 Subject: [PATCH 14/20] fix(keycloak): verify the existing five-realm cohort --- .github/workflows/verify-keycloak-cohort-restores.yml | 10 +++++----- README.md | 10 +++++----- scripts/capture-keycloak-cohort-backups.sh | 3 +-- scripts/clean-keycloak-cohort-resources.sh | 4 ++-- scripts/keycloak-cohort-capture-dispatch.sh | 2 +- scripts/restore-keycloak-cohort-backups.sh | 7 +++---- scripts/test-brio-deployment-contracts.sh | 2 +- scripts/test-keycloak-cohort-evidence.sh | 1 - scripts/validate-postgres-config.sh | 3 +-- scripts/verify-keycloak-cohort-evidence.py | 5 ++--- 10 files changed, 21 insertions(+), 26 deletions(-) diff --git a/.github/workflows/verify-keycloak-cohort-restores.yml b/.github/workflows/verify-keycloak-cohort-restores.yml index 1881a1a..4bb0e23 100644 --- a/.github/workflows/verify-keycloak-cohort-restores.yml +++ b/.github/workflows/verify-keycloak-cohort-restores.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: keycloak_release_sha: - description: Exact protected Keycloak main SHA whose pinned 26.7.3 runtime must start all six restored databases + description: Exact protected Keycloak main SHA whose pinned 26.7.3 runtime must start all five restored databases required: true type: string @@ -17,7 +17,7 @@ concurrency: jobs: verify: - name: restore-six-databases-and-start-keycloak + name: restore-five-databases-and-start-keycloak runs-on: group: Postgres Release labels: [self-hosted, linux, x64, makepad, makepad-postgres-release] @@ -124,7 +124,7 @@ jobs: # Run identity and digest are intentionally client-expanded forced-command arguments. # shellcheck disable=SC2029 ssh "${ssh_opts[@]}" "${target}" "capture ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${helper_digest} ${cleaner_digest}" - for database in keycloak_betacrew keycloak_catwlk keycloak_makepad keycloak_runtrace keycloak_vestiaire keycloak_vif; do + for database in keycloak_catwlk keycloak_makepad keycloak_runtrace keycloak_vestiaire keycloak_vif; do partial="${job_root}/backups/.${database}.dump.partial" # The forced-command fetch tuple is intentionally client-expanded. # shellcheck disable=SC2029 @@ -133,7 +133,7 @@ jobs: [[ -s "${partial}" && ! -L "${partial}" ]] mv -T "${partial}" "${job_root}/backups/${database}.dump" done - [[ $(find "${job_root}/backups" -mindepth 1 -maxdepth 1 -type f -name '*.dump' | wc -l) -eq 6 ]] + [[ $(find "${job_root}/backups" -mindepth 1 -maxdepth 1 -type f -name '*.dump' | wc -l) -eq 5 ]] - name: Build exact checked-out Catwlk provider runtime shell: bash @@ -214,7 +214,7 @@ jobs: --head-sha "${GITHUB_SHA}" --keycloak-release-sha "${KEYCLOAK_RELEASE_SHA}" [[ $(find "${evidence_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n') == keycloak-cohort-restore-evidence.json ]] - - name: Publish immutable six-database compatibility evidence + - name: Publish immutable five-database compatibility evidence uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/README.md b/README.md index 5a6c113..b0ade45 100644 --- a/README.md +++ b/README.md @@ -400,7 +400,7 @@ the reviewed SHA-256 both before and after each full per-job copy. ## Keycloak 26.7.3 cohort restore evidence -Before the six-realm Keycloak release, dispatch protected workflow `Verify +Before the five-realm Keycloak release, dispatch protected workflow `Verify Keycloak Cohort Restore Compatibility` with the exact lowercase current Keycloak protected-main SHA. There is no mutable rollout repository variable. The workflow resolves `Makepad-fr/keycloak` main independently, checks out that @@ -409,12 +409,12 @@ exact release, and verifies its pinned runtime and upstream version `26.7.3`. The protected release runner captures fresh custom-format, no-owner, -no-privilege dumps of exactly `keycloak_betacrew`, `keycloak_catwlk`, -`keycloak_makepad`, `keycloak_runtrace`, `keycloak_vestiaire`, and +no-privilege dumps of exactly `keycloak_catwlk`, `keycloak_makepad`, +`keycloak_runtrace`, `keycloak_vestiaire`, and `keycloak_vif` from the exact healthy production Compose container. Every dump is structurally inspected. Each is then restored into a fresh internal Docker network and disposable PostgreSQL instance. Catwlk uses the custom DHI-derived -provider image built from the exact checked-out Keycloak release; the other five +provider image built from the exact checked-out Keycloak release; the other four instances use the pinned base image. Each runtime must become ready and the v2 secret-safe fingerprints for realm settings/themes/SMTP, authentication flows, roles/composites, clients/scopes/mappers, identity providers, components, and @@ -432,7 +432,7 @@ Success uploads exactly one artifact named `makepad.keycloak-cohort-restore-evidence.v2`. It binds the exact PostgreSQL workflow/run/attempt/main SHA, exact Keycloak release SHA/base image/version, the immutable locally built Catwlk image ID, fingerprint schema, and the sorted -six-instance list. Each entry contains its slug, database, fresh backup SHA-256, +five-instance list. Each entry contains its slug, database, fresh backup SHA-256, exact runtime identity, category and combined hashes, and `passed` restore, Keycloak-startup, and configuration-regression statuses. The Keycloak deployment consumer must resolve that exact completed diff --git a/scripts/capture-keycloak-cohort-backups.sh b/scripts/capture-keycloak-cohort-backups.sh index 7803899..20e6e1a 100755 --- a/scripts/capture-keycloak-cohort-backups.sh +++ b/scripts/capture-keycloak-cohort-backups.sh @@ -53,7 +53,6 @@ IFS='|' read -r compose_project compose_service observed_image state health <<<" } databases=( - keycloak_betacrew keycloak_catwlk keycloak_makepad keycloak_runtrace @@ -87,4 +86,4 @@ observed=$(find "${output_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | [[ "${observed}" == "${expected}" ]] || { echo "Cohort backup directory has an unexpected entry set." >&2; exit 1; } sha256sum "${output_dir}"/*.dump >/dev/null trap - EXIT HUP INT TERM -echo "Captured and structurally validated the exact six Keycloak databases." +echo "Captured and structurally validated the exact five Keycloak databases." diff --git a/scripts/clean-keycloak-cohort-resources.sh b/scripts/clean-keycloak-cohort-resources.sh index f4a22a9..0775639 100755 --- a/scripts/clean-keycloak-cohort-resources.sh +++ b/scripts/clean-keycloak-cohort-resources.sh @@ -13,7 +13,7 @@ remove_expired_container() { details=$(docker container inspect "${identifier}" --format '{{.Name}}|{{index .Config.Labels "makepad.cleanup.contract"}}|{{index .Config.Labels "makepad.cleanup.expires-epoch"}}') IFS='|' read -r name observed_contract expires <<<"${details}" name=${name#/} - [[ "${name}" =~ ^pg-kc-(db|app)-[1-9][0-9]*-[1-9][0-9]*-(betacrew|catwlk|makepad|runtrace|vestiaire|vif)$ \ + [[ "${name}" =~ ^pg-kc-(db|app)-[1-9][0-9]*-[1-9][0-9]*-(catwlk|makepad|runtrace|vestiaire|vif)$ \ && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { echo "Refusing malformed labeled cohort container ${identifier}." >&2 return 1 @@ -25,7 +25,7 @@ remove_expired_network() { local identifier=$1 details name observed_contract expires details=$(docker network inspect "${identifier}" --format '{{.Name}}|{{index .Labels "makepad.cleanup.contract"}}|{{index .Labels "makepad.cleanup.expires-epoch"}}') IFS='|' read -r name observed_contract expires <<<"${details}" - [[ "${name}" =~ ^pg-kc-[1-9][0-9]*-[1-9][0-9]*-(betacrew|catwlk|makepad|runtrace|vestiaire|vif)$ \ + [[ "${name}" =~ ^pg-kc-[1-9][0-9]*-[1-9][0-9]*-(catwlk|makepad|runtrace|vestiaire|vif)$ \ && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { echo "Refusing malformed labeled cohort network ${identifier}." >&2 return 1 diff --git a/scripts/keycloak-cohort-capture-dispatch.sh b/scripts/keycloak-cohort-capture-dispatch.sh index a7943aa..58b6f73 100755 --- a/scripts/keycloak-cohort-capture-dispatch.sh +++ b/scripts/keycloak-cohort-capture-dispatch.sh @@ -44,7 +44,7 @@ case "${operation}" in validate_run "${words[1]}" "${words[2]}" file=${words[3]} case "${file}" in - keycloak_betacrew.dump|keycloak_catwlk.dump|keycloak_makepad.dump|keycloak_runtrace.dump|keycloak_vestiaire.dump|keycloak_vif.dump) ;; + keycloak_catwlk.dump|keycloak_makepad.dump|keycloak_runtrace.dump|keycloak_vestiaire.dump|keycloak_vif.dump) ;; *) echo "Unsupported cohort artifact." >&2; exit 2 ;; esac [[ -f "${cohort_dir}/${file}" && ! -L "${cohort_dir}/${file}" && -s "${cohort_dir}/${file}" ]] || exit 1 diff --git a/scripts/restore-keycloak-cohort-backups.sh b/scripts/restore-keycloak-cohort-backups.sh index 9806669..5f4e69f 100755 --- a/scripts/restore-keycloak-cohort-backups.sh +++ b/scripts/restore-keycloak-cohort-backups.sh @@ -3,7 +3,7 @@ set -euo pipefail export LC_ALL=C if (($# != 4)); then - echo "usage: restore-keycloak-cohort-backups.sh " >&2 + echo "usage: restore-keycloak-cohort-backups.sh " >&2 exit 2 fi @@ -25,7 +25,6 @@ run_attempt=${GITHUB_RUN_ATTEMPT:-1} for command_name in docker git python3 sha256sum; do command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; }; done declare -A databases=( - [betacrew]=keycloak_betacrew [catwlk]=keycloak_catwlk [makepad]=keycloak_makepad [runtrace]=keycloak_runtrace @@ -43,7 +42,7 @@ declare -A category_tables=( ) expected=$(for slug in "${!databases[@]}"; do printf '%s.dump\n' "${databases[$slug]}"; done | sort) observed=$(find "${backup_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | sort) -[[ "${observed}" == "${expected}" ]] || { echo "Backup input is not the exact six-database cohort." >&2; exit 1; } +[[ "${observed}" == "${expected}" ]] || { echo "Backup input is not the exact five-database cohort." >&2; exit 1; } if find "${backup_dir}" -mindepth 1 -maxdepth 1 -type l -print -quit | grep -q .; then echo "Backup input contains a symlink." >&2; exit 1; fi release_sha=$(git -C "${keycloak_source}" rev-parse HEAD) @@ -228,4 +227,4 @@ rm -f "${records}" rmdir "${result_dir}/.runtime" chmod 0600 "${result_dir}/instances.json" trap - EXIT HUP INT TERM -echo "Restored all six Keycloak databases with exact runtimes and verified complete configuration fingerprints." +echo "Restored all five Keycloak databases with exact runtimes and verified complete configuration fingerprints." diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh index c36feb8..fe689de 100755 --- a/scripts/test-brio-deployment-contracts.sh +++ b/scripts/test-brio-deployment-contracts.sh @@ -121,7 +121,7 @@ for marker in ( "restored-databases-compatible", "keycloak_release_sha", ): - require(marker in cohort_workflow + cohort_validator, f"six-database cohort evidence contract missing: {marker}") + require(marker in cohort_workflow + cohort_validator, f"five-database cohort evidence contract missing: {marker}") require("vars." not in cohort_workflow, "cohort evidence cannot rely on a mutable repository variable") require("Ensure interrupted cohort material expires on the release host" in cohort_workflow, "cohort workflow must verify the release-host TTL guard before credentials or dumps") require(cohort_workflow.index("Ensure interrupted cohort material expires on the release host") < cohort_workflow.index("Configure isolated SSH and registry state"), "release-host TTL guard must precede credential material") diff --git a/scripts/test-keycloak-cohort-evidence.sh b/scripts/test-keycloak-cohort-evidence.sh index 9b5e5b1..850c241 100755 --- a/scripts/test-keycloak-cohort-evidence.sh +++ b/scripts/test-keycloak-cohort-evidence.sh @@ -21,7 +21,6 @@ import pathlib import sys databases = { - "betacrew": "keycloak_betacrew", "catwlk": "keycloak_catwlk", "makepad": "keycloak_makepad", "runtrace": "keycloak_runtrace", diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index deecf94..8b031e0 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -915,10 +915,9 @@ for required in ( "restore-keycloak-cohort-backups.sh", "verify-keycloak-cohort-evidence.py", ): - require(required in cohort_workflow + cohort_evidence_validator, f"Six-database Keycloak cohort producer is missing: {required}") + require(required in cohort_workflow + cohort_evidence_validator, f"Five-database Keycloak cohort producer is missing: {required}") require("vars." not in cohort_workflow, "The cohort producer must not accept a mutable repository variable as release evidence.") for slug, database in ( - ("betacrew", "keycloak_betacrew"), ("catwlk", "keycloak_catwlk"), ("makepad", "keycloak_makepad"), ("runtrace", "keycloak_runtrace"), diff --git a/scripts/verify-keycloak-cohort-evidence.py b/scripts/verify-keycloak-cohort-evidence.py index a59a20f..f855a8e 100755 --- a/scripts/verify-keycloak-cohort-evidence.py +++ b/scripts/verify-keycloak-cohort-evidence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Validate the immutable six-database Keycloak restore evidence contract.""" +"""Validate the immutable five-database Keycloak restore evidence contract.""" from __future__ import annotations @@ -22,7 +22,6 @@ "required_actions", } DATABASES = { - "betacrew": "keycloak_betacrew", "catwlk": "keycloak_catwlk", "makepad": "keycloak_makepad", "runtrace": "keycloak_runtrace", @@ -110,7 +109,7 @@ def validate( raise ValueError("cohort compatibility did not pass") instances = top["instances"] if not isinstance(instances, list) or len(instances) != len(DATABASES): - raise ValueError("instances must contain the exact six-database cohort") + raise ValueError("instances must contain the exact five-database cohort") expected_slugs = sorted(DATABASES) actual_slugs: list[str] = [] for index, raw_instance in enumerate(instances): From 0aee8e561883a4e4f5aa28d089ea8f40bff6b5a2 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:08:57 +0200 Subject: [PATCH 15/20] ci(postgres): use the existing Makepad runner --- .github/actionlint.yaml | 6 - .github/workflows/ci.yml | 43 +- .github/workflows/deploy-brio-identity-db.yml | 4 +- .github/workflows/manual-deploy.yml | 4 +- .github/workflows/pr-ci-result.yml | 62 -- .../workflows/release-brio-identity-db.yml | 4 +- .../verify-keycloak-cohort-restores.yml | 4 +- README.md | 89 +-- host/systemd/postgres-ci-queue-alert.service | 12 - .../postgres-ci-queue-controller.service | 22 - scripts/ci-base-image.py | 32 - scripts/configure-postgres-ci-runner-group.sh | 281 ------- scripts/dispatch-ci-attestation.mjs | 35 - scripts/postgres-ci-queue-controller.mjs | 203 ------ scripts/publish-pr-ci-check.mjs | 196 ----- ...econcile-github-environment-main-policy.py | 1 - scripts/run-ci.sh | 22 +- scripts/run-postgres-ci-jit-vm.sh | 685 ------------------ scripts/run-postgres-ci-queue-controller.sh | 9 - scripts/test-brio-deployment-contracts.sh | 23 +- scripts/test-postgres-ci-jit-result.sh | 77 -- scripts/test-postgres-ci-queue-controller.mjs | 126 ---- scripts/test-pr-ci-check.mjs | 152 ---- scripts/validate-postgres-config.sh | 101 +-- scripts/verify-postgres-ci-jit-result.py | 165 ----- 25 files changed, 72 insertions(+), 2286 deletions(-) delete mode 100644 .github/workflows/pr-ci-result.yml delete mode 100644 host/systemd/postgres-ci-queue-alert.service delete mode 100644 host/systemd/postgres-ci-queue-controller.service delete mode 100755 scripts/ci-base-image.py delete mode 100755 scripts/configure-postgres-ci-runner-group.sh delete mode 100644 scripts/dispatch-ci-attestation.mjs delete mode 100644 scripts/postgres-ci-queue-controller.mjs delete mode 100644 scripts/publish-pr-ci-check.mjs delete mode 100755 scripts/run-postgres-ci-jit-vm.sh delete mode 100755 scripts/run-postgres-ci-queue-controller.sh delete mode 100755 scripts/test-postgres-ci-jit-result.sh delete mode 100644 scripts/test-postgres-ci-queue-controller.mjs delete mode 100644 scripts/test-pr-ci-check.mjs delete mode 100755 scripts/verify-postgres-ci-jit-result.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 62d93da..628281c 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,9 +1,3 @@ self-hosted-runner: labels: - makepad - - makepad-postgres-ci - - makepad-postgres-deploy - - makepad-postgres-pr-ephemeral - - makepad-postgres-ci-attestor - - makepad-postgres-main-ci - - makepad-postgres-release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e383b7..9380aaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,9 @@ name: CI on: push: branches: [main] - # Workflow code comes from protected main. Same-repository PR code executes - # only on a one-job disposable runner with no environment secrets. - pull_request_target: + # Only same-repository candidate branches can reach the existing Makepad + # runner. Fork jobs are skipped before a runner is assigned. + pull_request: types: [opened, synchronize, reopened, ready_for_review] permissions: @@ -15,13 +15,10 @@ jobs: validate-pr: name: policy-and-integration if: >- - github.event_name == 'pull_request_target' && + github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.base.ref == 'main' && github.event.pull_request.draft == false - runs-on: - group: org/Postgres PR Ephemeral - labels: [self-hosted, linux, x64, makepad-postgres-pr-ephemeral] + runs-on: [self-hosted, linux, x64, makepad] timeout-minutes: 45 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 @@ -37,15 +34,23 @@ jobs: set -euo pipefail [[ "${EXPECTED_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]] [[ "$(git rev-parse HEAD)" == "${EXPECTED_HEAD_SHA}" ]] - - name: Run complete candidate suite in disposable isolation - run: ./scripts/run-ci.sh + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: 1.25.13 + cache: false + - name: Run complete candidate suite + shell: bash + run: | + set -euo pipefail + tools_dir="${RUNNER_TEMP}/postgres-pr-tools" + install -d -m 0700 "${tools_dir}" + GOBIN="${tools_dir}" go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + PATH="${tools_dir}:${PATH}" ./scripts/run-ci.sh validate-main: name: protected-main-policy-and-integration if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: - group: org/Postgres Main CI - labels: [self-hosted, linux, x64, makepad-postgres-main-ci] + runs-on: [self-hosted, linux, x64, makepad] timeout-minutes: 45 steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 @@ -59,5 +64,15 @@ jobs: [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" ]] [[ "${GITHUB_REF}" == "refs/heads/main" ]] [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: 1.25.13 + cache: false - name: Run complete protected-main suite - run: ./scripts/run-ci.sh + shell: bash + run: | + set -euo pipefail + tools_dir="${RUNNER_TEMP}/postgres-main-tools" + install -d -m 0700 "${tools_dir}" + GOBIN="${tools_dir}" go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + PATH="${tools_dir}:${PATH}" ./scripts/run-ci.sh diff --git a/.github/workflows/deploy-brio-identity-db.yml b/.github/workflows/deploy-brio-identity-db.yml index fe82228..597fb46 100644 --- a/.github/workflows/deploy-brio-identity-db.yml +++ b/.github/workflows/deploy-brio-identity-db.yml @@ -22,9 +22,7 @@ permissions: jobs: deploy: - runs-on: - group: Postgres Deploy - labels: [self-hosted, linux, x64, makepad, makepad-postgres-deploy] + runs-on: [self-hosted, linux, x64, makepad] environment: staging-brio-identity-db timeout-minutes: 60 steps: diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 52845a1..610e84b 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -17,9 +17,7 @@ concurrency: jobs: deploy: - runs-on: - group: Postgres Deploy - labels: [self-hosted, linux, x64, makepad, makepad-postgres-deploy] + runs-on: [self-hosted, linux, x64, makepad] environment: ${{ inputs.environment }} timeout-minutes: 30 permissions: diff --git a/.github/workflows/pr-ci-result.yml b/.github/workflows/pr-ci-result.yml deleted file mode 100644 index a6d4545..0000000 --- a/.github/workflows/pr-ci-result.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Verify signed PR CI teardown - -on: - repository_dispatch: - types: [postgres-pr-ci-attestation] - -permissions: - actions: read - contents: read - pull-requests: read - -concurrency: - group: postgres-pr-ci-attestor-${{ github.event.client_payload.attestation.run.id }}-${{ github.event.client_payload.attestation.run.attempt }} - cancel-in-progress: false - -jobs: - publish: - if: >- - github.repository == 'Makepad-fr/postgres' && - github.ref == 'refs/heads/main' && - github.event.action == 'postgres-pr-ci-attestation' && - github.event.sender.type == 'Bot' && - github.event.sender.id == fromJSON(vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID) - # This persistent attestor is physically separate from the disposable host - # that executes PR code. It has no deployment credentials or Docker access. - runs-on: - group: org/Postgres PR Ephemeral - labels: [self-hosted, linux, x64, makepad-postgres-ci-attestor] - environment: postgres-ci-attestation - timeout-minutes: 5 - steps: - - name: Check out protected attestor source - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - persist-credentials: false - ref: ${{ github.sha }} - fetch-depth: 1 - - name: Verify trusted attestor source and Launcher App sender - shell: bash - run: | - set -euo pipefail - [[ "${GITHUB_REPOSITORY}" == Makepad-fr/postgres ]] - [[ "${GITHUB_REF}" == refs/heads/main ]] - [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] - [[ "${{ github.event.sender.id }}" == "${{ vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID }}" ]] - - name: Verify signed teardown and publish App-bound result - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - POSTGRES_CI_LAUNCHER_APP_SENDER_ID: ${{ vars.POSTGRES_CI_LAUNCHER_APP_SENDER_ID }} - POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256: ${{ vars.POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256 }} - POSTGRES_CI_ATTESTATION_PUBLIC_KEY: ${{ vars.POSTGRES_CI_ATTESTATION_PUBLIC_KEY }} - POSTGRES_PR_CHECK_APP_ID: ${{ vars.POSTGRES_PR_CHECK_APP_ID }} - POSTGRES_PR_CHECK_APP_PRIVATE_KEY: ${{ secrets.POSTGRES_PR_CHECK_APP_PRIVATE_KEY }} - run: | - set -euo pipefail - : "${POSTGRES_PR_CHECK_APP_ID:?set protected postgres-ci-attestation App ID}" - : "${POSTGRES_PR_CHECK_APP_PRIVATE_KEY:?set protected postgres-ci-attestation App private key}" - : "${POSTGRES_CI_LAUNCHER_APP_SENDER_ID:?set the immutable Launcher App bot ID}" - : "${POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256:?set the reviewed base-image digest}" - : "${POSTGRES_CI_ATTESTATION_PUBLIC_KEY:?set the Ed25519 hypervisor public key}" - node scripts/publish-pr-ci-check.mjs diff --git a/.github/workflows/release-brio-identity-db.yml b/.github/workflows/release-brio-identity-db.yml index 47a209e..ecdcf97 100644 --- a/.github/workflows/release-brio-identity-db.yml +++ b/.github/workflows/release-brio-identity-db.yml @@ -23,9 +23,7 @@ permissions: jobs: attest: name: protected-cross-repository-attestation - runs-on: - group: Postgres Release - labels: [self-hosted, linux, x64, makepad, makepad-postgres-release] + runs-on: [self-hosted, linux, x64, makepad] environment: release-brio-identity-db timeout-minutes: 45 steps: diff --git a/.github/workflows/verify-keycloak-cohort-restores.yml b/.github/workflows/verify-keycloak-cohort-restores.yml index 4bb0e23..c161bb1 100644 --- a/.github/workflows/verify-keycloak-cohort-restores.yml +++ b/.github/workflows/verify-keycloak-cohort-restores.yml @@ -18,9 +18,7 @@ concurrency: jobs: verify: name: restore-five-databases-and-start-keycloak - runs-on: - group: Postgres Release - labels: [self-hosted, linux, x64, makepad, makepad-postgres-release] + runs-on: [self-hosted, linux, x64, makepad] environment: keycloak-cohort-restore timeout-minutes: 120 steps: diff --git a/README.md b/README.md index b0ade45..dc48313 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,13 @@ Identity Database` workflow on the standalone database VM. The production Swarm override contains no Brio identity backup service and must never be used to bootstrap or back up the Brio Keycloak database. -Both deployment workflows require the protected `Postgres Deploy` runner group -and repository-scoped `makepad-postgres-deploy` label. Protected-main CI uses -the separate `Postgres Main CI` group and `makepad-postgres-main-ci` label. -Pull-request code is never executed on either persistent host; the disposable -PR boundary is documented below. A generic Makepad runner cannot execute these -jobs. Both deployment workflows also reject every Git ref except `main`; -configure the GitHub environments with the same deployment-branch restriction -and required reviewers. +CI and deployment workflows use the existing Makepad self-hosted Linux runner +with the exact `self-hosted`, `Linux`, `X64`, and `makepad` labels. Pull-request +jobs reject forks before a runner is assigned, check out the exact candidate +head, receive no protected environment, and run with read-only repository +permissions. Deployment workflows reject every Git ref except `main`; configure +their GitHub environments with the same deployment-branch restriction and +required reviewers. Swarm deployments share one target-wide concurrency group across canary and production. Every run uploads to a unique @@ -234,9 +233,6 @@ GitHub environment variables. The exact Brio inventory is: | `PostgreSQL · Brio identity release orchestrator` | `release-brio-identity-db` | secret `KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN` | | `PostgreSQL · Keycloak cohort source reader` | `keycloak-cohort-restore` | secret `KEYCLOAK_COHORT_SOURCE_TOKEN` | | `Makepad Docker Hardened Images` | `keycloak-cohort-restore` | canonical fields `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT`, mirrored as secrets `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD` | -| `PostgreSQL · PR Checks App` | `postgres-ci-attestation` | variable `POSTGRES_PR_CHECK_APP_ID` and secret `POSTGRES_PR_CHECK_APP_PRIVATE_KEY` | -| `PostgreSQL · JIT Launcher App` | `postgres-ci-attestation` | public variable `POSTGRES_CI_LAUNCHER_APP_SENDER_ID`; private App fields remain on the controller host only | -| `PostgreSQL · JIT hypervisor attestation` | `postgres-ci-attestation` | public variables `POSTGRES_CI_ATTESTATION_PUBLIC_KEY` and `POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256`; the signing key remains on the hypervisor only | The `canary`, `production`, `staging-brio-identity-db`, and `keycloak-cohort-restore` environments also hold reviewed non-secret constants @@ -263,7 +259,7 @@ equivalent restriction. A release is blocked if an item or field is missing, if that exact policy or required reviewers are absent, or if GitHub differs from the reviewed Proton version. -Audit all six policies without changing provider state: +Audit all five policies without changing provider state: ```bash python3 scripts/reconcile-github-environment-main-policy.py audit @@ -285,11 +281,6 @@ Run this only from an administrator workstation whose `gh` session has environment-administration permission. The helper never reads or writes environment secrets. -Host-only JIT Launcher, attestation-signing, runner-controller, and alert -credentials are also canonical in the Proton items documented below, but are -intentionally never copied into Actions; only their public identities and -reviewed digests are mirrored to `postgres-ci-attestation`. - If automatic standalone rollback cannot re-establish the exact healthy target, the deploy script first deletes all incoming job credentials, then retains a root-owned mode-0700 recovery bundle under @@ -337,66 +328,10 @@ The protected `Verify Brio Identity Database Path` workflow has the run name database/role, TLS 1.2 or 1.3, and server-observed source `88.99.209.165`; no Keycloak database credential is granted to the PostgreSQL runner. -Pull requests use protected-base `pull_request_target` workflow code and reject -forks before checking out the exact same-repository head. The public repository -does not have a persistent PR runner. A dedicated root-only hypervisor queue -controller authorizes the exact queued run, attempt, job, PR head, PR base SHA, -protected workflow SHA, group, and label through GitHub's APIs. It durably -records a deterministic launch/resource manifest before launch, obtains a -one-job JIT configuration, and boots a fresh -self-contained qcow2 VM with the exclusive -`makepad-postgres-pr-ephemeral` label. The hypervisor firewall denies private, -WireGuard, link-local, metadata, multicast, IPv6, and hypervisor destinations; -only public DNS and TLS egress are allowed. The guest contains no repository, -deployment, Proton Pass, App, SSH, cloud, or service credential. - -After the job stops, the hypervisor destroys and proves absent the VM, disk, -cloud-init seed, network, firewall table, and GitHub runner registration. Only -then may it sign canonical `makepad.postgres.ci-attestation.v1` evidence with -its root-only Ed25519 key and dispatch it with the dedicated Launcher App. A -physically separate `makepad-postgres-ci-attestor` host in the selected-workflow -`org/Postgres PR Ephemeral` group runs only protected -`pr-ci-result.yml`; it has no Docker, deployment, or Launcher credentials. It -verifies the immutable numeric Launcher-App sender ID, signature, freshness, -nonce replay, reviewed base-image digest, exact authoritative run/job/runner -identity and conclusion, all teardown flags, and an independent 404 lookup for -the removed runner before the Checks-only App can publish the required -`postgres-ci` result. Failed or uncertain cleanup never produces a successful -check. Main pushes run independently on `org/Postgres Main CI`. The systemd -service uses control-group termination. On every controller start, all -`launching` or `recovery-required` records are reconciled before queue polling: -the exact VM, network, nftables table, job directory, and named runner -registration must all be proven absent. Recovered jobs are never executed or -attested again. Authoritative run/job completion is polled for a bounded period -after teardown to tolerate API propagation without rerunning untrusted code. - -Reconcile the four exact selected-workflow groups with -`scripts/configure-postgres-ci-runner-group.sh`, streaming its organization -runner-controller credential on stdin. Because `Makepad-fr/postgres` is public, -the groups explicitly allow public repositories but select only this exact -repository and protected-main workflow files. Repository-level runners and -runners exposed by unrelated groups are rejected. No persistent runner may -carry the JIT-only label. Install and supervise -`host/systemd/postgres-ci-queue-controller.service`; an abnormal launcher exit -must trigger the independent host alert service and no blind retry occurs. - -Long-lived CI controller material is canonical in Proton Pass before it is -installed at its narrow runtime boundary: - -| Proton Pass item | Exact runtime fields and authority | -| --- | --- | -| `PostgreSQL · PR Checks App` | protected `postgres-ci-attestation` environment variable `POSTGRES_PR_CHECK_APP_ID` and secret `POSTGRES_PR_CHECK_APP_PRIVATE_KEY`; App installed only on this repository with Metadata read, Actions read, Checks write, and organization self-hosted-runners read | -| `PostgreSQL · JIT Launcher App` | root-only hypervisor `POSTGRES_CI_LAUNCHER_APP_ID`, `POSTGRES_CI_LAUNCHER_APP_INSTALLATION_ID`, and mode-0400 `POSTGRES_CI_LAUNCHER_APP_PRIVATE_KEY_FILE`; repository variable `POSTGRES_CI_LAUNCHER_APP_SENDER_ID`; App installed only on this repository with Metadata read, Actions read, Contents write for repository dispatch, Issues write for secondary alerts, Pull requests read, and organization self-hosted-runners write | -| `PostgreSQL · JIT hypervisor attestation` | root-only mode-0400 `POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE`; repository variable `POSTGRES_CI_ATTESTATION_PUBLIC_KEY`; reviewed repository variable and root-only value `POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256`/`POSTGRES_CI_BASE_IMAGE_SHA256` | -| `PostgreSQL · runner-group controller` | administrator workstation input streamed to `scripts/configure-postgres-ci-runner-group.sh`; organization runner-group write and repository Metadata read only, never installed on a runner or hypervisor | -| `PostgreSQL · CI hypervisor alert` | root-only host alert URL file consumed only by the systemd `OnFailure` handler; never mirrored to GitHub Actions | - -The Launcher and Checks Apps are different installations and keys. Store their -numeric IDs, installation IDs, public-key fingerprints, approved base-image -digest, and rotation history beside the Proton items so reconciliation can -compare identities without printing secrets. The hypervisor's immutable base -image is root-owned, non-writable, has no backing/data chain, and is verified by -the reviewed SHA-256 both before and after each full per-job copy. +Pull-request and protected-main checks run through the repository's native +`CI` workflow on the existing Makepad runner. The workflow never uses +`pull_request_target`, never grants write permissions, and never exposes a +deployment environment to pull-request code. ## Keycloak 26.7.3 cohort restore evidence diff --git a/host/systemd/postgres-ci-queue-alert.service b/host/systemd/postgres-ci-queue-alert.service deleted file mode 100644 index 92572be..0000000 --- a/host/systemd/postgres-ci-queue-alert.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Alert on Postgres JIT queue-controller failure - -[Service] -Type=oneshot -User=root -Group=root -EnvironmentFile=/etc/makepad/postgres-ci/alert.env -ExecStart=/usr/local/libexec/makepad/send-postgres-host-alert postgres-ci-queue-controller -NoNewPrivileges=true -ProtectHome=true -ProtectSystem=strict diff --git a/host/systemd/postgres-ci-queue-controller.service b/host/systemd/postgres-ci-queue-controller.service deleted file mode 100644 index 227b4f1..0000000 --- a/host/systemd/postgres-ci-queue-controller.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Postgres supervised one-job JIT queue controller -After=network-online.target libvirtd.service -Wants=network-online.target -OnFailure=postgres-ci-queue-alert.service - -[Service] -Type=simple -User=root -Group=root -EnvironmentFile=/etc/makepad/postgres-ci/controller.env -ExecStart=/opt/makepad/postgres-ci/current/scripts/run-postgres-ci-queue-controller.sh -Restart=on-failure -RestartSec=15s -KillMode=control-group -NoNewPrivileges=true -ProtectHome=true -ProtectSystem=strict -ReadWritePaths=/run/lock /run /var/lib/makepad/postgres-ci /var/lib/libvirt - -[Install] -WantedBy=multi-user.target diff --git a/scripts/ci-base-image.py b/scripts/ci-base-image.py deleted file mode 100755 index 90c2993..0000000 --- a/scripts/ci-base-image.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -from pathlib import Path - - -def file_digest(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb", buffering=0) as source: - while block := source.read(1024 * 1024): - digest.update(block) - return digest.hexdigest() - - -def assert_digest(path: Path, expected: str) -> None: - if file_digest(path) != expected: - raise ValueError("reviewed base-image digest changed") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("path", type=Path) - parser.add_argument("expected") - args = parser.parse_args() - assert_digest(args.path, args.expected) - print(args.expected) - - -if __name__ == "__main__": - main() diff --git a/scripts/configure-postgres-ci-runner-group.sh b/scripts/configure-postgres-ci-runner-group.sh deleted file mode 100755 index a4100cf..0000000 --- a/scripts/configure-postgres-ci-runner-group.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Runner labels select a host; these selected-workflow organization groups are -# the policy boundary that prevents branch-authored workflow code from -# selecting the CI attestor. The JIT label is never persistent. - -readonly organization="Makepad-fr" -readonly repository="postgres" -readonly api_version="2022-11-28" - -die() { - printf '%s\n' "$*" >&2 - exit 1 -} - -[[ $# -eq 0 ]] || die "usage: configure-postgres-ci-runner-group.sh < GITHUB_ORG_RUNNER_CONTROLLER_TOKEN" -for command_name in gh python3 sort; do - command -v "${command_name}" >/dev/null || die "${command_name} is required" -done - -IFS= read -r controller_token || die "An organization runner-controller token is required on standard input" -[[ "${controller_token}" =~ ^(github_pat_|ghp_|ghs_|ghu_)[A-Za-z0-9_]+$ ]] || die "The controller token has an invalid format" -export GH_TOKEN="${controller_token}" -unset controller_token - -repository_id=$(gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "repos/${organization}/${repository}" --jq .id) -[[ "${repository_id}" =~ ^[1-9][0-9]*$ ]] || die "Could not resolve the Postgres repository ID" - -groups=( - 'Postgres PR Ephemeral|Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/pr-ci-result.yml@refs/heads/main|makepad-postgres-ci-attestor|makepad-postgres-pr-ephemeral' - 'Postgres Main CI|Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main|makepad-postgres-main-ci|' - 'Postgres Deploy|Makepad-fr/postgres/.github/workflows/manual-deploy.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/deploy-brio-identity-db.yml@refs/heads/main|makepad-postgres-deploy|' - 'Postgres Release|Makepad-fr/postgres/.github/workflows/release-brio-identity-db.yml@refs/heads/main,Makepad-fr/postgres/.github/workflows/verify-keycloak-cohort-restores.yml@refs/heads/main|makepad-postgres-release|' -) - -temporary_directory=$(mktemp -d) -chmod 0700 "${temporary_directory}" -cleanup() { - find "${temporary_directory}" -depth -mindepth 1 -delete - rmdir -- "${temporary_directory}" - unset GH_TOKEN -} -trap cleanup EXIT -all_configured_runner_ids="${temporary_directory}/configured-runner-ids" -: >"${all_configured_runner_ids}" -reconciled_groups="${temporary_directory}/reconciled-groups" -: >"${reconciled_groups}" - -# Reconcile every group before checking runner placement. This deliberately -# creates the fail-closed trust domain even when the attestor has not been -# registered yet; validation happens only after the desired state is applied. -for entry in "${groups[@]}"; do - IFS='|' read -r group_name selected_workflows required_labels forbidden_persistent_labels <<<"${entry}" - group_list="${temporary_directory}/groups.json" - gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups?per_page=100" >"${group_list}" - group_id=$(python3 - "${group_list}" "${group_name}" <<'PY' -import json -import pathlib -import sys - -payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) -groups = payload.get("runner_groups", []) -if payload.get("total_count", len(groups)) > len(groups): - raise SystemExit("more than 100 organization runner groups require explicit pagination") -matches = [group.get("id") for group in groups if group.get("name") == sys.argv[2]] -if len(matches) > 1: - raise SystemExit(f"duplicate runner groups named {sys.argv[2]}") -if matches: - print(matches[0]) -PY - ) - - payload=$(python3 - "${group_name}" "${selected_workflows}" "${repository_id}" <<'PY' -import json -import sys - -print(json.dumps({ - "name": sys.argv[1], - "visibility": "selected", - "allows_public_repositories": True, - "restricted_to_workflows": True, - "selected_workflows": sys.argv[2].split(","), - "selected_repository_ids": [int(sys.argv[3])], -}, separators=(",", ":"))) -PY - ) - - if [[ -z "${group_id}" ]]; then - created=$(printf '%s' "${payload}" | gh api --method POST \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups" --input -) - group_id=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' <<<"${created}") - else - update_payload=$(python3 - "${group_name}" "${selected_workflows}" <<'PY' -import json -import sys - -print(json.dumps({ - "name": sys.argv[1], - "visibility": "selected", - "allows_public_repositories": True, - "restricted_to_workflows": True, - "selected_workflows": sys.argv[2].split(","), -}, separators=(",", ":"))) -PY - ) - printf '%s' "${update_payload}" | gh api --method PATCH \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}" --input - >/dev/null - fi - [[ "${group_id}" =~ ^[1-9][0-9]*$ ]] || die "Invalid runner group ID for ${group_name}" - - gh api --method PUT --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}/repositories/${repository_id}" >/dev/null - - repositories="${temporary_directory}/repositories-${group_id}.json" - gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}/repositories?per_page=100" >"${repositories}" - while IFS= read -r unrelated_repository_id; do - [[ -z "${unrelated_repository_id}" ]] || gh api --method DELETE \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}/repositories/${unrelated_repository_id}" >/dev/null - done < <(python3 - "${repositories}" "${repository_id}" <<'PY' -import json -import pathlib -import sys - -payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) -repositories = payload.get("repositories", []) -if payload.get("total_count", len(repositories)) > len(repositories): - raise SystemExit("more than 100 selected repositories require explicit pagination") -expected = int(sys.argv[2]) -for repository in repositories: - repository_id = repository.get("id") - if isinstance(repository_id, int) and repository_id != expected: - print(repository_id) -PY - ) - - printf '%s|%s|%s|%s|%s\n' "${group_name}" "${group_id}" "${selected_workflows}" "${required_labels}" \ - "${forbidden_persistent_labels}" \ - >>"${reconciled_groups}" - printf 'Reconciled runner-group configuration %s (%s).\n' "${group_name}" "${group_id}" -done - -# Read back every group only after the complete reconciliation pass. A missing -# host can therefore fail bootstrap without preventing creation or repair of a -# later group. -while IFS='|' read -r group_name group_id selected_workflows required_labels forbidden_persistent_labels; do - observed_group="${temporary_directory}/group-${group_id}.json" - observed_repositories="${temporary_directory}/observed-repositories-${group_id}.json" - observed_runners="${temporary_directory}/runners-${group_id}.json" - gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}" >"${observed_group}" - gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}/repositories?per_page=100" >"${observed_repositories}" - gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${group_id}/runners?per_page=100" >"${observed_runners}" - python3 - "${observed_group}" "${observed_repositories}" "${observed_runners}" \ - "${group_name}" "${selected_workflows}" "${required_labels}" "${repository_id}" \ - "${all_configured_runner_ids}" "${forbidden_persistent_labels}" <<'PY' -import json -import pathlib -import sys - -group = json.loads(pathlib.Path(sys.argv[1]).read_text()) -repository_payload = json.loads(pathlib.Path(sys.argv[2]).read_text()) -runner_payload = json.loads(pathlib.Path(sys.argv[3]).read_text()) -repositories = repository_payload.get("repositories", []) -runners = runner_payload.get("runners", []) -expected = { - "name": sys.argv[4], - "visibility": "selected", - "allows_public_repositories": True, - "restricted_to_workflows": True, - "workflow_restrictions_read_only": False, -} -for key, value in expected.items(): - if group.get(key) != value: - raise SystemExit(f"runner group {sys.argv[4]} has unexpected {key}: {group.get(key)!r}") -if sorted(group.get("selected_workflows", [])) != sorted(sys.argv[5].split(",")): - raise SystemExit(f"runner group {sys.argv[4]} has unexpected selected_workflows") -if repository_payload.get("total_count", len(repositories)) > len(repositories): - raise SystemExit(f"runner group {sys.argv[4]} has more than 100 selected repositories") -if [repository.get("id") for repository in repositories] != [int(sys.argv[7])]: - raise SystemExit(f"runner group {sys.argv[4]} is not restricted to Postgres") -if runner_payload.get("total_count", len(runners)) > len(runners): - raise SystemExit(f"runner group {sys.argv[4]} has more than 100 runners") -available_labels = { - label.get("name").lower() - for runner in runners - for label in runner.get("labels", []) - if isinstance(label.get("name"), str) -} -missing = sorted(set(sys.argv[6].split(",")) - available_labels) -if missing: - raise SystemExit(f"runner group {sys.argv[4]} has no host for labels: {', '.join(missing)}") -required = sys.argv[6].split(",") -forbidden_persistent = {label for label in sys.argv[9].split(",") if label} -present_forbidden = sorted(forbidden_persistent & available_labels) -if present_forbidden: - raise SystemExit( - f"runner group {sys.argv[4]} has a persistent runner carrying JIT-only labels: " - f"{', '.join(present_forbidden)}" - ) -label_owners = { - label: { - runner.get("id") - for runner in runners - if label in { - str(item.get("name", "")).lower() - for item in runner.get("labels", []) - if isinstance(item.get("name"), str) - } - } - for label in required -} -default_labels = {"self-hosted", "linux", "x64", "makepad"} -for runner in runners: - labels = { - str(item.get("name", "")).lower() - for item in runner.get("labels", []) - if isinstance(item.get("name"), str) - } - owned = set(required) & labels - unexpected = labels - default_labels - set(required) - forbidden_persistent - if len(owned) != 1 or unexpected: - raise SystemExit( - f"runner group {sys.argv[4]} contains an unapproved runner/label set on " - f"{runner.get('name', runner.get('id'))}: {sorted(labels)}" - ) -for label, owners in label_owners.items(): - if len(owners) != 1: - raise SystemExit( - f"runner group {sys.argv[4]} must have exactly one persistent host for {label}" - ) -for index, label in enumerate(required): - for other in required[index + 1:]: - if label_owners[label] & label_owners[other]: - raise SystemExit( - f"runner group {sys.argv[4]} places mutually trusted labels " - f"{label} and {other} on the same host" - ) -with pathlib.Path(sys.argv[8]).open("a", encoding="utf-8") as destination: - for runner in runners: - runner_id = runner.get("id") - if isinstance(runner_id, int): - destination.write(f"{runner_id}\n") -PY - printf 'Validated runner group %s (%s).\n' "${group_name}" "${group_id}" -done <"${reconciled_groups}" - -# Repository-level or unrelated organization runners bypass these two groups. -# Refuse to declare bootstrap complete while any such runner is available. -accessible_runners="${temporary_directory}/repository-runners.json" -gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "repos/${organization}/${repository}/actions/runners?per_page=100" >"${accessible_runners}" -python3 - "${accessible_runners}" "${all_configured_runner_ids}" <<'PY' -import json -import pathlib -import sys - -payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) -runners = payload.get("runners", []) -if payload.get("total_count", len(runners)) > len(runners): - raise SystemExit("more than 100 Postgres-accessible runners require explicit pagination") -configured = { - int(value) - for value in pathlib.Path(sys.argv[2]).read_text().splitlines() - if value.strip() -} -unexpected = [runner for runner in runners if runner.get("id") not in configured] -if unexpected: - names = ", ".join(str(runner.get("name", runner.get("id"))) for runner in unexpected) - raise SystemExit(f"Postgres still exposes runners outside its restricted groups: {names}") -PY - -printf 'Postgres runner access is restricted to the four selected-workflow groups.\n' diff --git a/scripts/dispatch-ci-attestation.mjs b/scripts/dispatch-ci-attestation.mjs deleted file mode 100644 index 480c237..0000000 --- a/scripts/dispatch-ci-attestation.mjs +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env node -import { sign } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import process from "node:process"; -import { canonicalJSON } from "./publish-pr-ci-check.mjs"; - -const evidencePath = process.env.POSTGRES_CI_ATTESTATION_JSON_FILE; -const privateKeyPath = process.env.POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE; -if (!evidencePath?.startsWith("/") || !privateKeyPath?.startsWith("/")) throw new Error("absolute attestation and private-key paths are required"); -const [source, privateKey] = await Promise.all([readFile(evidencePath, "utf8"), readFile(privateKeyPath, "utf8")]); -const attestation = JSON.parse(source); -const canonical = canonicalJSON(attestation); -if (`${canonical}\n` !== source && canonical !== source) throw new Error("attestation file is not canonical JSON"); - -let token = ""; -for await (const chunk of process.stdin) token += chunk; -token = token.trim(); -if (!/^ghs_[A-Za-z0-9_]{20,}$/.test(token)) throw new Error("a dedicated Launcher App installation token is required on stdin"); -const signature = sign(null, Buffer.from(canonical), privateKey).toString("base64url"); -const response = await fetch("https://api.github.com/repos/Makepad-fr/postgres/dispatches", { - method: "POST", - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "User-Agent": "makepad-postgres-ci-launcher", - "X-GitHub-Api-Version": "2022-11-28", - }, - body: JSON.stringify({event_type: "postgres-pr-ci-attestation", client_payload: {attestation, signature}}), - redirect: "error", - signal: AbortSignal.timeout(30_000), -}); -token = ""; -if (response.status !== 204) throw new Error(`Launcher App attestation dispatch failed with ${response.status}`); -process.stdout.write(`Dispatched signed teardown attestation for run ${attestation.run.id}, attempt ${attestation.run.attempt}.\n`); diff --git a/scripts/postgres-ci-queue-controller.mjs b/scripts/postgres-ci-queue-controller.mjs deleted file mode 100644 index 3899858..0000000 --- a/scripts/postgres-ci-queue-controller.mjs +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env node -import crypto, { createSign } from "node:crypto"; -import { spawn } from "node:child_process"; -import { lstat, mkdir, readFile, realpath, rename, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -const REPOSITORY = "Makepad-fr/postgres"; -const WORKFLOW_PATH = ".github/workflows/ci.yml"; -const LABELS = ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]; - -const required = (name, env = process.env) => { - const value = env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -}; - -const github = async ({ token, method = "GET", endpoint, body, fetchImpl = fetch }) => { - const response = await fetchImpl(`https://api.github.com${endpoint}`, { - method, - headers: {Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": "makepad-postgres-ci-controller", "X-GitHub-Api-Version": "2022-11-28"}, - body: body === undefined ? undefined : JSON.stringify(body), - redirect: "error", - signal: AbortSignal.timeout(30_000), - }); - const text = await response.text(); - let payload = {}; - if (text) payload = JSON.parse(text); - if (!response.ok) throw new Error(`GitHub ${method} ${endpoint} failed with ${response.status}`); - return payload; -}; - -const appJWT = ({appID, key, now = Date.now()}) => { - if (!/^[1-9]\d*$/.test(appID)) throw new Error("Launcher App ID must be numeric"); - const issued = Math.floor(now / 1000) - 60; - const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); - const unsigned = `${encode({alg: "RS256", typ: "JWT"})}.${encode({iat: issued, exp: issued + 540, iss: appID})}`; - const signer = createSign("RSA-SHA256"); - signer.update(unsigned); - signer.end(); - return `${unsigned}.${signer.sign(key, "base64url")}`; -}; - -export const selectAuthorizedJobs = ({ runs, jobsByRun, pullRequests, repositoryID }) => { - if (!Array.isArray(runs.workflow_runs) || runs.total_count !== runs.workflow_runs.length) throw new Error("workflow-run response is truncated"); - const selected = []; - for (const run of runs.workflow_runs) { - if (!Number.isSafeInteger(run.id) || run.id <= 0 || !Number.isSafeInteger(run.run_attempt) || run.run_attempt <= 0) continue; - const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; - if (run.name !== "CI" || run.path !== WORKFLOW_PATH || run.status !== "queued" || run.repository?.id !== repositoryID || !/^[a-f0-9]{40}$/.test(run.head_sha || "")) continue; - let sourceSHA; - let pullRequestNumber = null; - if (run.event === "pull_request_target") { - if (associations.length !== 1 || !Number.isSafeInteger(associations[0]?.number)) continue; - const association = associations[0]; - const pull = pullRequests.get(association.number); - if (association.head?.repo?.id !== repositoryID || association.base?.repo?.id !== repositoryID || association.base?.ref !== "main" || association.base?.sha !== run.head_sha || !/^[a-f0-9]{40}$/.test(association.head?.sha || "") || pull?.number !== association.number || pull?.head?.sha !== association.head?.sha || pull?.head?.repo?.id !== repositoryID || pull?.base?.repo?.id !== repositoryID || pull?.base?.ref !== "main" || pull?.base?.sha !== run.head_sha) continue; - sourceSHA = association.head.sha; - pullRequestNumber = association.number; - } else if (run.event === "push") { - if (run.head_branch !== "main") continue; - sourceSHA = run.head_sha; - } else { - continue; - } - const response = jobsByRun.get(`${run.id}:${run.run_attempt}`); - if (!response || !Array.isArray(response.jobs) || response.total_count !== response.jobs.length) throw new Error("workflow-job response is missing or truncated"); - for (const job of response.jobs) { - const labels = Array.isArray(job.labels) ? job.labels.map((value) => String(value).toLowerCase()).sort() : []; - if (Number.isSafeInteger(job.id) && job.id > 0 && job.name === "policy-and-integration" && job.status === "queued" && job.run_id === run.id && job.head_sha === run.head_sha && job.workflow_name === "CI" && labels.length === LABELS.length && labels.every((value, index) => value === [...LABELS].sort()[index])) { - selected.push({runID: run.id, attempt: run.run_attempt, jobID: job.id, event: run.event, sourceSHA, workflowSHA: run.head_sha, pullRequestNumber}); - } - } - } - return selected.sort((left, right) => left.jobID - right.jobID); -}; - -const atomicState = async (file, state) => { - const incoming = `${file}.incoming-${process.pid}-${crypto.randomBytes(8).toString("hex")}`; - await writeFile(incoming, `${JSON.stringify(state, null, 2)}\n`, {mode: 0o600, flag: "wx"}); - await rename(incoming, file); -}; - -const runLauncher = ({launcher, token, metadata, environment, arguments: launcherArguments = []}) => new Promise((resolve, reject) => { - const child = spawn(launcher, launcherArguments, { - env: {...environment, POSTGRES_CI_RUN_ID: String(metadata.runID || ""), POSTGRES_CI_RUN_ATTEMPT: String(metadata.attempt || ""), POSTGRES_CI_JOB_ID: String(metadata.jobID || ""), POSTGRES_CI_RUN_EVENT: metadata.event || "", POSTGRES_CI_HEAD_SHA: metadata.sourceSHA || "", POSTGRES_CI_WORKFLOW_SHA: metadata.workflowSHA || "", POSTGRES_CI_ATTESTATION_NONCE: metadata.nonce || "", POSTGRES_CI_LAUNCH_ID: metadata.launchID || ""}, - stdio: ["pipe", "inherit", "inherit"], - }); - child.stdin.end(`${token}\n`); - child.once("error", reject); - child.once("exit", (code, signal) => code === 0 && signal === null ? resolve() : reject(new Error(`launcher exited ${code ?? signal}`))); -}); - -export const reconcileIncompleteJobs = async ({state, persist, reconcile}) => { - for (const [jobID, record] of Object.entries(state.jobs).sort(([left], [right]) => Number(left) - Number(right))) { - if (!record || !["launching", "recovery-required"].includes(record.status)) continue; - if (!/^j[1-9][0-9]{0,15}-[a-f0-9]{16}$/.test(record.launchID || "")) { - throw new Error(`incomplete job ${jobID} has no safe deterministic resource manifest`); - } - try { - await reconcile(record); - record.status = "failed-recovered"; - record.failure = "controller restart reconciled an incomplete disposable launch"; - record.finishedAt = new Date().toISOString(); - await persist(); - } catch (error) { - record.status = "recovery-required"; - record.failure = error instanceof Error ? error.message.slice(0, 200) : "unknown reconciliation failure"; - await persist(); - throw error; - } - } -}; - -export const controller = async ({environment = process.env, fetchImpl = fetch, once = false} = {}) => { - if (process.getuid?.() !== 0) throw new Error("queue controller must run as root on the dedicated hypervisor"); - const repositoryID = Number(required("POSTGRES_CI_REPOSITORY_ID", environment)); - const appID = required("POSTGRES_CI_LAUNCHER_APP_ID", environment); - const installationID = required("POSTGRES_CI_LAUNCHER_APP_INSTALLATION_ID", environment); - const privateKeyFile = required("POSTGRES_CI_LAUNCHER_APP_PRIVATE_KEY_FILE", environment); - const stateDirectory = required("POSTGRES_CI_CONTROLLER_STATE_DIRECTORY", environment); - const launcher = required("POSTGRES_CI_LAUNCHER", environment); - if (!Number.isSafeInteger(repositoryID) || repositoryID <= 0 || !/^[1-9]\d*$/.test(installationID)) throw new Error("repository and installation IDs must be positive integers"); - if (!/^\/var\/lib\/makepad\/postgres-ci\/[A-Za-z0-9._/-]+$/.test(stateDirectory) || stateDirectory.includes("..") || path.normalize(stateDirectory) !== stateDirectory) throw new Error("controller state directory is outside the root-owned Postgres PR Ephemeral tree"); - for (const [file, expectedMode] of [[privateKeyFile, 0o400], [launcher, 0o755]]) { - if (!path.isAbsolute(file)) throw new Error(`controller file is not absolute: ${file}`); - const value = await lstat(file); - if (!value.isFile() || value.isSymbolicLink() || value.uid !== 0 || (value.mode & 0o777) !== expectedMode || await realpath(file) !== file) throw new Error(`insecure controller file: ${file}`); - } - const key = await readFile(privateKeyFile, "utf8"); - await mkdir(stateDirectory, {recursive: true, mode: 0o700}); - const directory = await lstat(stateDirectory); - if (!directory.isDirectory() || directory.isSymbolicLink() || directory.uid !== 0 || (directory.mode & 0o777) !== 0o700 || await realpath(stateDirectory) !== stateDirectory) throw new Error("controller state directory must be a root-only real path"); - const stateFile = path.join(stateDirectory, "jobs.json"); - let state = {version: 2, jobs: {}}; - try { state = JSON.parse(await readFile(stateFile, "utf8")); } catch (error) { if (error.code !== "ENOENT") throw error; } - if (state.version === 1 && state.jobs && typeof state.jobs === "object") { - if (Object.values(state.jobs).some((record) => record?.status === "launching")) throw new Error("legacy controller state contains an unreconciled launch; operator recovery is required"); - state = {version: 2, jobs: state.jobs}; - await atomicState(stateFile, state); - } - if (state.version !== 2 || !state.jobs || typeof state.jobs !== "object" || Array.isArray(state.jobs)) throw new Error("controller state is invalid"); - - do { - const jwt = appJWT({appID, key}); - const installation = await github({token: jwt, method: "POST", endpoint: `/app/installations/${installationID}/access_tokens`, body: {repositories: ["postgres"], permissions: {actions: "read", contents: "write", issues: "write", organization_self_hosted_runners: "write", pull_requests: "read"}}, fetchImpl}); - const token = installation.token; - if (typeof token !== "string" || !token.startsWith("ghs_")) throw new Error("Launcher App did not issue an installation token"); - await reconcileIncompleteJobs({ - state, - persist: () => atomicState(stateFile, state), - reconcile: (record) => runLauncher({launcher, token, metadata: record, environment, arguments: ["--reconcile", record.launchID]}), - }); - const runs = await github({token, endpoint: `/repos/${REPOSITORY}/actions/workflows/ci.yml/runs?status=queued&per_page=100`, fetchImpl}); - const jobsByRun = new Map(); - const pullRequests = new Map(); - for (const run of runs.workflow_runs || []) { - jobsByRun.set(`${run.id}:${run.run_attempt}`, await github({token, endpoint: `/repos/${REPOSITORY}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100`, fetchImpl})); - const number = run.pull_requests?.[0]?.number; - if (Number.isSafeInteger(number) && !pullRequests.has(number)) pullRequests.set(number, await github({token, endpoint: `/repos/${REPOSITORY}/pulls/${number}`, fetchImpl})); - } - const pending = selectAuthorizedJobs({runs, jobsByRun, pullRequests, repositoryID}).filter((job) => !state.jobs[String(job.jobID)]); - for (const job of pending) { - const nonce = crypto.randomBytes(32).toString("base64url"); - const launchID = `j${job.jobID}-${crypto.randomBytes(8).toString("hex")}`; - state.jobs[String(job.jobID)] = {...job, nonce, launchID, status: "launching", createdAt: new Date().toISOString()}; - await atomicState(stateFile, state); - try { - await runLauncher({launcher, token, metadata: {...job, nonce, launchID}, environment}); - state.jobs[String(job.jobID)].status = "completed"; - } catch (error) { - // Any nonzero launcher exit is cleanup-uncertain. Keep the deterministic - // launch identity eligible for mandatory startup reconciliation; the - // controller must never convert an uncertain launch into terminal state. - state.jobs[String(job.jobID)].status = "recovery-required"; - state.jobs[String(job.jobID)].failure = error instanceof Error ? error.message.slice(0, 200) : "unknown launcher failure"; - state.jobs[String(job.jobID)].recoveryRequiredAt = new Date().toISOString(); - // Persist the no-retry decision before any network alert. Exiting - // nonzero then activates the independent host OnFailure channel; the - // GitHub issue below is useful secondary evidence, not the sole alert. - await atomicState(stateFile, state); - const title = `Postgres JIT launcher failed for job ${job.jobID}`; - try { - await github({token, method: "POST", endpoint: `/repos/${REPOSITORY}/issues`, body: {title, body: `The supervised hypervisor controller could not complete run ${job.runID}, attempt ${job.attempt}, job ${job.jobID}. No success attestation was issued. Inspect the root-only hypervisor journal.`}, fetchImpl}); - } catch { - // The systemd OnFailure webhook remains independent of GitHub. - } - throw error; - } - state.jobs[String(job.jobID)].finishedAt = new Date().toISOString(); - await atomicState(stateFile, state); - } - if (once) break; - const pollSeconds = Math.min(120, Math.max(15, Number(environment.POSTGRES_CI_POLL_SECONDS || 30))); - await new Promise((resolve) => setTimeout(resolve, pollSeconds * 1000)); - } while (true); -}; - -const invokedAsCLI = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; -if (invokedAsCLI) controller({once: process.argv.includes("--once")}).catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exitCode = 1; -}); diff --git a/scripts/publish-pr-ci-check.mjs b/scripts/publish-pr-ci-check.mjs deleted file mode 100644 index a5e7e9d..0000000 --- a/scripts/publish-pr-ci-check.mjs +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env node -import { createSign, verify as verifySignature } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { pathToFileURL } from "node:url"; - -const EXPECTED_REPOSITORY = "Makepad-fr/postgres"; -const EXPECTED_WORKFLOW_PATH = ".github/workflows/ci.yml"; -const EXPECTED_WORKFLOW_NAME = "CI"; -const EXPECTED_RUNNER_GROUP = "Postgres PR Ephemeral"; -const EXPECTED_RUNNER_LABELS = ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]; -const EXPECTED_SCHEMA = "makepad.postgres.ci-attestation.v1"; -const MAX_ATTESTATION_AGE_MS = 10 * 60 * 1000; -const MAX_FUTURE_SKEW_MS = 60 * 1000; -export const CHECK_NAMES = ["postgres-ci"]; - -const required = (name, environment = process.env) => { - const value = environment[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -}; - -const exactKeys = (value, keys, label) => { - if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`); - const actual = Object.keys(value).sort(); - const expected = [...keys].sort(); - if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) throw new Error(`${label} has unexpected fields`); -}; - -export const canonicalJSON = (value) => { - if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); - if (typeof value === "number" && Number.isSafeInteger(value)) return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(canonicalJSON).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJSON(value[key])}`).join(",")}}`; - } - throw new Error("attestation contains a non-canonical JSON value"); -}; - -const base64url = (value) => Buffer.from(value).toString("base64url"); - -export const createAppJWT = ({ appID, privateKey, now = new Date() }) => { - if (!/^[1-9]\d*$/.test(appID)) throw new Error("GitHub App ID must be a positive integer"); - if (!/^-----BEGIN (?:RSA )?PRIVATE KEY-----/.test(privateKey.trim())) throw new Error("GitHub App private key must be a PEM private key"); - const issuedAt = Math.floor(now.getTime() / 1000) - 60; - const header = base64url(JSON.stringify({ alg: "RS256", typ: "JWT" })); - const payload = base64url(JSON.stringify({ iat: issuedAt, exp: issuedAt + 540, iss: appID })); - const unsigned = `${header}.${payload}`; - const signer = createSign("RSA-SHA256"); - signer.update(unsigned); - signer.end(); - return `${unsigned}.${signer.sign(privateKey, "base64url")}`; -}; - -const githubResponse = async ({ token, method = "GET", path, body, fetchImpl = fetch }) => { - const response = await fetchImpl(`https://api.github.com${path}`, { - method, - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28" - }, - body: body === undefined ? undefined : JSON.stringify(body), - redirect: "error", - signal: AbortSignal.timeout(30_000) - }); - const text = await response.text(); - let payload = {}; - if (text) { - try { payload = JSON.parse(text); } - catch { throw new Error(`GitHub ${method} ${path} returned non-JSON (${response.status})`); } - } - return { ok: response.ok, status: response.status, payload }; -}; - -const githubJSON = async (options) => { - const response = await githubResponse(options); - if (!response.ok) { - const message = typeof response.payload.message === "string" ? response.payload.message : "request failed"; - throw new Error(`GitHub ${options.method || "GET"} ${options.path} failed (${response.status}): ${message}`); - } - return response.payload; -}; - -export const verifySignedAttestation = ({ event, publicKey, approvedDigest, launcherSenderID, now = new Date() }) => { - if (!/^[1-9]\d*$/.test(String(launcherSenderID))) throw new Error("Launcher App sender ID must be a positive integer"); - if (event?.action !== "postgres-pr-ci-attestation") throw new Error("unexpected repository dispatch action"); - if (event?.repository?.full_name !== EXPECTED_REPOSITORY) throw new Error("attestation targets the wrong repository"); - if (event?.sender?.type !== "Bot" || String(event?.sender?.id) !== String(launcherSenderID)) throw new Error("attestation dispatch was not sent by the dedicated Launcher App"); - exactKeys(event.client_payload, ["attestation", "signature"], "dispatch payload"); - const attestation = event.client_payload.attestation; - const signature = event.client_payload.signature; - exactKeys(attestation, ["schema", "repository", "workflow", "ref", "run", "runner", "base_image_sha256", "nonce", "issued_at", "registration_absent", "teardown"], "attestation"); - exactKeys(attestation.workflow, ["name", "path"], "attestation workflow"); - exactKeys(attestation.run, ["id", "attempt", "job_id", "job_name", "event", "head_sha", "workflow_sha", "conclusion"], "attestation run"); - exactKeys(attestation.runner, ["id", "name", "group_id", "group_name", "labels"], "attestation runner"); - exactKeys(attestation.teardown, ["vm", "network", "firewall", "disk"], "attestation teardown"); - if (attestation.schema !== EXPECTED_SCHEMA || attestation.repository !== EXPECTED_REPOSITORY) throw new Error("attestation schema or repository mismatch"); - if (attestation.workflow.name !== EXPECTED_WORKFLOW_NAME || attestation.workflow.path !== EXPECTED_WORKFLOW_PATH || attestation.ref !== "refs/heads/main") throw new Error("attestation workflow or protected ref mismatch"); - for (const [label, value] of Object.entries({run_id: attestation.run.id, run_attempt: attestation.run.attempt, job_id: attestation.run.job_id, runner_id: attestation.runner.id, runner_group_id: attestation.runner.group_id})) { - if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${label} must be a positive safe integer`); - } - if (attestation.run.job_name !== "policy-and-integration" || !["pull_request_target", "push"].includes(attestation.run.event) || !/^[a-f0-9]{40}$/.test(attestation.run.head_sha) || !/^[a-f0-9]{40}$/.test(attestation.run.workflow_sha) || (attestation.run.event === "push" && attestation.run.head_sha !== attestation.run.workflow_sha) || !["success", "failure"].includes(attestation.run.conclusion)) throw new Error("attested job identity or conclusion is invalid"); - if (!/^postgres-ci-jit-[a-z0-9-]{8,80}$/.test(attestation.runner.name) || attestation.runner.group_name !== EXPECTED_RUNNER_GROUP) throw new Error("attested runner identity is invalid"); - const labels = Array.isArray(attestation.runner.labels) ? attestation.runner.labels : []; - if (labels.length !== EXPECTED_RUNNER_LABELS.length || labels.some((label, index) => label !== EXPECTED_RUNNER_LABELS[index])) throw new Error("attested runner labels are not the exact JIT label set"); - if (attestation.base_image_sha256 !== approvedDigest || !/^[a-f0-9]{64}$/.test(approvedDigest)) throw new Error("attested base image digest is not approved"); - if (!/^[A-Za-z0-9_-]{43}$/.test(attestation.nonce)) throw new Error("attestation nonce is invalid"); - if (typeof attestation.issued_at !== "string" || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(attestation.issued_at)) throw new Error("attestation issued_at must be canonical UTC RFC 3339"); - const issuedAt = Date.parse(attestation.issued_at); - if (!Number.isFinite(issuedAt) || issuedAt < now.getTime() - MAX_ATTESTATION_AGE_MS || issuedAt > now.getTime() + MAX_FUTURE_SKEW_MS) throw new Error("attestation is stale or from the future"); - if (attestation.registration_absent !== true || Object.values(attestation.teardown).some((value) => value !== true)) throw new Error("runner registration or hypervisor teardown is incomplete"); - if (typeof signature !== "string" || !/^[A-Za-z0-9_-]{80,100}$/.test(signature)) throw new Error("attestation signature is invalid"); - let signatureValid = false; - try { signatureValid = verifySignature(null, Buffer.from(canonicalJSON(attestation)), publicKey, Buffer.from(signature, "base64url")); } - catch { signatureValid = false; } - if (!signatureValid) throw new Error("attestation signature verification failed"); - return attestation; -}; - -export const validateAuthoritativeEvidence = ({ attestation, run, jobs, job, pullRequest = null, runnerListStatus = 200, runnerLookupStatus }) => { - if (run.id !== attestation.run.id || run.run_attempt !== attestation.run.attempt || run.event !== attestation.run.event || run.head_sha !== attestation.run.workflow_sha || run.head_branch !== "main" || run.path !== EXPECTED_WORKFLOW_PATH || run.name !== EXPECTED_WORKFLOW_NAME || run.status !== "completed" || run.repository?.full_name !== EXPECTED_REPOSITORY || !Number.isSafeInteger(run.repository?.id)) throw new Error("authoritative workflow run does not match the attestation"); - if (jobs.total_count !== (jobs.jobs || []).length) throw new Error("authoritative job response is truncated"); - const matches = (jobs.jobs || []).filter((candidate) => candidate.id === attestation.run.job_id); - if (matches.length !== 1 || matches[0].id !== job.id) throw new Error("attested job is not unique in the authoritative run attempt"); - const labels = Array.isArray(job.labels) ? job.labels.map((label) => String(label).toLowerCase()).sort() : []; - const expectedLabels = [...EXPECTED_RUNNER_LABELS].sort(); - if (job.run_id !== run.id || job.head_sha !== attestation.run.workflow_sha || job.workflow_name !== EXPECTED_WORKFLOW_NAME || job.name !== attestation.run.job_name || job.status !== "completed" || job.runner_id !== attestation.runner.id || job.runner_name !== attestation.runner.name || job.runner_group_id !== attestation.runner.group_id || job.runner_group_name !== attestation.runner.group_name || labels.length !== expectedLabels.length || labels.some((label, index) => label !== expectedLabels[index])) throw new Error("authoritative job runner identity differs from the signed attestation"); - const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; - let pullRequestNumber = null; - if (attestation.run.event === "pull_request_target") { - if (associations.length !== 1) throw new Error("source run must identify exactly one pull request"); - const association = associations[0]; - if (association.head?.sha !== attestation.run.head_sha || association.head?.repo?.id !== run.repository?.id || association.base?.repo?.id !== run.repository?.id || association.base?.ref !== "main" || association.base?.sha !== attestation.run.workflow_sha || pullRequest?.number !== association.number || pullRequest.head?.sha !== attestation.run.head_sha || pullRequest.head?.repo?.full_name !== EXPECTED_REPOSITORY || pullRequest.base?.sha !== attestation.run.workflow_sha || pullRequest.base?.repo?.full_name !== EXPECTED_REPOSITORY || pullRequest.base?.ref !== "main") throw new Error("authoritative pull request differs from the signed head and base identities"); - pullRequestNumber = pullRequest.number; - } else if (attestation.run.head_sha !== attestation.run.workflow_sha) { - throw new Error("protected-main push source differs from its workflow SHA"); - } - const expectedConclusion = run.conclusion === "success" && job.conclusion === "success" ? "success" : "failure"; - if (attestation.run.conclusion !== expectedConclusion) throw new Error("signed conclusion differs from authoritative test result"); - if (runnerListStatus !== 200 || runnerLookupStatus !== 404) throw new Error("attested JIT runner is still registered or registration absence is uncertain"); - return { event: attestation.run.event, headSHA: attestation.run.head_sha, workflowSHA: attestation.run.workflow_sha, pullRequestNumber, conclusion: expectedConclusion, sourceRunID: run.id, sourceRunAttempt: run.run_attempt, detailsURL: run.html_url, nonce: attestation.nonce }; -}; - -export const assertNoAttestationReplay = ({existing, appID, prefix}) => { - if (!Number.isSafeInteger(existing.total_count) || existing.total_count !== (existing.check_runs || []).length) throw new Error("cannot prove Postgres PR Ephemeral replay protection"); - if ((existing.check_runs || []).some((check) => String(check.app?.id) === String(appID) && String(check.external_id || "").startsWith(prefix))) throw new Error("attestation replay detected for this run attempt"); -}; - -export const publishPRCheck = async ({ environment = process.env, fetchImpl = fetch, now = new Date() } = {}) => { - if (required("GITHUB_REPOSITORY", environment) !== EXPECTED_REPOSITORY || required("GITHUB_REF", environment) !== "refs/heads/main") throw new Error("PR attestation must run for Makepad-fr/postgres protected main"); - const event = JSON.parse(await readFile(required("GITHUB_EVENT_PATH", environment), "utf8")); - const attestation = verifySignedAttestation({ event, publicKey: required("POSTGRES_CI_ATTESTATION_PUBLIC_KEY", environment), approvedDigest: required("POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256", environment), launcherSenderID: required("POSTGRES_CI_LAUNCHER_APP_SENDER_ID", environment), now }); - const repositoryToken = required("GITHUB_TOKEN", environment); - const run = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/runs/${attestation.run.id}`, fetchImpl }); - const jobs = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/runs/${attestation.run.id}/attempts/${attestation.run.attempt}/jobs?per_page=100`, fetchImpl }); - const job = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/actions/jobs/${attestation.run.job_id}`, fetchImpl }); - const associations = Array.isArray(run.pull_requests) ? run.pull_requests : []; - let pullRequest = null; - if (attestation.run.event === "pull_request_target") { - if (associations.length !== 1 || !Number.isSafeInteger(associations[0]?.number)) throw new Error("source run has no unique pull request association"); - pullRequest = await githubJSON({ token: repositoryToken, path: `/repos/${EXPECTED_REPOSITORY}/pulls/${associations[0].number}`, fetchImpl }); - } - - const appID = required("POSTGRES_PR_CHECK_APP_ID", environment); - const appJWT = createAppJWT({ appID, privateKey: required("POSTGRES_PR_CHECK_APP_PRIVATE_KEY", environment), now }); - const installation = await githubJSON({ token: appJWT, path: `/repos/${EXPECTED_REPOSITORY}/installation`, fetchImpl }); - if (String(installation.app_id) !== appID || !Number.isSafeInteger(installation.id)) throw new Error("configured Checks App is not the Postgres installation"); - const installationToken = await githubJSON({ token: appJWT, method: "POST", path: `/app/installations/${installation.id}/access_tokens`, body: { repositories: ["postgres"], permissions: { checks: "write", organization_self_hosted_runners: "read" } }, fetchImpl }); - if (typeof installationToken.token !== "string" || !installationToken.token) throw new Error("Checks App installation did not issue a token"); - const runnerList = await githubResponse({ token: installationToken.token, path: "/orgs/Makepad-fr/actions/runners?per_page=1", fetchImpl }); - const runnerLookup = await githubResponse({ token: installationToken.token, path: `/orgs/Makepad-fr/actions/runners/${attestation.runner.id}`, fetchImpl }); - const verified = validateAuthoritativeEvidence({ attestation, run, jobs, job, pullRequest, runnerListStatus: runnerList.status, runnerLookupStatus: runnerLookup.status }); - - const externalID = `postgres-ci:${verified.event}:${verified.sourceRunID}:${verified.sourceRunAttempt}:${verified.nonce}`; - const checkRunIDs = {}; - for (const checkName of CHECK_NAMES) { - const existing = await githubJSON({ token: installationToken.token, path: `/repos/${EXPECTED_REPOSITORY}/commits/${verified.headSHA}/check-runs?check_name=${encodeURIComponent(checkName)}&filter=all&per_page=100`, fetchImpl }); - const prefix = `postgres-ci:${verified.event}:${verified.sourceRunID}:${verified.sourceRunAttempt}:`; - assertNoAttestationReplay({existing, appID, prefix}); - const scope = verified.event === "pull_request_target" ? `PR #${verified.pullRequestNumber}` : "protected-main push"; - const checkBody = { name: checkName, head_sha: verified.headSHA, status: "completed", conclusion: verified.conclusion, external_id: externalID, details_url: verified.detailsURL, completed_at: now.toISOString(), output: { title: verified.conclusion === "success" ? "Disposable CI and teardown verified" : "Disposable CI failed; teardown verified", summary: `Signed hypervisor evidence for ${scope}, run ${verified.sourceRunID}, attempt ${verified.sourceRunAttempt}.` } }; - const published = await githubJSON({ token: installationToken.token, method: "POST", path: `/repos/${EXPECTED_REPOSITORY}/check-runs`, body: checkBody, fetchImpl }); - if (published.name !== checkName || published.head_sha !== verified.headSHA || published.external_id !== externalID || published.conclusion !== verified.conclusion || String(published.app?.id) !== appID || !Number.isSafeInteger(published.id)) throw new Error(`published ${checkName} check does not match the verified signed result`); - checkRunIDs[checkName] = published.id; - } - return { ...verified, checkRunIDs, appID }; -}; - -const invokedAsCLI = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; -if (invokedAsCLI) { - publishPRCheck().then((result) => process.stdout.write(`Published ${CHECK_NAMES.join(",")}=${result.conclusion} for ${result.headSHA}.\n`)).catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exitCode = 1; - }); -} diff --git a/scripts/reconcile-github-environment-main-policy.py b/scripts/reconcile-github-environment-main-policy.py index 9da605c..7f745d4 100755 --- a/scripts/reconcile-github-environment-main-policy.py +++ b/scripts/reconcile-github-environment-main-policy.py @@ -17,7 +17,6 @@ "staging-brio-identity-db", "release-brio-identity-db", "keycloak-cohort-restore", - "postgres-ci-attestation", ) MAX_POLICY_PAGES = 1000 diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh index 6cdab10..bc3cbdc 100755 --- a/scripts/run-ci.sh +++ b/scripts/run-ci.sh @@ -3,9 +3,11 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "${repo_root}" +readonly shellcheck_image='docker.io/koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d' ./scripts/validate-postgres-config.sh -shellcheck \ +docker version >/dev/null +shellcheck_paths=( \ scripts/run-brio-encrypted-backup.sh \ scripts/run-brio-encrypted-backup-loop.sh \ scripts/deploy-brio-canary-postgres.sh \ @@ -28,15 +30,16 @@ shellcheck \ scripts/test-brio-release-evidence.sh \ scripts/test-keycloak-cohort-evidence.sh \ scripts/test-keycloak-cohort-hardening.sh \ - scripts/test-postgres-ci-jit-result.sh \ scripts/capture-keycloak-cohort-backups.sh \ scripts/restore-keycloak-cohort-backups.sh \ - scripts/run-postgres-ci-jit-vm.sh \ - scripts/run-postgres-ci-queue-controller.sh \ - scripts/configure-postgres-ci-runner-group.sh \ scripts/fixtures/brio-deployment-failure-fixture.sh \ scripts/fixtures/keycloak-cohort-cleaner-fixture.sh \ - scripts/fixtures/keycloak-cohort-dispatch-fixture.sh + scripts/fixtures/keycloak-cohort-dispatch-fixture.sh \ +) +docker run --rm --network none \ + --volume "${repo_root}:/workspace:ro" \ + --workdir /workspace \ + "${shellcheck_image}" "${shellcheck_paths[@]}" python3 - <<'PY' import ast from pathlib import Path @@ -44,19 +47,12 @@ from pathlib import Path for source in ( "scripts/verify-brio-release-evidence.py", "scripts/verify-keycloak-cohort-evidence.py", - "scripts/ci-base-image.py", - "scripts/verify-postgres-ci-jit-result.py", "scripts/reconcile-github-environment-main-policy.py", "scripts/test-github-environment-main-policy.py", ): ast.parse(Path(source).read_text(), filename=source) PY PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-github-environment-main-policy.py -node --check scripts/publish-pr-ci-check.mjs -node --check scripts/postgres-ci-queue-controller.mjs -node --check scripts/dispatch-ci-attestation.mjs -node --test scripts/test-pr-ci-check.mjs scripts/test-postgres-ci-queue-controller.mjs -./scripts/test-postgres-ci-jit-result.sh actionlint git show --check --format= HEAD git diff --check diff --git a/scripts/run-postgres-ci-jit-vm.sh b/scripts/run-postgres-ci-jit-vm.sh deleted file mode 100755 index da93d63..0000000 --- a/scripts/run-postgres-ci-jit-vm.sh +++ /dev/null @@ -1,685 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -export LC_ALL=C - -# Trusted-hypervisor launcher for one Postgres PR job. It obtains a GitHub JIT -# configuration, boots a fresh self-contained VM, and destroys the VM, disk, -# registration seed, network, firewall, and runner registration. Only then does -# it sign and dispatch per-run evidence with the dedicated Launcher App. - -readonly organization="Makepad-fr" -readonly runner_group="Postgres PR Ephemeral" -readonly runner_label="makepad-postgres-pr-ephemeral" -readonly api_version="2022-11-28" -script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) -readonly script_directory - -die() { - printf '%s\n' "$*" >&2 - exit 1 -} - -[[ "$(id -u)" -eq 0 ]] || die "the JIT VM launcher must run as root on the dedicated CI hypervisor" -umask 077 -for trusted_helper in ci-base-image.py dispatch-ci-attestation.mjs verify-postgres-ci-jit-result.py; do - trusted_path="${script_directory}/${trusted_helper}" - [[ -f "${trusted_path}" && ! -L "${trusted_path}" && $(stat -c '%u' "${trusted_path}") == 0 ]] || \ - die "trusted launcher helper is missing, symlinked, or not root-owned: ${trusted_helper}" - trusted_mode=$(stat -c '%a' "${trusted_path}") - (( (8#${trusted_mode} & 8#022) == 0 )) || die "trusted launcher helper is writable outside root: ${trusted_helper}" -done - -job_root=${POSTGRES_CI_JOB_ROOT:-/var/lib/makepad/postgres-ci/jobs} -[[ "${job_root}" =~ ^/var/lib/makepad/postgres-ci/[A-Za-z0-9._/-]+$ && "${job_root}" != *..* ]] || die "POSTGRES_CI_JOB_ROOT is unsafe" - -if [[ $# -eq 2 && "$1" == --reconcile ]]; then - launch_id=$2 - [[ "${launch_id}" =~ ^j[1-9][0-9]{0,15}-[a-f0-9]{16}$ ]] || die "reconciliation launch ID is invalid" - for command_name in gh nft sha256sum virsh; do - command -v "${command_name}" >/dev/null || die "${command_name} is required for reconciliation" - done - IFS= read -r controller_token || die "a dedicated Launcher App installation token is required on standard input" - [[ "${controller_token}" =~ ^ghs_[A-Za-z0-9_]+$ ]] || die "the Launcher App token has an invalid format" - resource_hash=$(printf '%s' "${launch_id}" | sha256sum | cut -c1-10) - temporary_directory="${job_root}/postgres-ci-jit-${launch_id}" - vm_name="postgres-ci-${launch_id}" - runner_name="postgres-ci-jit-${launch_id}" - network_name="mdci-${launch_id}" - nft_table="mdci_${resource_hash}" - reconciliation_failed=false - if virsh dominfo "${vm_name}" >/dev/null 2>&1; then - virsh destroy "${vm_name}" >/dev/null 2>&1 || true - virsh undefine "${vm_name}" --nvram >/dev/null 2>&1 || virsh undefine "${vm_name}" >/dev/null 2>&1 || reconciliation_failed=true - fi - virsh dominfo "${vm_name}" >/dev/null 2>&1 && reconciliation_failed=true - if nft list table inet "${nft_table}" >/dev/null 2>&1; then nft delete table inet "${nft_table}" >/dev/null 2>&1 || reconciliation_failed=true; fi - nft list table inet "${nft_table}" >/dev/null 2>&1 && reconciliation_failed=true - if virsh net-info "${network_name}" >/dev/null 2>&1; then - virsh net-destroy "${network_name}" >/dev/null 2>&1 || true - virsh net-undefine "${network_name}" >/dev/null 2>&1 || reconciliation_failed=true - fi - virsh net-info "${network_name}" >/dev/null 2>&1 && reconciliation_failed=true - runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners?per_page=100" \ - --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) || reconciliation_failed=true - while IFS= read -r runner_id; do - [[ -z "${runner_id}" ]] && continue - [[ "${runner_id}" =~ ^[1-9][0-9]*$ ]] && GH_TOKEN="${controller_token}" gh api --method DELETE \ - --header "X-GitHub-Api-Version: ${api_version}" "orgs/${organization}/actions/runners/${runner_id}" >/dev/null 2>&1 \ - || reconciliation_failed=true - done <<<"${runner_ids:-}" - remaining=$(GH_TOKEN="${controller_token}" gh api --paginate \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners?per_page=100" \ - --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) || reconciliation_failed=true - [[ -z "${remaining:-}" ]] || reconciliation_failed=true - if [[ -e "${temporary_directory}" || -L "${temporary_directory}" ]]; then - [[ -d "${temporary_directory}" && ! -L "${temporary_directory}" && "$(stat -c '%u:%a' "${temporary_directory}")" == 0:700 ]] || die "reconciliation work directory is unsafe" - find "${temporary_directory}" -depth -mindepth 1 -delete - rmdir -- "${temporary_directory}" || reconciliation_failed=true - fi - unset controller_token - [[ "${reconciliation_failed}" == false ]] || die "incomplete JIT launch could not be fully reconciled" - printf 'Reconciled incomplete disposable launch %s.\n' "${launch_id}" - exit 0 -fi - -[[ $# -eq 0 ]] || die "usage: set exact POSTGRES_CI_RUN_* metadata and stream a Launcher App installation token on stdin" -[[ "$(uname -m)" == x86_64 ]] || die "the Postgres JIT base image and workflow require an x86_64 hypervisor" -[[ -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] || die "hardware-backed KVM is required for the one-job runner VM" -for command_name in cksum cloud-localds flock gh ip lsattr mktemp nft node python3 qemu-img seq sha256sum virsh virt-install; do - command -v "${command_name}" >/dev/null || die "${command_name} is required" -done - -base_image=${POSTGRES_CI_BASE_IMAGE:-} -expected_image_sha256=${POSTGRES_CI_BASE_IMAGE_SHA256:-} -public_dns=${POSTGRES_CI_PUBLIC_DNS_IPV4:-1.1.1.1} -run_id=${POSTGRES_CI_RUN_ID:-} -run_attempt=${POSTGRES_CI_RUN_ATTEMPT:-} -job_id=${POSTGRES_CI_JOB_ID:-} -run_event=${POSTGRES_CI_RUN_EVENT:-} -head_sha=${POSTGRES_CI_HEAD_SHA:-} -workflow_sha=${POSTGRES_CI_WORKFLOW_SHA:-} -attestation_nonce=${POSTGRES_CI_ATTESTATION_NONCE:-} -launch_id=${POSTGRES_CI_LAUNCH_ID:-} -attestation_private_key=${POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE:-} -result_poll_attempts=${POSTGRES_CI_RESULT_POLL_ATTEMPTS:-24} -result_poll_seconds=${POSTGRES_CI_RESULT_POLL_SECONDS:-5} -[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ && "${job_id}" =~ ^[1-9][0-9]*$ ]] || die "exact positive run, attempt, and job IDs are required" -[[ "${run_event}" == pull_request_target || "${run_event}" == push ]] || die "POSTGRES_CI_RUN_EVENT must be pull_request_target or push" -[[ "${head_sha}" =~ ^[a-f0-9]{40}$ ]] || die "POSTGRES_CI_HEAD_SHA must be the exact lowercase source SHA" -[[ "${workflow_sha}" =~ ^[a-f0-9]{40}$ ]] || die "POSTGRES_CI_WORKFLOW_SHA must be the protected workflow execution SHA" -if [[ "${run_event}" == push && "${head_sha}" != "${workflow_sha}" ]]; then - die "protected-main push source and workflow SHAs must be identical" -fi -[[ "${attestation_nonce}" =~ ^[A-Za-z0-9_-]{43}$ ]] || die "POSTGRES_CI_ATTESTATION_NONCE must be 32 random base64url bytes" -[[ "${launch_id}" =~ ^j[1-9][0-9]{0,15}-[a-f0-9]{16}$ && "${launch_id}" == "j${job_id}-"* ]] || die "POSTGRES_CI_LAUNCH_ID must bind the exact job to a deterministic resource set" -[[ "${result_poll_attempts}" =~ ^[1-9][0-9]*$ && "${result_poll_seconds}" =~ ^[0-9]+$ ]] || die "result polling controls must be non-negative integers" -((result_poll_attempts <= 60 && result_poll_seconds <= 30)) || die "result polling controls exceed the reviewed safety bound" -[[ "${attestation_private_key}" == /* && -f "${attestation_private_key}" && ! -L "${attestation_private_key}" ]] || die "a regular absolute Ed25519 attestation private-key file is required" -[[ "$(stat -c '%u:%a' "${attestation_private_key}")" == "0:400" ]] || die "the attestation private key must be root-owned mode 0400" -[[ "${base_image}" == /* && -f "${base_image}" && ! -L "${base_image}" ]] || die "POSTGRES_CI_BASE_IMAGE must be an absolute regular file" -[[ "${expected_image_sha256}" =~ ^[a-f0-9]{64}$ ]] || die "POSTGRES_CI_BASE_IMAGE_SHA256 must be a lowercase SHA-256 digest" -[[ "$(stat -c '%u' "${base_image}")" == 0 ]] || die "the base image must be owned by root" -base_mode=$(stat -c '%a' "${base_image}") -(( (8#${base_mode} & 8#022) == 0 )) || die "the base image must not be group- or world-writable" -python3 - "${base_image}" <<'PY' -import os -import pathlib -import stat -import sys - -path = pathlib.Path(sys.argv[1]) -for component in [pathlib.Path("/")] + list(reversed(path.parents[:-1])) + [path]: - value = os.lstat(component) - if stat.S_ISLNK(value.st_mode) or value.st_uid != 0 or value.st_mode & 0o022: - raise SystemExit(f"insecure base-image path component: {component}") -PY -attributes=$(lsattr -d -- "${base_image}" 2>/dev/null | awk '{print $1}') -[[ "${attributes}" == *i* ]] || die "the reviewed base image must have the filesystem immutable attribute" -python3 "${script_directory}/ci-base-image.py" "${base_image}" "${expected_image_sha256}" >/dev/null || die "the trusted base-image digest does not match" -qemu-img info --output=json "${base_image}" | python3 -c ' -import json, sys -payload = json.load(sys.stdin) -size = payload.get("virtual-size") -if payload.get("format") != "qcow2" or payload.get("backing-filename") or payload.get("data-file") or not isinstance(size, int) or not 8 * 1024**3 <= size <= 64 * 1024**3: - raise SystemExit("trusted base image must be qcow2 with an 8-64 GiB virtual disk") -' -python3 - "${public_dns}" <<'PY' -import ipaddress -import sys - -address = ipaddress.ip_address(sys.argv[1]) -if address.version != 4 or not address.is_global: - raise SystemExit("POSTGRES_CI_PUBLIC_DNS_IPV4 must be a globally routable IPv4 address") -PY - -IFS= read -r controller_token || die "a dedicated Launcher App installation token is required on standard input" -[[ "${controller_token}" =~ ^ghs_[A-Za-z0-9_]+$ ]] || die "the Launcher App token has an invalid format" - -install -d -m 0700 -o root -g root "${job_root}" -[[ -d "${job_root}" && ! -L "${job_root}" && "$(stat -c '%u:%a' "${job_root}")" == 0:700 ]] -temporary_directory="${job_root}/postgres-ci-jit-${launch_id}" -[[ ! -e "${temporary_directory}" && ! -L "${temporary_directory}" ]] || die "deterministic JIT resource directory already exists" -install -d -m 0700 -o root -g root "${temporary_directory}" -resource_hash=$(printf '%s' "${launch_id}" | sha256sum | cut -c1-10) -vm_name="postgres-ci-${launch_id}" -runner_name="postgres-ci-jit-${launch_id}" -network_name="mdci-${launch_id}" -bridge_name="md${resource_hash}" -nft_table="mdci_${resource_hash}" -overlay_path="${temporary_directory}/runner.qcow2" -seed_path="${temporary_directory}/seed.iso" -network_xml="${temporary_directory}/network.xml" -user_data="${temporary_directory}/user-data" -meta_data="${temporary_directory}/meta-data" -network_started=false -domain_defined=false -nft_created=false -jit_runner_id="" -attestation_eligible=false - -write_resource_manifest() { - local registered_id=${1:-} incoming="${temporary_directory}/.resources.json.incoming" - python3 - "${incoming}" "${launch_id}" "${job_id}" "${vm_name}" "${runner_name}" "${network_name}" "${bridge_name}" "${nft_table}" "${registered_id}" <<'PY' -import json, pathlib, sys -runner_id = int(sys.argv[9]) if sys.argv[9] else None -payload = {"version":1,"launch_id":sys.argv[2],"job_id":int(sys.argv[3]),"vm":sys.argv[4],"runner":sys.argv[5],"network":sys.argv[6],"bridge":sys.argv[7],"nft_table":sys.argv[8],"runner_id":runner_id} -pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n") -PY - chmod 0600 "${incoming}" - mv -fT "${incoming}" "${temporary_directory}/resources.json" - sync -f "${temporary_directory}" 2>/dev/null || sync -} -write_resource_manifest - -cleanup() { - local original_status=$? - local cleanup_failed=false - local retain_vm_files=false - local job_conclusion="" - trap - EXIT INT TERM HUP - unset encoded_jit_config - set +e - if [[ "${domain_defined}" == true ]]; then - virsh destroy "${vm_name}" >/dev/null 2>&1 - virsh undefine "${vm_name}" --nvram >/dev/null 2>&1 || virsh undefine "${vm_name}" >/dev/null 2>&1 - if virsh dominfo "${vm_name}" >/dev/null 2>&1; then - printf 'Failed to destroy and undefine ephemeral runner VM %s.\n' "${vm_name}" >&2 - cleanup_failed=true - retain_vm_files=true - fi - fi - if [[ "${nft_created}" == true ]]; then - nft delete table inet "${nft_table}" >/dev/null 2>&1 - if nft list table inet "${nft_table}" >/dev/null 2>&1; then - printf 'Failed to remove ephemeral runner firewall table %s.\n' "${nft_table}" >&2 - cleanup_failed=true - fi - fi - if [[ "${network_started}" == true ]]; then - virsh net-destroy "${network_name}" >/dev/null 2>&1 - virsh net-undefine "${network_name}" >/dev/null 2>&1 - if virsh net-info "${network_name}" >/dev/null 2>&1; then - printf 'Failed to remove ephemeral runner network %s.\n' "${network_name}" >&2 - cleanup_failed=true - fi - fi - if [[ -n "${controller_token:-}" && -n "${runner_name:-}" ]]; then - runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners?per_page=100" \ - --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) - lookup_status=$? - if [[ "${lookup_status}" -ne 0 ]]; then - printf 'Failed to inspect the JIT runner registration during teardown.\n' >&2 - cleanup_failed=true - else - while IFS= read -r runner_id; do - [[ -z "${runner_id}" ]] && continue - if [[ ! "${runner_id}" =~ ^[1-9][0-9]*$ || ( -n "${jit_runner_id}" && "${runner_id}" != "${jit_runner_id}" ) ]] || \ - ! GH_TOKEN="${controller_token}" gh api --method DELETE \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners/${runner_id}" >/dev/null 2>&1; then - printf 'Failed to remove JIT runner registration %s.\n' "${runner_id}" >&2 - cleanup_failed=true - fi - done <<<"${runner_ids}" - remaining_runner_ids=$(GH_TOKEN="${controller_token}" gh api --paginate \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners?per_page=100" \ - --jq ".runners[] | select(.name == \"${runner_name}\") | .id" 2>/dev/null) - remaining_status=$? - if [[ "${remaining_status}" -ne 0 || -n "${remaining_runner_ids}" ]]; then - printf 'JIT runner registration removal could not be verified.\n' >&2 - cleanup_failed=true - fi - fi - fi - if [[ "${retain_vm_files}" == false ]]; then - if [[ -d "${temporary_directory}" && ! -L "${temporary_directory}" && "${temporary_directory}" == "${job_root}"/postgres-ci-jit-* ]]; then - find "${temporary_directory}" -depth -mindepth 1 -delete - rmdir -- "${temporary_directory}" - if [[ -e "${temporary_directory}" || -L "${temporary_directory}" ]]; then - cleanup_failed=true - fi - else - cleanup_failed=true - fi - fi - if [[ "${cleanup_failed}" == true ]]; then - printf 'Ephemeral CI teardown is incomplete; inspect the hypervisor alert immediately.\n' >&2 - if [[ "${retain_vm_files}" == true ]]; then - printf 'VM files are quarantined at %s until the domain is destroyed.\n' "${temporary_directory}" >&2 - fi - original_status=1 - fi - if [[ "${attestation_eligible}" == true && "${cleanup_failed}" == false ]]; then - # The VM has stopped and every hypervisor resource plus GitHub registration - # has been independently shown absent. Now bind the authoritative job result - # to that teardown before the hypervisor-only Ed25519 key signs anything. - run_payload_file=$(mktemp /run/postgres-ci-run-XXXXXXXX.json) - jobs_payload_file=$(mktemp /run/postgres-ci-jobs-XXXXXXXX.json) - chmod 0600 "${run_payload_file}" "${jobs_payload_file}" - completed_payload=false - for poll_attempt in $(seq 1 "${result_poll_attempts}"); do - if GH_TOKEN="${controller_token}" gh api \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "repos/${organization}/postgres/actions/runs/${run_id}" >"${run_payload_file}" 2>/dev/null && \ - GH_TOKEN="${controller_token}" gh api \ - --header "X-GitHub-Api-Version: ${api_version}" \ - "repos/${organization}/postgres/actions/runs/${run_id}/attempts/${run_attempt}/jobs?per_page=100" \ - >"${jobs_payload_file}" 2>/dev/null && \ - python3 - "${run_payload_file}" "${jobs_payload_file}" "${job_id}" <<'PY' -import json, pathlib, sys -run=json.loads(pathlib.Path(sys.argv[1]).read_text()); response=json.loads(pathlib.Path(sys.argv[2]).read_text()); jobs=response.get("jobs", []) -if not isinstance(jobs, list) or response.get("total_count") != len(jobs): raise SystemExit(1) -matches=[job for job in jobs if job.get("id") == int(sys.argv[3])] -raise SystemExit(0 if run.get("status") == "completed" and len(matches) == 1 and matches[0].get("status") == "completed" else 1) -PY - then - completed_payload=true - break - fi - if ((poll_attempt < result_poll_attempts && result_poll_seconds > 0)); then sleep "${result_poll_seconds}"; fi - done - if [[ "${completed_payload}" != true ]]; then - printf 'Authoritative workflow state did not converge before the bounded attestation deadline.\n' >&2 - cleanup_failed=true - original_status=1 - fi - if [[ "${cleanup_failed}" == false ]]; then - job_conclusion=$(python3 "${script_directory}/verify-postgres-ci-jit-result.py" \ - "${run_payload_file}" "${jobs_payload_file}" "${run_id}" "${run_attempt}" \ - "${job_id}" "${run_event}" "${head_sha}" "${workflow_sha}" "${jit_runner_id}" \ - "${runner_name}" "${runner_group_id}") - fi - rm -f -- "${run_payload_file}" "${jobs_payload_file}" || { - cleanup_failed=true - original_status=1 - } - if [[ "${job_conclusion}" != success && "${job_conclusion}" != failure ]]; then - printf 'Unable to bind authoritative job conclusion to teardown.\n' >&2 - cleanup_failed=true - original_status=1 - else - attestation_file=$(mktemp /run/postgres-ci-attestation-XXXXXXXX.json) - chmod 0600 "${attestation_file}" - python3 - "${attestation_file}" "${run_id}" "${run_attempt}" "${job_id}" "${run_event}" \ - "${head_sha}" "${workflow_sha}" "${job_conclusion}" "${jit_runner_id}" "${runner_name}" \ - "${runner_group_id}" "${runner_group}" "${runner_label}" "${expected_image_sha256}" \ - "${attestation_nonce}" <<'PY' -import datetime -import json -import pathlib -import sys - -payload = { - "base_image_sha256": sys.argv[14], - "issued_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), - "nonce": sys.argv[15], - "ref": "refs/heads/main", - "registration_absent": True, - "repository": "Makepad-fr/postgres", - "run": { - "attempt": int(sys.argv[3]), - "conclusion": sys.argv[8], - "event": sys.argv[5], - "head_sha": sys.argv[6], - "id": int(sys.argv[2]), - "job_id": int(sys.argv[4]), - "job_name": "policy-and-integration", - "workflow_sha": sys.argv[7], - }, - "runner": { - "group_id": int(sys.argv[11]), - "group_name": sys.argv[12], - "id": int(sys.argv[9]), - "labels": ["self-hosted", "linux", "x64", sys.argv[13]], - "name": sys.argv[10], - }, - "schema": "makepad.postgres.ci-attestation.v1", - "teardown": {"disk": True, "firewall": True, "network": True, "vm": True}, - "workflow": {"name": "CI", "path": ".github/workflows/ci.yml"}, -} -pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) -PY - if ! printf '%s\n' "${controller_token}" | \ - POSTGRES_CI_ATTESTATION_JSON_FILE="${attestation_file}" \ - POSTGRES_CI_ATTESTATION_PRIVATE_KEY_FILE="${attestation_private_key}" \ - node "${script_directory}/dispatch-ci-attestation.mjs"; then - printf 'Signed teardown evidence could not be dispatched by the Launcher App.\n' >&2 - cleanup_failed=true - original_status=1 - fi - rm -f -- "${attestation_file}" || { - printf 'Could not remove transient attestation material.\n' >&2 - cleanup_failed=true - original_status=1 - } - fi - fi - unset controller_token - if [[ "${cleanup_failed}" == true ]]; then - printf 'The queue supervisor must raise an independent launcher failure alert.\n' >&2 - fi - exit "${original_status}" -} -trap cleanup EXIT -trap 'exit 129' HUP -trap 'exit 130' INT -trap 'exit 143' TERM - -group_payload="${temporary_directory}/runner-groups.json" -GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups?per_page=100" >"${group_payload}" -runner_group_id=$(python3 - "${group_payload}" "${runner_group}" <<'PY' -import json -import pathlib -import sys - -payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) -groups = payload.get("runner_groups", []) -if payload.get("total_count", len(groups)) > len(groups): - raise SystemExit("more than 100 runner groups require explicit pagination") -matches = [item for item in groups if item.get("name") == sys.argv[2]] -if len(matches) != 1: - raise SystemExit("the exact Postgres PR Ephemeral runner group does not exist uniquely") -print(matches[0]["id"]) -PY -) -[[ "${runner_group_id}" =~ ^[1-9][0-9]*$ ]] || die "runner group ID is invalid" - -group_details="${temporary_directory}/runner-group.json" -group_repositories="${temporary_directory}/runner-group-repositories.json" -repository_payload="${temporary_directory}/repository.json" -GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${runner_group_id}" >"${group_details}" -GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runner-groups/${runner_group_id}/repositories?per_page=100" >"${group_repositories}" -GH_TOKEN="${controller_token}" gh api --header "X-GitHub-Api-Version: ${api_version}" \ - "repos/${organization}/postgres" >"${repository_payload}" -python3 - "${group_details}" "${group_repositories}" "${repository_payload}" <<'PY' -import json -import pathlib -import sys - -group = json.loads(pathlib.Path(sys.argv[1]).read_text()) -repository_selection = json.loads(pathlib.Path(sys.argv[2]).read_text()) -repository = json.loads(pathlib.Path(sys.argv[3]).read_text()) -selected = repository_selection.get("repositories", []) -expected_workflows = { - "Makepad-fr/postgres/.github/workflows/ci.yml@refs/heads/main", - "Makepad-fr/postgres/.github/workflows/pr-ci-result.yml@refs/heads/main", -} -if ( - group.get("name") != "Postgres PR Ephemeral" - or group.get("visibility") != "selected" - or group.get("allows_public_repositories") is not True - or group.get("restricted_to_workflows") is not True - or group.get("workflow_restrictions_read_only") is not False - or set(group.get("selected_workflows", [])) != expected_workflows -): - raise SystemExit("Postgres PR Ephemeral runner group is not restricted to the exact protected workflows") -if repository_selection.get("total_count", len(selected)) > len(selected): - raise SystemExit("runner-group repository selection is truncated") -if ( - repository.get("full_name") != "Makepad-fr/postgres" - or repository.get("private") is not False - or not isinstance(repository.get("id"), int) - or [item.get("id") for item in selected] != [repository["id"]] -): - raise SystemExit("Postgres PR Ephemeral runner group is not restricted to the public PostgreSQL repository") -PY - -jit_request="${temporary_directory}/jit-request.json" -jit_response="${temporary_directory}/jit-response.json" -python3 - "${runner_name}" "${runner_group_id}" "${runner_label}" >"${jit_request}" <<'PY' -import json -import sys - -print(json.dumps({ - "name": sys.argv[1], - "runner_group_id": int(sys.argv[2]), - "work_folder": "_work", - "labels": ["self-hosted", "Linux", "X64", sys.argv[3]], -}, separators=(",", ":"))) -PY -chmod 0600 "${jit_request}" -GH_TOKEN="${controller_token}" gh api --method POST --header "X-GitHub-Api-Version: ${api_version}" \ - "orgs/${organization}/actions/runners/generate-jitconfig" \ - --input "${jit_request}" >"${jit_response}" -jit_runner_id=$(python3 - "${jit_response}" "${runner_name}" "${runner_label}" <<'PY' -import json -import pathlib -import sys - -payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) -runner = payload.get("runner", {}) -runner_id = runner.get("id") -labels = { - str(item.get("name", "")).lower() - for item in runner.get("labels", []) - if isinstance(item, dict) -} -if ( - not isinstance(runner_id, int) - or runner_id <= 0 - or runner.get("name") != sys.argv[2] - or runner.get("status") != "offline" - or not {"self-hosted", "linux", "x64", sys.argv[3]}.issubset(labels) -): - raise SystemExit("GitHub returned an invalid JIT runner identity") -print(runner_id) -PY -) -write_resource_manifest "${jit_runner_id}" -encoded_jit_config=$(python3 - "${jit_response}" <<'PY' -import json -import pathlib -import re -import sys - -value = json.loads(pathlib.Path(sys.argv[1]).read_text()).get("encoded_jit_config", "") -if not re.fullmatch(r"[A-Za-z0-9_+/\-]{40,8192}={0,2}", value): - raise SystemExit("GitHub returned an invalid JIT configuration") -print(value) -PY -) - -# A fresh libvirt network is created for every VM. Its host-side nftables hook -# blocks private, WireGuard, link-local/metadata, multicast, every hypervisor -# address, and all egress except public DNS plus TLS. Guest root cannot remove -# these hypervisor rules. -exec 8>/run/lock/postgres-ci-network.lock -flock -x 8 -suffix_checksum=$(printf '%s' "${launch_id}" | cksum) -suffix_checksum=${suffix_checksum%% *} -subnet="" -for offset in $(seq 0 199); do - network_octet=$(((suffix_checksum + offset) % 200 + 20)) - candidate="172.31.${network_octet}" - if [[ -z "$(ip -4 route show exact "${candidate}.0/24")" ]]; then - subnet="${candidate}" - break - fi -done -[[ -n "${subnet}" ]] || die "no unused ephemeral runner subnet is available" -cat >"${network_xml}" < - ${network_name} - - - - - - - -EOF -chmod 0600 "${network_xml}" -virsh net-define "${network_xml}" >/dev/null -network_started=true -virsh net-start "${network_name}" >/dev/null -flock -u 8 - -nft_created=true -nft -f - </dev/null || die "base image mutated while the self-contained job disk was created" - -guest_script=$(cat <<'GUEST' -#!/usr/bin/env bash -set -euo pipefail -power_off() { - local status=$? - trap - EXIT - find /run/postgres-jit -depth -mindepth 1 -delete 2>/dev/null || true - rmdir /run/postgres-jit 2>/dev/null || true - systemctl poweroff --no-block - exit "${status}" -} -trap power_off EXIT -[[ -x /opt/actions-runner/run.sh ]] -[[ -f /run/postgres-jit/config && ! -L /run/postgres-jit/config ]] -jit_config=$("${meta_data}" -chmod 0600 "${user_data}" "${meta_data}" -cloud-localds "${seed_path}" "${user_data}" "${meta_data}" -chmod 0600 "${seed_path}" - -domain_defined=true -virt-install \ - --name "${vm_name}" \ - --virt-type kvm \ - --memory 6144 \ - --vcpus 4 \ - --import \ - --osinfo detect=on,require=off \ - --disk "path=${overlay_path},format=qcow2,bus=virtio,cache=none" \ - --disk "path=${seed_path},device=cdrom,readonly=on" \ - --network "network=${network_name},model=virtio" \ - --graphics none \ - --noautoconsole >/dev/null - -deadline=$((SECONDS + 2700)) -while (( SECONDS < deadline )); do - state=$(virsh domstate "${vm_name}" 2>/dev/null || true) - case "${state}" in - "shut off"|"crashed") break ;; - esac - sleep 5 -done -state=$(virsh domstate "${vm_name}" 2>/dev/null || true) -[[ "${state}" == "shut off" ]] || die "the one-job runner did not shut down within 45 minutes" -attestation_eligible=true -printf 'One-job JIT VM %s stopped; destroying all ephemeral state.\n' "${vm_name}" diff --git a/scripts/run-postgres-ci-queue-controller.sh b/scripts/run-postgres-ci-queue-controller.sh deleted file mode 100755 index 3aa0a2a..0000000 --- a/scripts/run-postgres-ci-queue-controller.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -[[ "$(id -u)" -eq 0 ]] || { echo "controller supervisor must run as root" >&2; exit 1; } -exec 9>/run/lock/postgres-ci-queue-controller.lock -flock -n 9 || { echo "another Postgres queue controller owns the hypervisor" >&2; exit 1; } -script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) -exec node "${script_directory}/postgres-ci-queue-controller.mjs" "$@" diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh index fe689de..61cce60 100755 --- a/scripts/test-brio-deployment-contracts.sh +++ b/scripts/test-brio-deployment-contracts.sh @@ -13,10 +13,6 @@ manual = (root / ".github/workflows/manual-deploy.yml").read_text() identity_workflow = (root / ".github/workflows/deploy-brio-identity-db.yml").read_text() release_workflow = (root / ".github/workflows/release-brio-identity-db.yml").read_text() ci_workflow = (root / ".github/workflows/ci.yml").read_text() -finalizer_workflow = (root / ".github/workflows/pr-ci-result.yml").read_text() -check_publisher = (root / "scripts/publish-pr-ci-check.mjs").read_text() -jit_launcher = (root / "scripts/run-postgres-ci-jit-vm.sh").read_text() -queue_controller = (root / "scripts/postgres-ci-queue-controller.mjs").read_text() cohort_workflow = (root / ".github/workflows/verify-keycloak-cohort-restores.yml").read_text() cohort_validator = (root / "scripts/verify-keycloak-cohort-evidence.py").read_text() identity = (root / "scripts/deploy-brio-identity-db-host.sh").read_text() @@ -95,25 +91,10 @@ for marker in ( ): require(marker in release_workflow, f"protected release orchestrator missing: {marker}") require("actions/upload-artifact@" not in release_workflow, "release orchestrator must not synthesize or republish attestation") -require("pull_request_target:" in ci_workflow, "PR CI must use protected-base workflow code") +require("pull_request:" in ci_workflow and "pull_request_target:" not in ci_workflow, "PR CI must use the native pull-request event") require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks") require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact head") -require("repository_dispatch:" in finalizer_workflow and "types: [postgres-pr-ci-attestation]" in finalizer_workflow and "environment: postgres-ci-attestation" in finalizer_workflow, "PR CI result must require signed hypervisor teardown") -require("POSTGRES_PR_CHECK_APP_PRIVATE_KEY" in finalizer_workflow and 'CHECK_NAMES = ["postgres-ci"]' in check_publisher, "required PR check must be App-bound") -for marker in ( - "makepad.postgres.ci-attestation.v1", - "verifySignature", - "registration_absent", - "runnerLookupStatus !== 404", - "makepad-postgres-pr-ephemeral", -): - require(marker in check_publisher + jit_launcher, f"signed disposable PR boundary missing: {marker}") -for marker in ("generate-jitconfig", "--jitconfig", "virsh undefine", "nft delete table", "dispatch-ci-attestation.mjs", "resources.json", "--reconcile", "POSTGRES_CI_RESULT_POLL_ATTEMPTS"): - require(marker in jit_launcher, f"JIT hypervisor teardown contract missing: {marker}") -require('job.name === "policy-and-integration"' in queue_controller and "await runLauncher" in queue_controller, "queue controller must bind and supervise the exact disposable PR job") -require("await reconcileIncompleteJobs" in queue_controller and "launchID" in queue_controller, "queue controller must reconcile deterministic incomplete launches before polling") -require('association.base?.sha !== run.head_sha' in queue_controller, "queue controller must bind the exact PR base SHA") -require('association.base?.sha !== attestation.run.workflow_sha' in check_publisher, "attestor must bind the exact PR base SHA") +require(ci_workflow.count("runs-on: [self-hosted, linux, x64, makepad]") == 2, "PR and protected-main CI must use the existing Makepad Linux runner") for marker in ( "name: Verify Keycloak Cohort Restore Compatibility", "keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }}", diff --git a/scripts/test-postgres-ci-jit-result.sh b/scripts/test-postgres-ci-jit-result.sh deleted file mode 100755 index c5f6d50..0000000 --- a/scripts/test-postgres-ci-jit-result.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -work_dir=$(mktemp -d) -cleanup() { - find "${work_dir}" -mindepth 1 -delete - rmdir "${work_dir}" -} -trap cleanup EXIT - -source_sha=$(printf 'a%.0s' {1..40}) -workflow_sha=$(printf 'b%.0s' {1..40}) -run_file="${work_dir}/run.json" -jobs_file="${work_dir}/jobs.json" -python3 - "${run_file}" "${jobs_file}" "${source_sha}" "${workflow_sha}" <<'PY' -import json -import pathlib -import sys - -run = { - "id": 101, - "run_attempt": 2, - "event": "pull_request_target", - "head_sha": sys.argv[4], - "head_branch": "main", - "name": "CI", - "path": ".github/workflows/ci.yml", - "status": "completed", - "conclusion": "success", - "repository": {"id": 77, "full_name": "Makepad-fr/postgres"}, - "pull_requests": [{ - "number": 9, - "head": {"sha": sys.argv[3], "repo": {"id": 77}}, - "base": {"ref": "main", "sha": sys.argv[4], "repo": {"id": 77}}, - }], -} -job = { - "id": 202, - "run_id": 101, - "head_sha": sys.argv[4], - "workflow_name": "CI", - "runner_id": 303, - "runner_name": "postgres-ci-jit-j202-1111111111111111", - "runner_group_id": 404, - "runner_group_name": "Postgres PR Ephemeral", - "name": "policy-and-integration", - "status": "completed", - "conclusion": "success", - "labels": ["self-hosted", "Linux", "X64", "makepad-postgres-pr-ephemeral"], -} -pathlib.Path(sys.argv[1]).write_text(json.dumps(run)) -pathlib.Path(sys.argv[2]).write_text(json.dumps({"total_count": 1, "jobs": [job]})) -PY - -result=$(python3 "${script_dir}/verify-postgres-ci-jit-result.py" \ - "${run_file}" "${jobs_file}" 101 2 202 pull_request_target \ - "${source_sha}" "${workflow_sha}" 303 postgres-ci-jit-j202-1111111111111111 404) -[[ "${result}" == success ]] - -python3 - "${run_file}" <<'PY' -import json -import pathlib -import sys -path = pathlib.Path(sys.argv[1]) -value = json.loads(path.read_text()) -value["pull_requests"][0]["base"]["sha"] = "c" * 40 -path.write_text(json.dumps(value)) -PY -if python3 "${script_dir}/verify-postgres-ci-jit-result.py" \ - "${run_file}" "${jobs_file}" 101 2 202 pull_request_target \ - "${source_sha}" "${workflow_sha}" 303 postgres-ci-jit-j202-1111111111111111 404 >/dev/null 2>&1; then - echo "JIT result verifier accepted a PR association with the wrong base SHA." >&2 - exit 1 -fi - -echo "Postgres JIT authoritative-result tests passed." diff --git a/scripts/test-postgres-ci-queue-controller.mjs b/scripts/test-postgres-ci-queue-controller.mjs deleted file mode 100644 index 97f2500..0000000 --- a/scripts/test-postgres-ci-queue-controller.mjs +++ /dev/null @@ -1,126 +0,0 @@ -import assert from "node:assert/strict"; -import {readFile} from "node:fs/promises"; -import path from "node:path"; -import test from "node:test"; -import {fileURLToPath, pathToFileURL} from "node:url"; - -const candidateRoot = path.resolve(process.env.POSTGRES_CANDIDATE_ROOT || fileURLToPath(new URL("..", import.meta.url))); -const controllerURL = pathToFileURL(path.join(candidateRoot, "scripts/postgres-ci-queue-controller.mjs")); -const {reconcileIncompleteJobs, selectAuthorizedJobs} = await import(controllerURL.href); -const launcherURL = pathToFileURL(path.join(candidateRoot, "scripts/run-postgres-ci-jit-vm.sh")); - -const repositoryID = 77; -const prBase = () => { - const association = {number: 9, head: {sha: "a".repeat(40), repo: {id: repositoryID}}, base: {ref: "main", sha: "b".repeat(40), repo: {id: repositoryID}}}; - const run = {id: 101, run_attempt: 2, name: "CI", path: ".github/workflows/ci.yml", event: "pull_request_target", status: "queued", head_sha: "b".repeat(40), repository: {id: repositoryID}, pull_requests: [association]}; - const job = {id: 202, run_id: 101, head_sha: "b".repeat(40), workflow_name: "CI", name: "policy-and-integration", status: "queued", labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"]}; - return { - runs: {total_count: 1, workflow_runs: [run]}, - jobsByRun: new Map([["101:2", {total_count: 1, jobs: [job]}]]), - pullRequests: new Map([[9, {number: 9, head: {sha: "a".repeat(40), repo: {id: repositoryID}}, base: {ref: "main", sha: "b".repeat(40), repo: {id: repositoryID}}}]]), - }; -}; - -test("selects the exact queued protected-base same-repository PR job without relying on a nonexistent job attempt field", () => { - assert.deepEqual(selectAuthorizedJobs({...prBase(), repositoryID}), [{runID: 101, attempt: 2, jobID: 202, event: "pull_request_target", sourceSHA: "a".repeat(40), workflowSHA: "b".repeat(40), pullRequestNumber: 9}]); -}); - -test("selects an exact protected-main push job so release CI cannot remain queued", () => { - const value = prBase(); - const run = value.runs.workflow_runs[0]; - run.event = "push"; - run.head_branch = "main"; - run.pull_requests = []; - value.pullRequests.clear(); - assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), [{runID: 101, attempt: 2, jobID: 202, event: "push", sourceSHA: "b".repeat(40), workflowSHA: "b".repeat(40), pullRequestNumber: null}]); -}); - -test("rejects fork, wrong workflow, extra-label, moved-head, and non-queued jobs", () => { - for (const mutate of [ - (value) => { value.runs.workflow_runs[0].pull_requests[0].head.repo.id = 999; }, - (value) => { value.runs.workflow_runs[0].path = ".github/workflows/evil.yml"; }, - (value) => { value.jobsByRun.get("101:2").jobs[0].labels.push("persistent"); }, - (value) => { value.pullRequests.get(9).head.sha = "b".repeat(40); }, - (value) => { value.runs.workflow_runs[0].pull_requests[0].base.sha = "d".repeat(40); }, - (value) => { value.jobsByRun.get("101:2").jobs[0].status = "in_progress"; }, - ]) { - const value = prBase(); - mutate(value); - assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), []); - } -}); - -test("rejects a non-main push, mismatched job head, and wrong job workflow", () => { - for (const mutate of [ - (value) => { value.runs.workflow_runs[0].head_branch = "feature"; }, - (value) => { value.jobsByRun.get("101:2").jobs[0].head_sha = "c".repeat(40); }, - (value) => { value.jobsByRun.get("101:2").jobs[0].workflow_name = "Other"; }, - ]) { - const value = prBase(); - value.runs.workflow_runs[0].event = "push"; - value.runs.workflow_runs[0].head_branch = "main"; - value.runs.workflow_runs[0].pull_requests = []; - mutate(value); - assert.deepEqual(selectAuthorizedJobs({...value, repositoryID}), []); - } -}); - -test("the durable controller records deterministic resources before launch and never selects recorded IDs again", async () => { - const source = await readFile(controllerURL, "utf8"); - assert.match(source, /state\.jobs\[String\(job\.jobID\)\] = \{\.\.\.job, nonce, launchID, status: "launching"/); - assert.match(source, /filter\(\(job\) => !state\.jobs\[String\(job\.jobID\)\]\)/); - assert.match(source, /await runLauncher/); - assert.match(source, /issues/); - const launcherFailure = source.indexOf("// Any nonzero launcher exit is cleanup-uncertain."); - const failed = source.indexOf('status = "recovery-required"', launcherFailure); - const persisted = source.indexOf("await atomicState(stateFile, state);", failed); - const issue = source.indexOf("/issues", failed); - const rethrow = source.indexOf("throw error;", failed); - assert.ok(launcherFailure > 0 && launcherFailure < failed && failed < persisted && persisted < issue && issue < rethrow); - assert.match(source, /systemd OnFailure webhook remains independent of GitHub/); - assert.match(source, /pull_requests: "read"/); - assert.match(source, /await reconcileIncompleteJobs/); - assert.doesNotMatch(source, /status = "failed"/); -}); - -test("startup reconciliation marks every incomplete launch failed-recovered and persists each transition", async () => { - const state = {version: 2, jobs: { - "202": {status: "launching", launchID: "j202-1111111111111111"}, - "203": {status: "recovery-required", launchID: "j203-2222222222222222"}, - "204": {status: "completed", launchID: "j204-3333333333333333"}, - }}; - const reconciled = []; - let persisted = 0; - await reconcileIncompleteJobs({state, persist: async () => { persisted += 1; }, reconcile: async (record) => { reconciled.push(record.launchID); }}); - assert.deepEqual(reconciled, ["j202-1111111111111111", "j203-2222222222222222"]); - assert.equal(persisted, 2); - assert.equal(state.jobs["202"].status, "failed-recovered"); - assert.equal(state.jobs["203"].status, "failed-recovered"); -}); - -test("failed startup reconciliation stays recovery-required and blocks polling", async () => { - const state = {version: 2, jobs: {"202": {status: "launching", launchID: "j202-1111111111111111"}}}; - await assert.rejects(reconcileIncompleteJobs({state, persist: async () => {}, reconcile: async () => { throw new Error("still present"); }}), /still present/); - assert.equal(state.jobs["202"].status, "recovery-required"); -}); - -test("the hypervisor signs only after every disposable resource and registration is proven absent", async () => { - const source = await readFile(launcherURL, "utf8"); - const vmRemoval = source.indexOf('virsh undefine "${vm_name}"'); - const firewallRemoval = source.indexOf('nft delete table inet "${nft_table}"'); - const networkRemoval = source.indexOf('virsh net-undefine "${network_name}"'); - const registrationRemoval = source.indexOf('actions/runners/${runner_id}'); - const absenceCheck = source.indexOf('remaining_runner_ids='); - const teardownGate = source.indexOf('"${attestation_eligible}" == true && "${cleanup_failed}" == false'); - const signingDispatch = source.indexOf('node "${script_directory}/dispatch-ci-attestation.mjs"', teardownGate); - assert.ok(vmRemoval > 0 && firewallRemoval > vmRemoval && networkRemoval > firewallRemoval); - assert.ok(registrationRemoval > 0 && absenceCheck > registrationRemoval); - assert.ok(teardownGate > absenceCheck && signingDispatch > teardownGate); - assert.equal((source.match(/generate-jitconfig/g) || []).length, 2); // API endpoint and explanatory comment. - assert.match(source, /run\.sh --jitconfig/); - assert.match(source, /resources\.json/); - assert.match(source, /--reconcile/); - assert.match(source, /POSTGRES_CI_RESULT_POLL_ATTEMPTS/); - assert.match(source, /repository\.get\("private"\) is not False/); - assert.match(source, /group\.get\("allows_public_repositories"\) is not True/); -}); diff --git a/scripts/test-pr-ci-check.mjs b/scripts/test-pr-ci-check.mjs deleted file mode 100644 index ee8d275..0000000 --- a/scripts/test-pr-ci-check.mjs +++ /dev/null @@ -1,152 +0,0 @@ -import assert from "node:assert/strict"; -import { generateKeyPairSync, sign } from "node:crypto"; -import path from "node:path"; -import test from "node:test"; -import {fileURLToPath, pathToFileURL} from "node:url"; - -const candidateRoot = path.resolve(process.env.POSTGRES_CANDIDATE_ROOT || fileURLToPath(new URL("..", import.meta.url))); -const { - assertNoAttestationReplay, - canonicalJSON, - validateAuthoritativeEvidence, - verifySignedAttestation, -} = await import(pathToFileURL(path.join(candidateRoot, "scripts/publish-pr-ci-check.mjs")).href); - -const now = new Date("2026-09-05T10:00:00Z"); -const digest = "a".repeat(64); -const {privateKey, publicKey} = generateKeyPairSync("ed25519"); - -const baseAttestation = () => ({ - base_image_sha256: digest, - issued_at: now.toISOString().replace(".000Z", "Z"), - nonce: "A".repeat(43), - ref: "refs/heads/main", - registration_absent: true, - repository: "Makepad-fr/postgres", - run: {attempt: 2, conclusion: "success", event: "pull_request_target", head_sha: "b".repeat(40), workflow_sha: "c".repeat(40), id: 1234, job_id: 5678, job_name: "policy-and-integration"}, - runner: {group_id: 12, group_name: "Postgres PR Ephemeral", id: 44, labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"], name: "postgres-ci-jit-20260905100000-deadbeef"}, - schema: "makepad.postgres.ci-attestation.v1", - teardown: {disk: true, firewall: true, network: true, vm: true}, - workflow: {name: "CI", path: ".github/workflows/ci.yml"}, -}); - -const signedEvent = (attestation = baseAttestation(), senderID = 9001) => ({ - action: "postgres-pr-ci-attestation", - repository: {full_name: "Makepad-fr/postgres"}, - sender: {id: senderID, type: "Bot"}, - client_payload: { - attestation, - signature: sign(null, Buffer.from(canonicalJSON(attestation)), privateKey).toString("base64url"), - }, -}); - -const verify = (event, overrides = {}) => verifySignedAttestation({event, publicKey, approvedDigest: digest, launcherSenderID: "9001", now, ...overrides}); - -const authoritative = (attestation = baseAttestation()) => { - const job = { - id: 5678, - run_id: 1234, - head_sha: "c".repeat(40), - workflow_name: "CI", - name: "policy-and-integration", - status: "completed", - conclusion: attestation.run.conclusion, - runner_id: 44, - runner_name: "postgres-ci-jit-20260905100000-deadbeef", - runner_group_id: 12, - runner_group_name: "Postgres PR Ephemeral", - labels: ["self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"], - }; - const association = {number: 7, head: {sha: "b".repeat(40), repo: {id: 88}}, base: {ref: "main", sha: "c".repeat(40), repo: {id: 88}}}; - return { - attestation, - run: {id: 1234, run_attempt: 2, event: attestation.run.event, head_sha: attestation.run.workflow_sha, head_branch: "main", path: ".github/workflows/ci.yml", name: "CI", status: "completed", conclusion: attestation.run.conclusion, repository: {id: 88, full_name: "Makepad-fr/postgres"}, pull_requests: [association], html_url: "https://github.example/run/1234"}, - jobs: {total_count: 1, jobs: [job]}, - job, - pullRequest: {number: 7, head: {sha: "b".repeat(40), repo: {full_name: "Makepad-fr/postgres"}}, base: {ref: "main", sha: "c".repeat(40), repo: {full_name: "Makepad-fr/postgres"}}}, - runnerLookupStatus: 404, - }; -}; - -test("accepts fresh hypervisor-signed teardown evidence from immutable Launcher App sender", () => { - assert.equal(verify(signedEvent()).run.job_id, 5678); -}); - -test("rejects forged evidence", () => { - const event = signedEvent(); - event.client_payload.attestation.run.head_sha = "c".repeat(40); - assert.throws(() => verify(event), /signature verification failed/); -}); - -test("rejects stale evidence", () => { - const attestation = baseAttestation(); - attestation.issued_at = "2026-09-05T09:40:00Z"; - assert.throws(() => verify(signedEvent(attestation)), /stale or from the future/); -}); - -test("rejects an unapproved base image digest", () => { - assert.throws(() => verify(signedEvent(), {approvedDigest: "c".repeat(64)}), /not approved/); -}); - -test("rejects incomplete hypervisor teardown", () => { - const attestation = baseAttestation(); - attestation.teardown.network = false; - assert.throws(() => verify(signedEvent(attestation)), /teardown is incomplete/); -}); - -test("rejects mutable sender-name forgery with the wrong numeric App sender ID", () => { - assert.throws(() => verify(signedEvent(baseAttestation(), 9002)), /dedicated Launcher App/); -}); - -test("rejects authoritative runner mismatch and a still-registered runner", () => { - const mismatch = authoritative(); - mismatch.job.runner_id = 45; - assert.throws(() => validateAuthoritativeEvidence(mismatch), /runner identity differs/); - const registered = authoritative(); - registered.runnerLookupStatus = 200; - assert.throws(() => validateAuthoritativeEvidence(registered), /still registered/); - const noListAuthority = authoritative(); - noListAuthority.runnerListStatus = 403; - assert.throws(() => validateAuthoritativeEvidence(noListAuthority), /absence is uncertain/); -}); - -test("accepts a failing test result only as a failing check after verified teardown", () => { - const attestation = baseAttestation(); - attestation.run.conclusion = "failure"; - const verified = validateAuthoritativeEvidence(authoritative(attestation)); - assert.equal(verified.conclusion, "failure"); -}); - -test("accepts protected-main push evidence only when source and workflow SHAs match", () => { - const attestation = baseAttestation(); - attestation.run.event = "push"; - attestation.run.head_sha = attestation.run.workflow_sha; - const evidence = authoritative(attestation); - evidence.run.pull_requests = []; - evidence.pullRequest = null; - const verified = validateAuthoritativeEvidence(evidence); - assert.equal(verified.event, "push"); - const mismatch = baseAttestation(); - mismatch.run.event = "push"; - assert.throws(() => verify(signedEvent(mismatch)), /identity or conclusion is invalid/); -}); - -test("rejects an authoritative workflow execution SHA mismatch", () => { - const evidence = authoritative(); - evidence.job.head_sha = "d".repeat(40); - assert.throws(() => validateAuthoritativeEvidence(evidence), /runner identity differs/); -}); - -test("rejects a pull association whose exact base SHA differs from the workflow SHA", () => { - const evidence = authoritative(); - evidence.run.pull_requests[0].base.sha = "d".repeat(40); - assert.throws(() => validateAuthoritativeEvidence(evidence), /head and base identities/); -}); - -test("rejects replay for the same run attempt and Checks App", () => { - assert.throws(() => assertNoAttestationReplay({ - appID: "500", - prefix: "postgres-ci:pull_request_target:1234:2:", - existing: {total_count: 1, check_runs: [{app: {id: 500}, external_id: `postgres-ci:pull_request_target:1234:2:${"A".repeat(43)}`}]}, - }), /replay detected/); -}); diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 8b031e0..37972c3 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -67,6 +67,11 @@ readme = read_required_text(repo_root / "README.md", "README") base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") host_compose = read_required_text(repo_root / "compose.host.yml", "host Compose file") runtrace_hba = read_required_text(repo_root / "config/runtrace-pg_hba.conf", "Runtrace HBA policy") +hba_records = [ + tuple(line.split()) + for line in runtrace_hba.splitlines() + if line.strip() and not line.lstrip().startswith("#") +] runtrace_backup = read_required_text(repo_root / "scripts/run-runtrace-backup.sh", "Runtrace backup script") runtrace_backup_loop = read_required_text(repo_root / "scripts/run-runtrace-backup-loop.sh", "Runtrace backup loop") runtrace_restore = read_required_text(repo_root / "scripts/verify-runtrace-restore.sh", "Runtrace restore verifier") @@ -95,13 +100,6 @@ identity_deploy = read_required_text(identity_deploy_path, "Brio identity DB-VM identity_workflow = read_required_text(repo_root / ".github/workflows/deploy-brio-identity-db.yml", "Brio identity DB-VM workflow") release_workflow = read_required_text(repo_root / ".github/workflows/release-brio-identity-db.yml", "Brio identity database release orchestrator") cohort_workflow = read_required_text(repo_root / ".github/workflows/verify-keycloak-cohort-restores.yml", "Keycloak cohort restore workflow") -pr_finalizer_workflow = read_required_text(repo_root / ".github/workflows/pr-ci-result.yml", "PR CI finalizer") -pr_check_publisher = read_required_text(repo_root / "scripts/publish-pr-ci-check.mjs", "PR CI check publisher") -pr_queue_controller = read_required_text(repo_root / "scripts/postgres-ci-queue-controller.mjs", "PR JIT queue controller") -pr_jit_launcher = read_required_text(repo_root / "scripts/run-postgres-ci-jit-vm.sh", "PR JIT VM launcher") -pr_jit_result_validator_path = repo_root / "scripts/verify-postgres-ci-jit-result.py" -pr_jit_result_validator = read_required_text(pr_jit_result_validator_path, "PR JIT authoritative-result validator") -pr_runner_policy = read_required_text(repo_root / "scripts/configure-postgres-ci-runner-group.sh", "runner-group policy reconciler") environment_policy_reconciler = read_required_text(repo_root / "scripts/reconcile-github-environment-main-policy.py", "GitHub environment policy reconciler") environment_policy_test = read_required_text(repo_root / "scripts/test-github-environment-main-policy.py", "GitHub environment policy test") release_evidence_validator = read_required_text(repo_root / "scripts/verify-brio-release-evidence.py", "Brio release evidence validator") @@ -134,7 +132,6 @@ for environment in ( "staging-brio-identity-db", "release-brio-identity-db", "keycloak-cohort-restore", - "postgres-ci-attestation", ): require(f'"{environment}"' in environment_policy_reconciler, f"Environment policy reconciler must include {environment}.") for required in ( @@ -610,7 +607,6 @@ for workflow_name, workflow_text in ( ("identity DB-VM deploy", identity_workflow), ("identity database release", release_workflow), ("Keycloak cohort restore", cohort_workflow), - ("PR CI finalizer", pr_finalizer_workflow), ): checkout_ref = "uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5" checkout_count = workflow_text.count(checkout_ref) @@ -620,21 +616,20 @@ for workflow_name, workflow_text in ( f"Every self-hosted checkout in the {workflow_name} workflow must disable persisted Git credentials.", ) -# The repository is public. Every Actions job must therefore use one of the -# explicitly selected self-hosted runner groups; adding a hosted runner (or a -# string-form, ungrouped self-hosted label) is a release-policy violation. +# Every Actions job uses the existing Makepad Linux runner. Pull-request jobs +# additionally reject forks before a runner is assigned. workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorted( (repo_root / ".github/workflows").glob("*.yaml") ) require(workflow_paths, "At least one GitHub Actions workflow must exist.") for workflow_path in workflow_paths: workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") - runs_on_count = len(re.findall(r"(?m)^ runs-on:\s*$", workflow_text)) - grouped_count = len(re.findall(r"(?m)^ runs-on:\s*\n group: [^\n]+\n labels: \[[^\n]*self-hosted[^\n]*\]$", workflow_text)) + runs_on_count = len(re.findall(r"(?m)^ runs-on:", workflow_text)) + existing_runner_count = workflow_text.count(" runs-on: [self-hosted, linux, x64, makepad]") require(runs_on_count > 0, f"Workflow {workflow_path.name} must define at least one job runner.") require( - runs_on_count == grouped_count, - f"Every job in {workflow_path.name} must use a selected group and explicit self-hosted labels.", + runs_on_count == existing_runner_count, + f"Every job in {workflow_path.name} must use the existing Makepad Linux runner labels.", ) require( not re.search(r"(?i)(ubuntu|windows|macos)-(latest|[0-9]+)", workflow_text), @@ -684,20 +679,6 @@ credential_inventory = { ("keycloak-cohort-restore",), ("DOCKERHUB_USERNAME", "DOCKERHUB_PRO_PAT", "DHI_REGISTRY_USERNAME", "DHI_REGISTRY_PASSWORD"), ), - "PostgreSQL · PR Checks App": ( - ("postgres-ci-attestation",), - ( - "POSTGRES_PR_CHECK_APP_ID", "POSTGRES_PR_CHECK_APP_PRIVATE_KEY", - ), - ), - "PostgreSQL · JIT Launcher App": ( - ("postgres-ci-attestation",), - ("POSTGRES_CI_LAUNCHER_APP_SENDER_ID",), - ), - "PostgreSQL · JIT hypervisor attestation": ( - ("postgres-ci-attestation",), - ("POSTGRES_CI_ATTESTATION_PUBLIC_KEY", "POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256"), - ), } readme_lines = readme.splitlines() for item, (environments, fields) in credential_inventory.items(): @@ -736,11 +717,6 @@ for path in ( cohort_host_installer_path, cohort_cleaner_path, cohort_cleaner_installer_path, - repo_root / "scripts/run-postgres-ci-jit-vm.sh", - pr_jit_result_validator_path, - repo_root / "scripts/test-postgres-ci-jit-result.sh", - repo_root / "scripts/run-postgres-ci-queue-controller.sh", - repo_root / "scripts/configure-postgres-ci-runner-group.sh", ): require(os.access(path, os.X_OK), f"Brio deployment script must be executable: {path}") @@ -761,57 +737,13 @@ for required in ( "Remove remote job-scoped deployment material", ): require(required in manual_deploy_workflow, f"Canary workflow is missing secure Brio input/deploy control: {required}") -require("makepad-postgres-deploy" in manual_deploy_workflow, "Manual deployment must use the repository-scoped deploy runner label.") -require("group: Postgres Deploy" in manual_deploy_workflow, "Manual deployment must use the protected Postgres Deploy runner group.") +require("runs-on: [self-hosted, linux, x64, makepad]" in manual_deploy_workflow, "Manual deployment must use the existing Makepad Linux runner.") require('[[ "${GITHUB_REF}" == "refs/heads/main" ]]' in manual_deploy_workflow, "Manual deployment must refuse unreviewed refs.") -require("pull_request_target:" in ci_workflow, "PR CI must execute protected-base workflow code.") +require("pull_request:" in ci_workflow and "pull_request_target:" not in ci_workflow, "PR CI must use the native pull-request event.") require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks.") require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact candidate head.") -require("group: org/Postgres PR Ephemeral" in ci_workflow, "PR CI must use the selected-workflow ephemeral runner group.") -require("group: org/Postgres Main CI" in ci_workflow, "Main CI must use its protected selected-workflow runner group.") -require("repository_dispatch:" in pr_finalizer_workflow and "types: [postgres-pr-ci-attestation]" in pr_finalizer_workflow and "environment: postgres-ci-attestation" in pr_finalizer_workflow, "PR result publication must accept only protected signed teardown dispatches.") -require("POSTGRES_PR_CHECK_APP_PRIVATE_KEY" in pr_finalizer_workflow and 'CHECK_NAMES = ["postgres-ci"]' in pr_check_publisher, "The required PR result must be published by its dedicated Checks App.") -for required in ( - "makepad.postgres.ci-attestation.v1", - "verifySignature", - "registration_absent", - "runnerLookupStatus !== 404", - "POSTGRES_CI_ATTESTATION_PUBLIC_KEY", - "POSTGRES_CI_LAUNCHER_APP_SENDER_ID", -): - require(required in pr_check_publisher + pr_finalizer_workflow, f"Signed JIT teardown finalization is missing: {required}") -for required in ( - "generate-jitconfig", - "--jitconfig", - "qemu-img convert", - "virsh undefine", - "nft delete table", - "registration_absent", - "dispatch-ci-attestation.mjs", - "makepad-postgres-pr-ephemeral", - "resources.json", - "--reconcile", - "POSTGRES_CI_RESULT_POLL_ATTEMPTS", - "verify-postgres-ci-jit-result.py", -): - require(required in pr_jit_launcher, f"Disposable PR VM launcher is missing: {required}") -require('base.get("sha") != workflow_sha' in pr_jit_result_validator, "The final JIT attestation verifier must bind the PR base SHA to the workflow SHA.") -require("test-postgres-ci-jit-result.sh" in ci_runner, "CI must run the executable final JIT base-SHA regression test.") -for required in ( - 'job.name === "policy-and-integration"', - "state.jobs[String(job.jobID)]", - "await atomicState(stateFile, state)", - "await runLauncher", - "await reconcileIncompleteJobs", - "launchID", - 'organization_self_hosted_runners: "write"', -): - require(required in pr_queue_controller, f"Supervised JIT queue controller is missing: {required}") -require('"allows_public_repositories": True' in pr_runner_policy, "The selected-workflow runner policy must explicitly support the public PostgreSQL repository.") -require("makepad-postgres-ci-attestor" in pr_runner_policy and "makepad-postgres-pr-ephemeral" in pr_runner_policy, "Runner policy must separate the persistent attestor from the JIT-only label.") -require('association.head?.repo?.id !== run.repository?.id' in pr_check_publisher, "The PR Checks publisher must independently reject fork runs.") -require('association.base?.sha !== attestation.run.workflow_sha' in pr_check_publisher, "The PR Checks publisher must bind the exact PR base SHA.") -require('association.base?.sha !== run.head_sha' in pr_queue_controller, "The queue controller must bind the exact PR base SHA before launch.") +require(ci_workflow.count("runs-on: [self-hosted, linux, x64, makepad]") == 2, "PR and protected-main CI must use the existing Makepad Linux runner.") +require("permissions:\n contents: read" in ci_workflow, "CI must keep default permissions read-only.") require('"${RUNNER_TEMP}"/postgres-deploy-*|"${RUNNER_TEMP}"/postgres-brio-canary-runtime-*|"${RUNNER_TEMP}"/postgres-brio-vif-runtime-*' in manual_deploy_workflow, "Cleanup must allow only the exact job-scoped deployment directory prefixes.") require("for cleanup_target in" in manual_deploy_workflow, "Manual workflow cleanup must use a narrowly named cleanup target variable.") require("group: postgres-shared-swarm-target" in manual_deploy_workflow, "Canary and production must share one target-wide Swarm concurrency group.") @@ -878,8 +810,7 @@ for required in ( "brio-db-deployment-evidence.json", "makepad.brio-db-deployment-evidence.v1", "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", - "makepad-postgres-deploy", - "group: Postgres Deploy", + "runs-on: [self-hosted, linux, x64, makepad]", ): require(required in identity_workflow, f"Standalone identity DB workflow is missing: {required}") for required in ( diff --git a/scripts/verify-postgres-ci-jit-result.py b/scripts/verify-postgres-ci-jit-result.py deleted file mode 100755 index 388fe85..0000000 --- a/scripts/verify-postgres-ci-jit-result.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -"""Bind a disposed JIT runner to its exact authoritative GitHub result.""" - -from __future__ import annotations - -import argparse -import json -import re -from pathlib import Path - - -EXPECTED_REPOSITORY = "Makepad-fr/postgres" -EXPECTED_LABELS = {"self-hosted", "linux", "x64", "makepad-postgres-pr-ephemeral"} - - -def positive(value: object, label: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ValueError(f"{label} must be a positive integer") - return value - - -def load_object(path: Path, label: str) -> dict[str, object]: - if path.is_symlink() or not path.is_file() or path.stat().st_size > 2 * 1024 * 1024: - raise ValueError(f"{label} must be a small regular file") - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError(f"{label} must be a JSON object") - return value - - -def validate( - run: dict[str, object], - response: dict[str, object], - *, - run_id: int, - attempt: int, - job_id: int, - event: str, - source_sha: str, - workflow_sha: str, - runner_id: int, - runner_name: str, - runner_group_id: int, -) -> str: - for value, label in ( - (run_id, "run ID"), - (attempt, "run attempt"), - (job_id, "job ID"), - (runner_id, "runner ID"), - (runner_group_id, "runner group ID"), - ): - positive(value, label) - if event not in {"pull_request_target", "push"}: - raise ValueError("unsupported workflow event") - if not re.fullmatch(r"[a-f0-9]{40}", source_sha) or not re.fullmatch(r"[a-f0-9]{40}", workflow_sha): - raise ValueError("source and workflow SHAs must be lowercase commit IDs") - if not re.fullmatch(r"postgres-ci-jit-j[1-9][0-9]{0,15}-[a-f0-9]{16}", runner_name): - raise ValueError("runner name is outside the deterministic JIT namespace") - - jobs = response.get("jobs") - total = response.get("total_count") - if not isinstance(jobs, list) or isinstance(total, bool) or total != len(jobs): - raise ValueError("authoritative attempt-job response is truncated") - matches = [value for value in jobs if isinstance(value, dict) and value.get("id") == job_id] - if len(matches) != 1: - raise ValueError("exact job is not unique in the authoritative run attempt") - job = matches[0] - raw_labels = job.get("labels") - if not isinstance(raw_labels, list) or not all(isinstance(value, str) for value in raw_labels): - raise ValueError("authoritative job labels are invalid") - actual_labels = [value.lower() for value in raw_labels] - if len(actual_labels) != len(EXPECTED_LABELS) or set(actual_labels) != EXPECTED_LABELS: - raise ValueError("authoritative job identity does not match this hypervisor execution") - - repository = run.get("repository") - if not isinstance(repository, dict): - raise ValueError("authoritative repository identity is missing") - repository_id = positive(repository.get("id"), "repository ID") - if ( - run.get("id") != run_id - or run.get("run_attempt") != attempt - or run.get("event") != event - or run.get("head_sha") != workflow_sha - or run.get("head_branch") != "main" - or run.get("name") != "CI" - or run.get("path") != ".github/workflows/ci.yml" - or run.get("status") != "completed" - or repository.get("full_name") != EXPECTED_REPOSITORY - or job.get("run_id") != run_id - or job.get("id") != job_id - or job.get("head_sha") != workflow_sha - or job.get("workflow_name") != "CI" - or job.get("runner_id") != runner_id - or job.get("runner_name") != runner_name - or job.get("runner_group_id") != runner_group_id - or job.get("runner_group_name") != "Postgres PR Ephemeral" - or job.get("name") != "policy-and-integration" - or job.get("status") != "completed" - ): - raise ValueError("authoritative job identity does not match this hypervisor execution") - - if event == "pull_request_target": - associations = run.get("pull_requests") - if not isinstance(associations, list) or len(associations) != 1 or not isinstance(associations[0], dict): - raise ValueError("authoritative pull request association differs from the requested source") - association = associations[0] - head = association.get("head") - base = association.get("base") - if not isinstance(head, dict) or not isinstance(base, dict): - raise ValueError("authoritative pull request association differs from the requested source") - head_repository = head.get("repo") - base_repository = base.get("repo") - if ( - not isinstance(head_repository, dict) - or not isinstance(base_repository, dict) - or head.get("sha") != source_sha - or head_repository.get("id") != repository_id - or base_repository.get("id") != repository_id - or base.get("ref") != "main" - or base.get("sha") != workflow_sha - ): - raise ValueError("authoritative pull request association differs from the requested source") - elif source_sha != workflow_sha: - raise ValueError("protected-main push source differs from its workflow SHA") - - run_conclusion = run.get("conclusion") - job_conclusion = job.get("conclusion") - if run_conclusion != job_conclusion or run_conclusion not in {"success", "failure"}: - raise ValueError("authoritative run and job conclusions are not an exact supported result") - return run_conclusion - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("run_file", type=Path) - parser.add_argument("jobs_file", type=Path) - parser.add_argument("run_id", type=int) - parser.add_argument("attempt", type=int) - parser.add_argument("job_id", type=int) - parser.add_argument("event") - parser.add_argument("source_sha") - parser.add_argument("workflow_sha") - parser.add_argument("runner_id", type=int) - parser.add_argument("runner_name") - parser.add_argument("runner_group_id", type=int) - arguments = parser.parse_args() - print( - validate( - load_object(arguments.run_file, "run response"), - load_object(arguments.jobs_file, "jobs response"), - run_id=arguments.run_id, - attempt=arguments.attempt, - job_id=arguments.job_id, - event=arguments.event, - source_sha=arguments.source_sha, - workflow_sha=arguments.workflow_sha, - runner_id=arguments.runner_id, - runner_name=arguments.runner_name, - runner_group_id=arguments.runner_group_id, - ) - ) - - -if __name__ == "__main__": - main() From 5a477e2069e08092d9a82ab28d068ae643f68fe3 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:11:27 +0200 Subject: [PATCH 16/20] refactor(postgres): keep VIF bootstrap in its deploy path --- .github/workflows/manual-deploy.yml | 1 - README.md | 3 +- bootstrap/vif-app.sql | 36 ----------------- scripts/deploy-postgres-stack.sh | 48 +++++++++++++++++++---- scripts/test-brio-deployment-contracts.sh | 3 +- scripts/validate-postgres-config.sh | 11 +++--- 6 files changed, 47 insertions(+), 55 deletions(-) delete mode 100644 bootstrap/vif-app.sql diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 610e84b..d8bf923 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -149,7 +149,6 @@ jobs: cp scripts/brio-db-transaction.sh "${bundle_root}/scripts/brio-db-transaction.sh" cp scripts/ensure-brio-tmp-cleaner.sh "${bundle_root}/scripts/ensure-brio-tmp-cleaner.sh" cp bootstrap/brio-staging-app.sql "${bundle_root}/bootstrap/brio-staging-app.sql" - cp bootstrap/vif-app.sql "${bundle_root}/bootstrap/vif-app.sql" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" < :'vif_user' -) \gexec -SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh index 0ba486f..2f34a87 100755 --- a/scripts/deploy-postgres-stack.sh +++ b/scripts/deploy-postgres-stack.sh @@ -218,10 +218,6 @@ if [[ "${vif_enabled}" == "1" ]]; then echo "VIF credential must contain one line and no carriage return." >&2 exit 1 fi - [[ -f "${remote_dir}/bootstrap/vif-app.sql" && ! -L "${remote_dir}/bootstrap/vif-app.sql" ]] || { - echo "The VIF bootstrap SQL is missing from the job-scoped bundle." >&2 - exit 1 - } fi if [[ "${brio_staging_enabled}" == "1" ]]; then : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" @@ -366,13 +362,49 @@ if [[ "${postgres_ready}" != "1" ]]; then exit 1 fi -docker run --rm --network "${vif_db_network}" \ +docker run --rm --interactive --network "${vif_db_network}" \ -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ -v "${vif_db_password_file}:/run/secrets/vif_db_password:ro" \ - -v "${remote_dir}/bootstrap/vif-app.sql:/bootstrap/vif-app.sql:ro" \ "${postgres_image}" sh -euc ' export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" export VIF_PASSWORD="$(cat /run/secrets/vif_db_password)" exec psql -X -v ON_ERROR_STOP=1 -h makepad-postgres-vif -U "$1" -d postgres \ - -v vif_db="$2" -v vif_user="$3" -f /bootstrap/vif-app.sql - ' sh "${postgres_root_user}" "${vif_db_name}" "${vif_db_user}" >/dev/null + -v vif_db="$2" -v vif_user="$3" + ' sh "${postgres_root_user}" "${vif_db_name}" "${vif_db_user}" >/dev/null <<'SQL' +\set ON_ERROR_STOP on + +\if :{?vif_db} +\else + \echo 'missing required psql variable: vif_db' + SELECT 1 / 0; +\endif + +\if :{?vif_user} +\else + \echo 'missing required psql variable: vif_user' + SELECT 1 / 0; +\endif + +\getenv vif_password VIF_PASSWORD +SELECT CASE WHEN NULLIF(btrim(:'vif_password'), '') IS NULL THEN 'false' ELSE 'true' END AS vif_password_is_nonempty \gset +\if :vif_password_is_nonempty +\else + \echo 'empty required environment variable: VIF_PASSWORD' + SELECT 1 / 0; +\endif + +SELECT format('CREATE ROLE %I LOGIN', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec +SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec +SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = :'vif_db' + AND r.rolname <> :'vif_user' +) \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec +SQL diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh index 61cce60..478e009 100755 --- a/scripts/test-brio-deployment-contracts.sh +++ b/scripts/test-brio-deployment-contracts.sh @@ -18,7 +18,6 @@ cohort_validator = (root / "scripts/verify-keycloak-cohort-evidence.py").read_te identity = (root / "scripts/deploy-brio-identity-db-host.sh").read_text() canary = (root / "scripts/deploy-brio-canary-postgres.sh").read_text() stack = (root / "scripts/deploy-postgres-stack.sh").read_text() -vif = (root / "bootstrap/vif-app.sql").read_text() hba = (root / "config/runtrace-pg_hba.conf").read_text().splitlines() canary_env = (root / "envs/canary/.env.db").read_text() production_env = (root / "envs/production/.env.db").read_text() @@ -44,7 +43,7 @@ require("${REMOTE_DIR}/stack.yml" not in manual + stack, "shared stack.yml is fo require('stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml"' in stack, "stack config must stay in the run bundle") require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual + stack, "VIF secret must not persist in .env.deploy") require("-v vif_password=" not in stack, "VIF secret must not enter psql argv") -require("\\getenv vif_password VIF_PASSWORD" in vif, "VIF bootstrap must use getenv") +require("\\getenv vif_password VIF_PASSWORD" in stack, "VIF bootstrap must use getenv") for marker in ( "compose_project=postgres", diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 37972c3..c10e9b8 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -62,7 +62,6 @@ keycloak_runtrace_sql = read_required_text(repo_root / "bootstrap/keycloak-runtr openpanel_sql = read_required_text(repo_root / "bootstrap/openpanel-app.sql", "OpenPanel app SQL bootstrap") brio_sql = read_required_text(repo_root / "bootstrap/brio-staging-app.sql", "Brio staging app SQL bootstrap") keycloak_brio_sql = read_required_text(repo_root / "bootstrap/keycloak-brio-staging.sql", "targeted Brio Keycloak SQL bootstrap") -vif_sql = read_required_text(repo_root / "bootstrap/vif-app.sql", "VIF application SQL bootstrap") readme = read_required_text(repo_root / "README.md", "README") base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") host_compose = read_required_text(repo_root / "compose.host.yml", "host Compose file") @@ -378,15 +377,15 @@ require('wait_for_service_convergence "${stack_name}_brio_staging_backup" "${bri require("keycloak_brio_staging_backup" not in production_compose, "The Brio identity backup must never be routed through the production Swarm override.") require("Postgres did not become reachable via makepad-postgres-vif" in manual_deploy, "Manual deploy workflow must fail clearly when VIF readiness times out.") require( - not re.search(r"\S\\gexec", vif_sql), + not re.search(r"\S\\gexec", remote_deploy), "VIF bootstrap must separate every \\gexec command from SQL text by whitespace.", ) -require("ALTER ROLE %I LOGIN PASSWORD %L" in vif_sql, "VIF bootstrap must always refresh the VIF role password.") -require("ALTER DATABASE %I OWNER TO %I" in vif_sql, "VIF bootstrap must repair VIF database ownership drift.") -require("\\getenv vif_password VIF_PASSWORD" in vif_sql, "VIF bootstrap must read its password from the mounted-file environment only.") +require("ALTER ROLE %I LOGIN PASSWORD %L" in remote_deploy, "VIF bootstrap must always refresh the VIF role password.") +require("ALTER DATABASE %I OWNER TO %I" in remote_deploy, "VIF bootstrap must repair VIF database ownership drift.") +require("\\getenv vif_password VIF_PASSWORD" in remote_deploy, "VIF bootstrap must read its password from the mounted-file environment only.") require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual_deploy_workflow + remote_deploy, "VIF password must never be persisted in the deployment environment file.") require('-v vif_password=' not in remote_deploy, "VIF password must never be placed in psql command arguments.") -for required in ("postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", "vif-db-password", "bootstrap/vif-app.sql"): +for required in ("postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", "vif-db-password", "--interactive"): require(required in manual_deploy, f"VIF deployment is missing file-only credential control: {required}") require( sql.count("DO $$") == len(expected_instances), From 7c0f024d585e1b4bd419039d103b2e60a1bb4fab Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:13:48 +0200 Subject: [PATCH 17/20] ci(postgres): check the exact pull request range --- .github/workflows/ci.yml | 14 ++++++++++++-- scripts/run-ci.sh | 1 - 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9380aaf..c31d4d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,15 +25,19 @@ jobs: with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 1 + fetch-depth: 0 - name: Pin checkout to exact internal PR head shell: bash env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EXPECTED_BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -euo pipefail [[ "${EXPECTED_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${EXPECTED_BASE_SHA}" =~ ^[0-9a-f]{40}$ ]] [[ "$(git rev-parse HEAD)" == "${EXPECTED_HEAD_SHA}" ]] + git cat-file -e "${EXPECTED_BASE_SHA}^{commit}" + git diff --check "${EXPECTED_BASE_SHA}...${EXPECTED_HEAD_SHA}" - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version: 1.25.13 @@ -56,14 +60,20 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: persist-credentials: false - fetch-depth: 1 + fetch-depth: 0 - name: Pin protected main checkout shell: bash + env: + EXPECTED_BEFORE_SHA: ${{ github.event.before }} run: | set -euo pipefail [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" ]] [[ "${GITHUB_REF}" == "refs/heads/main" ]] [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + if [[ "${EXPECTED_BEFORE_SHA}" =~ ^[0-9a-f]{40}$ && "${EXPECTED_BEFORE_SHA}" != 0000000000000000000000000000000000000000 ]]; then + git cat-file -e "${EXPECTED_BEFORE_SHA}^{commit}" + git diff --check "${EXPECTED_BEFORE_SHA}..${GITHUB_SHA}" + fi - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version: 1.25.13 diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh index bc3cbdc..85e70af 100755 --- a/scripts/run-ci.sh +++ b/scripts/run-ci.sh @@ -54,7 +54,6 @@ for source in ( PY PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-github-environment-main-policy.py actionlint -git show --check --format= HEAD git diff --check ./scripts/test-brio-deploy-guards.sh ./scripts/test-brio-deployment-contracts.sh From e2629d90404f5f2c60f71c0138caa8d9cfc561a1 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:25:21 +0200 Subject: [PATCH 18/20] test(postgres): verify protected recovery as container root --- scripts/test-brio-deployment-contracts.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh index 478e009..fa8421b 100755 --- a/scripts/test-brio-deployment-contracts.sh +++ b/scripts/test-brio-deployment-contracts.sh @@ -175,7 +175,12 @@ docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleane ' "${script_dir}/ensure-brio-tmp-cleaner.sh" test-clean-production-ownership "${ownership_root}" "${cleaner_image}" [[ ! -e "${ownership_root}/postgres-brio-deploy-owned" ]] || { echo "Production cleaner retained a deploy-UID-owned expired secret directory." >&2; exit 1; } -[[ -f "${ownership_root}/postgres-brio-recovery-owned/RECOVERY_REQUIRED" ]] || { echo "Production cleaner removed recovery evidence." >&2; exit 1; } +docker run --rm --read-only --cap-drop ALL --cap-add DAC_OVERRIDE \ + --security-opt no-new-privileges --mount "type=bind,src=${ownership_root},dst=/fixture,readonly" \ + "${cleaner_image}" sh -euc 'test -f /fixture/postgres-brio-recovery-owned/RECOVERY_REQUIRED' || { + echo "Production cleaner removed recovery evidence." >&2 + exit 1 + } docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc 'find /fixture -mindepth 1 -depth -delete' echo "Brio deployment ordering, rollback, interruption, secret, and TTL contracts passed." From 094224abdee843eb11fbcca545ffbc9d9417c988 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:28:06 +0200 Subject: [PATCH 19/20] test(postgres): isolate cleaner ownership fixtures --- scripts/test-brio-deployment-contracts.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh index fa8421b..9b00bdc 100755 --- a/scripts/test-brio-deployment-contracts.sh +++ b/scripts/test-brio-deployment-contracts.sh @@ -140,8 +140,18 @@ for workflow in (manual, identity_workflow): PY cleaner_root=$(mktemp -d /tmp/postgres-brio-cleaner-test-contract-XXXXXX) +ownership_root="" +cleaner_image="" cleanup_test_root() { [[ "${cleaner_root}" =~ ^/tmp/postgres-brio-cleaner-test-contract-[A-Za-z0-9]+$ ]] || return 1 + if [[ -n "${ownership_root}" && ( -e "${ownership_root}" || -L "${ownership_root}" ) ]]; then + [[ "${ownership_root}" =~ ^/tmp/postgres-brio-cleaner-test-production-ownership-[A-Za-z0-9]+$ \ + && -d "${ownership_root}" && ! -L "${ownership_root}" \ + && "${cleaner_image}" == *@sha256:* ]] || return 1 + docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" \ + "${cleaner_image}" sh -euc 'find /fixture -mindepth 1 -depth -delete' + rmdir -- "${ownership_root}" + fi find "${cleaner_root}" -depth -delete } trap cleanup_test_root EXIT @@ -161,10 +171,9 @@ touch -t 202001010000 "${cleaner_root}/postgres-brio-old" "${cleaner_root}/postg # Reproduce production ownership: SSH-created runtime directories are mode 0700 # and owned by the deploy UID, not by the cleaner container. Minimal DAC/FOWNER # capabilities must delete expired material while retaining recovery markers. -ownership_root=/tmp/postgres-brio-cleaner-test-production-ownership -[[ ! -e "${ownership_root}" && ! -L "${ownership_root}" ]] || find "${ownership_root}" -depth -delete -install -d -m 0700 "${ownership_root}" cleaner_image=$(awk -F= '$1 == "POSTGRES_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +ownership_root=$(mktemp -d /tmp/postgres-brio-cleaner-test-production-ownership-XXXXXX) +chmod 0700 "${ownership_root}" docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc ' mkdir /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned printf "%s\n" secret > /fixture/postgres-brio-deploy-owned/credential @@ -181,6 +190,5 @@ docker run --rm --read-only --cap-drop ALL --cap-add DAC_OVERRIDE \ echo "Production cleaner removed recovery evidence." >&2 exit 1 } -docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc 'find /fixture -mindepth 1 -depth -delete' echo "Brio deployment ordering, rollback, interruption, secret, and TTL contracts passed." From 44a415dc46e9e5fc2e7c03d72a58ca72e507a6a0 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Sat, 5 Sep 2026 15:45:33 +0200 Subject: [PATCH 20/20] fix(postgres): own the Brio database network --- README.md | 2 +- scripts/deploy-postgres-stack.sh | 20 +++++++++++++++----- scripts/validate-postgres-config.sh | 7 +++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 19d0e05..9feb6ad 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The manual deploy workflow sources these Compose variables from environment secr - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK` production only - `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` <- `DEPLOY_BRIO_STAGING_DB_NETWORK` canary only -Every database network must be an attachable Swarm overlay created with `--opt encrypted=true`; Brio's dedicated network must additionally be `--internal`. The explicit value matters: Docker records a valueless `--opt encrypted` as an empty option rather than the required `true`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. +Every database network must be an attachable Swarm overlay created with `--opt encrypted=true`; Brio's dedicated network must additionally be `--internal`. PostgreSQL owns that shared Brio database network and labels it with `com.makepad.owner=Makepad-fr/postgres`, environment `staging`, instance `brio`, and purpose `database`; Brio consumes it but must not create or relabel it. The explicit encryption value matters: Docker records a valueless `--opt encrypted` as an empty option rather than the required `true`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. Application network topology is owned by the consuming application repositories. New Keycloak instances keep their own DB-facing Docker networks in the Keycloak repository and connect to this PostgreSQL server through the configured DB endpoint. diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh index 2f34a87..d995c0b 100755 --- a/scripts/deploy-postgres-stack.sh +++ b/scripts/deploy-postgres-stack.sh @@ -246,12 +246,22 @@ ensure_encrypted_overlay_network() { ensure_internal_encrypted_overlay_network() { local network_name=$1 if ! docker network inspect "${network_name}" >/dev/null 2>&1; then - docker network create --driver overlay --attachable --internal --opt encrypted=true "${network_name}" >/dev/null + # A concurrent consumer can observe the same absence. Validate the final + # object below instead of trusting the create command's exit status. + docker network create --driver overlay --attachable --internal --opt encrypted=true \ + --label com.makepad.owner=Makepad-fr/postgres \ + --label com.makepad.environment=staging \ + --label com.makepad.instance=brio \ + --label com.makepad.purpose=database "${network_name}" >/dev/null 2>&1 || true fi - local details - details=$(docker network inspect "${network_name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') - if [[ "${details}" != "overlay swarm true true true" ]]; then - echo "Brio database network ${network_name} must be an internal, encrypted, attachable Swarm overlay; got ${details}." >&2 + local details expected + details=$(docker network inspect "${network_name}" --format '{{.Driver}}|{{.Scope}}|{{.Internal}}|{{.Attachable}}|{{index .Options "encrypted"}}|{{index .Labels "com.makepad.owner"}}|{{index .Labels "com.makepad.environment"}}|{{index .Labels "com.makepad.instance"}}|{{index .Labels "com.makepad.purpose"}}') || { + echo "Brio database network ${network_name} does not exist after provisioning." >&2 + exit 1 + } + expected='overlay|swarm|true|true|true|Makepad-fr/postgres|staging|brio|database' + if [[ "${details}" != "${expected}" ]]; then + echo "Brio database network ${network_name} must have the required isolation and PostgreSQL ownership metadata." >&2 exit 1 fi } diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index c10e9b8..bd5513d 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -261,6 +261,13 @@ require("runtrace_hba_v2" not in canary_env + production_env, "Active environmen require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose Brio's certificate-matching database alias.") require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach Brio's isolated database network.") require("ensure_internal_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate Brio's internal encrypted database network.") +for required in ( + "com.makepad.owner=Makepad-fr/postgres", + "com.makepad.environment=staging", + "com.makepad.instance=brio", + "com.makepad.purpose=database", +): + require(required in manual_deploy, f"Brio database network ownership policy is missing: {required}") for content, role, database in ( (brio_sql, "brio_staging_app", "brio_staging"), (keycloak_brio_sql, "keycloak_brio_staging_app", "keycloak_brio_staging"),