diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..628281c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - makepad diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c31d4d6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,88 @@ +name: CI + +on: + push: + branches: [main] + # Only same-repository candidate branches can reach the existing Makepad + # runner. Fork jobs are skipped before a runner is assigned. + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +jobs: + validate-pr: + name: policy-and-integration + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.draft == false + runs-on: [self-hosted, linux, x64, makepad] + timeout-minutes: 45 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Pin checkout to exact internal PR head + shell: bash + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EXPECTED_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + [[ "${EXPECTED_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${EXPECTED_BASE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "$(git rev-parse HEAD)" == "${EXPECTED_HEAD_SHA}" ]] + git cat-file -e "${EXPECTED_BASE_SHA}^{commit}" + git diff --check "${EXPECTED_BASE_SHA}...${EXPECTED_HEAD_SHA}" + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: 1.25.13 + cache: false + - name: Run complete candidate suite + shell: bash + run: | + set -euo pipefail + tools_dir="${RUNNER_TEMP}/postgres-pr-tools" + install -d -m 0700 "${tools_dir}" + GOBIN="${tools_dir}" go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + PATH="${tools_dir}:${PATH}" ./scripts/run-ci.sh + + validate-main: + name: protected-main-policy-and-integration + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: [self-hosted, linux, x64, makepad] + timeout-minutes: 45 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + fetch-depth: 0 + - name: Pin protected main checkout + shell: bash + env: + EXPECTED_BEFORE_SHA: ${{ github.event.before }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" ]] + [[ "${GITHUB_REF}" == "refs/heads/main" ]] + [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + if [[ "${EXPECTED_BEFORE_SHA}" =~ ^[0-9a-f]{40}$ && "${EXPECTED_BEFORE_SHA}" != 0000000000000000000000000000000000000000 ]]; then + git cat-file -e "${EXPECTED_BEFORE_SHA}^{commit}" + git diff --check "${EXPECTED_BEFORE_SHA}..${GITHUB_SHA}" + fi + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: 1.25.13 + cache: false + - name: Run complete protected-main suite + shell: bash + run: | + set -euo pipefail + tools_dir="${RUNNER_TEMP}/postgres-main-tools" + install -d -m 0700 "${tools_dir}" + GOBIN="${tools_dir}" go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + PATH="${tools_dir}:${PATH}" ./scripts/run-ci.sh diff --git a/.github/workflows/deploy-brio-identity-db.yml b/.github/workflows/deploy-brio-identity-db.yml new file mode 100644 index 0000000..597fb46 --- /dev/null +++ b/.github/workflows/deploy-brio-identity-db.yml @@ -0,0 +1,225 @@ +name: Deploy Brio Identity Database + +on: + workflow_dispatch: + inputs: + restart_confirmation: + description: Type restart-standalone-postgres-for-brio-staging + required: true + type: string + backup_restore_confirmed: + description: Confirm a current encrypted backup was restore-tested before this DB-VM change + required: true + type: boolean + default: false + +concurrency: + group: postgres-standalone-db-vm + cancel-in-progress: false + +permissions: + contents: read + +jobs: + deploy: + runs-on: [self-hosted, linux, x64, makepad] + environment: staging-brio-identity-db + timeout-minutes: 60 + steps: + - name: Enforce protected standalone deployment gate + shell: bash + env: + RESTART_CONFIRMATION: ${{ inputs.restart_confirmation }} + BACKUP_RESTORE_CONFIRMED: ${{ inputs.backup_restore_confirmed }} + run: | + set -euo pipefail + [[ "${GITHUB_REF}" == "refs/heads/main" ]] || { echo "Standalone DB-VM deployment is allowed only from main." >&2; exit 1; } + [[ "${RESTART_CONFIRMATION}" == "restart-standalone-postgres-for-brio-staging" ]] || { echo "The exact restart acknowledgement is required." >&2; exit 1; } + [[ "${BACKUP_RESTORE_CONFIRMED}" == "true" ]] || { echo "A current successful encrypted restore test is required." >&2; exit 1; } + + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + + - name: Configure job-scoped SSH material + shell: bash + env: + SSH_PRIVATE_KEY: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS }} + run: | + set -euo pipefail + : "${SSH_PRIVATE_KEY:?set BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY}" + : "${SSH_KNOWN_HOSTS:?set BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS}" + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${ssh_dir}" ]] || { echo "Refusing to reuse a job-scoped SSH directory." >&2; exit 1; } + install -d -m 0700 "${ssh_dir}" + umask 077 + printf '%s\n' "${SSH_PRIVATE_KEY}" > "${ssh_dir}/id_ed25519" + printf '%s\n' "${SSH_KNOWN_HOSTS}" > "${ssh_dir}/known_hosts" + chmod 0600 "${ssh_dir}"/* + + - name: Prepare standalone DB-VM bundle and secrets + shell: bash + env: + KEYCLOAK_APP_PASSWORD: ${{ secrets.KEYCLOAK_BRIO_STAGING_DB_PASSWORD }} + KEYCLOAK_BACKUP_PASSWORD: ${{ secrets.KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD }} + BACKUP_RECIPIENT_CERT: ${{ secrets.BRIO_BACKUP_RECIPIENT_CERT_PEM }} + run: | + set -euo pipefail + : "${KEYCLOAK_APP_PASSWORD:?set KEYCLOAK_BRIO_STAGING_DB_PASSWORD}" + : "${KEYCLOAK_BACKUP_PASSWORD:?set KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD}" + : "${BACKUP_RECIPIENT_CERT:?set BRIO_BACKUP_RECIPIENT_CERT_PEM}" + bundle_dir="${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${bundle_dir}" && ! -e "${runtime_dir}" ]] || { echo "Refusing to reuse job-scoped deployment paths." >&2; exit 1; } + install -d -m 0700 "${bundle_dir}/bootstrap" "${bundle_dir}/config" "${bundle_dir}/envs/production" "${bundle_dir}/scripts" "${runtime_dir}" + cp compose.host.yml "${bundle_dir}/compose.host.yml" + cp envs/production/.env.db "${bundle_dir}/envs/production/.env.db" + cp config/runtrace-pg_hba.conf "${bundle_dir}/config/runtrace-pg_hba.conf" + cp bootstrap/keycloak-brio-staging.sql "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" + cp scripts/run-runtrace-backup.sh scripts/run-runtrace-backup-loop.sh "${bundle_dir}/scripts/" + cp scripts/run-brio-encrypted-backup.sh scripts/run-brio-encrypted-backup-loop.sh "${bundle_dir}/scripts/" + cp scripts/deploy-brio-identity-db-host.sh "${bundle_dir}/scripts/deploy-brio-identity-db-host.sh" + cp scripts/brio-db-transaction.sh "${bundle_dir}/scripts/brio-db-transaction.sh" + cp scripts/ensure-brio-tmp-cleaner.sh "${bundle_dir}/scripts/ensure-brio-tmp-cleaner.sh" + umask 077 + printf '%s' "${KEYCLOAK_APP_PASSWORD}" > "${runtime_dir}/keycloak-brio-staging-app-password" + printf '%s' "${KEYCLOAK_BACKUP_PASSWORD}" > "${runtime_dir}/keycloak-brio-staging-backup-password" + printf '%s' "${BACKUP_RECIPIENT_CERT}" > "${runtime_dir}/brio-backup-recipient-cert.pem" + chmod 0600 "${runtime_dir}"/* + + - name: Deploy only to the standalone database VM + shell: bash + env: + REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + DB_HOSTNAME: ${{ vars.BRIO_IDENTITY_DB_HOSTNAME }} + KEYCLOAK_DB_SOURCE_CIDR: ${{ vars.BRIO_KEYCLOAK_DB_SOURCE_CIDR }} + run: | + set -euo pipefail + : "${REMOTE_HOST:?}" "${REMOTE_USER:?}" "${DB_HOSTNAME:?set BRIO_IDENTITY_DB_HOSTNAME}" "${KEYCLOAK_DB_SOURCE_CIDR:?set BRIO_KEYCLOAK_DB_SOURCE_CIDR}" + [[ "${REMOTE_USER}" != "root" ]] || { echo "The standalone DB deploy SSH user must not be root." >&2; exit 1; } + [[ "${DB_HOSTNAME}" == "65.21.134.125" ]] || { echo "BRIO_IDENTITY_DB_HOSTNAME must be the reviewed standalone DB IP 65.21.134.125." >&2; exit 1; } + [[ "${KEYCLOAK_DB_SOURCE_CIDR}" == "88.99.209.165/32" ]] || { echo "BRIO_KEYCLOAK_DB_SOURCE_CIDR must be the reviewed Keycloak egress 88.99.209.165/32." >&2; exit 1; } + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + bundle_dir="${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_bundle="/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime="/tmp/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -p "${remote_port}") + scp_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -P "${remote_port}") + target="${REMOTE_USER}@${REMOTE_HOST}" + printf -v remote_bundle_q %q "${remote_bundle}" + printf -v remote_runtime_q %q "${remote_runtime}" + # Both remote paths are fixed, run-scoped, and escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "if [ -e ${remote_bundle_q} ] || [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote identity deployment path.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + scp "${scp_opts[@]}" -r "${bundle_dir}/." "${target}:${remote_bundle}/" + printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" + printf -v db_env_q %q "${remote_bundle}/envs/production/.env.db" + # Start the host TTL guard before transferring any secret material. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "chmod 0755 ${cleaner_q} && ${cleaner_q} ${db_env_q} && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}/"* "${target}:${remote_runtime}/" + # All remote paths are fixed and escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "chmod 0755 ${remote_bundle_q}/scripts/deploy-brio-identity-db-host.sh ${remote_bundle_q}/scripts/brio-db-transaction.sh ${remote_bundle_q}/scripts/run-runtrace-backup.sh ${remote_bundle_q}/scripts/run-runtrace-backup-loop.sh ${remote_bundle_q}/scripts/run-brio-encrypted-backup.sh ${remote_bundle_q}/scripts/run-brio-encrypted-backup-loop.sh && chmod 0600 ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem" + printf -v deploy_script_q %q "${remote_bundle}/scripts/deploy-brio-identity-db-host.sh" + printf -v db_hostname_q %q "${DB_HOSTNAME}" + printf -v cidr_q %q "${KEYCLOAK_DB_SOURCE_CIDR}" + # Validated values are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" \ + "BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes ${deploy_script_q} ${remote_bundle_q} ${remote_runtime_q} ${db_hostname_q} ${cidr_q}" + + - name: Create canonical standalone deployment evidence + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" && "${GITHUB_REF}" == "refs/heads/main" ]] || { + echo "Deployment evidence is produced only by Makepad-fr/postgres main." >&2 + exit 1 + } + [[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ && "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ \ + && "${GITHUB_SHA}" =~ ^[0-9a-f]{40}$ ]] || { echo "Invalid immutable deployment identity." >&2; exit 1; } + evidence_dir="${RUNNER_TEMP}/postgres-identity-evidence-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${evidence_dir}" && ! -L "${evidence_dir}" ]] || { echo "Refusing to reuse deployment evidence output." >&2; exit 1; } + install -d -m 0700 "${evidence_dir}" + umask 077 + python3 - "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT}" "${GITHUB_SHA}" \ + > "${evidence_dir}/brio-db-deployment-evidence.json" <<'PY' + import json, sys + print(json.dumps({ + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": int(sys.argv[1]), + "postgres_run_attempt": int(sys.argv[2]), + "postgres_head_sha": sys.argv[3], + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, sort_keys=True, separators=(",", ":"))) + PY + chmod 0600 "${evidence_dir}/brio-db-deployment-evidence.json" + + - name: Publish immutable standalone deployment evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/postgres-identity-evidence-${{ github.run_id }}-${{ github.run_attempt }}/brio-db-deployment-evidence.json + if-no-files-found: error + retention-days: 35 + + - name: Remove remote job-scoped identity secrets + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + run: | + set -euo pipefail + ssh_dir="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_dir}/id_ed25519" && -r "${ssh_dir}/known_hosts" ]] || exit 0 + remote_runtime="/tmp/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_bundle="/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -p "${remote_port}") + printf -v remote_runtime_q %q "${remote_runtime}" + printf -v remote_bundle_q %q "${remote_bundle}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${remote_runtime_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_runtime_q} ]; then rm -f ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem; if [ -f ${remote_runtime_q}/RECOVERY_REQUIRED ] && [ ! -L ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${remote_runtime_q} -depth -delete; fi; elif [ -e ${remote_runtime_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + + - name: Remove local job-scoped deployment material + if: always() + shell: bash + run: | + set -euo pipefail + for cleanup_target in \ + "${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-identity-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-identity-evidence-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do + case "${cleanup_target}" in + "${RUNNER_TEMP}"/postgres-identity-*|"${RUNNER_TEMP}"/postgres-brio-identity-runtime-*) + if [[ -L "${cleanup_target}" ]]; then + echo "Refusing cleanup of a symlink: ${cleanup_target}" >&2 + exit 1 + elif [[ -d "${cleanup_target}" ]]; then + find "${cleanup_target}" -mindepth 1 -delete + rmdir "${cleanup_target}" + elif [[ -e "${cleanup_target}" ]]; then + echo "Expected a cleanup directory: ${cleanup_target}" >&2 + exit 1 + fi + ;; + *) echo "Refusing unexpected cleanup path: ${cleanup_target}" >&2; exit 1 ;; + esac + done diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 32befcb..d8bf923 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -11,24 +11,99 @@ on: - canary - production +concurrency: + group: postgres-shared-swarm-target + cancel-in-progress: false + jobs: deploy: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64, makepad] environment: ${{ inputs.environment }} + timeout-minutes: 30 permissions: contents: read steps: + - name: Enforce reviewed deployment ref + shell: bash + run: | + set -euo pipefail + [[ "${GITHUB_REF}" == "refs/heads/main" ]] || { echo "PostgreSQL deployment is allowed only from main." >&2; exit 1; } + - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + + - name: Configure job-scoped SSH material + shell: bash + env: + DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + DEPLOY_SSH_KNOWN_HOSTS: ${{ secrets.DEPLOY_SSH_KNOWN_HOSTS }} + run: | + set -euo pipefail + : "${DEPLOY_SSH_PRIVATE_KEY:?set DEPLOY_SSH_PRIVATE_KEY}" + : "${DEPLOY_SSH_KNOWN_HOSTS:?set DEPLOY_SSH_KNOWN_HOSTS}" + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${ssh_directory}" ]]; then + echo "Refusing to reuse an existing job-scoped SSH directory: ${ssh_directory}" >&2 + exit 1 + fi + mkdir -m 0700 "${ssh_directory}" + printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" > "${ssh_directory}/id_ed25519" + chmod 0600 "${ssh_directory}/id_ed25519" + printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${ssh_directory}/known_hosts" + chmod 0600 "${ssh_directory}/known_hosts" - - name: Configure SSH key + - name: Materialize job-scoped Brio canary inputs + if: inputs.environment == 'canary' shell: bash + env: + BRIO_BACKUP_RECIPIENT_CERT_PEM: ${{ secrets.BRIO_BACKUP_RECIPIENT_CERT_PEM }} + BRIO_STAGING_BACKUP_DB_PASSWORD: ${{ secrets.BRIO_STAGING_BACKUP_DB_PASSWORD }} + BRIO_STAGING_DB_PASSWORD: ${{ secrets.BRIO_STAGING_DB_PASSWORD }} + POSTGRES_CANARY_SUPERUSER_PASSWORD: ${{ secrets.POSTGRES_CANARY_SUPERUSER_PASSWORD }} + POSTGRES_CA_PEM: ${{ secrets.POSTGRES_CA_PEM }} + POSTGRES_SERVER_CERT_PEM: ${{ secrets.POSTGRES_SERVER_CERT_PEM }} + POSTGRES_SERVER_KEY_PEM: ${{ secrets.POSTGRES_SERVER_KEY_PEM }} run: | set -euo pipefail - mkdir -p "${HOME}/.ssh" - chmod 700 "${HOME}/.ssh" - printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > "${HOME}/.ssh/id_ed25519" - chmod 600 "${HOME}/.ssh/id_ed25519" + : "${BRIO_BACKUP_RECIPIENT_CERT_PEM:?set BRIO_BACKUP_RECIPIENT_CERT_PEM}" + : "${BRIO_STAGING_BACKUP_DB_PASSWORD:?set BRIO_STAGING_BACKUP_DB_PASSWORD}" + : "${BRIO_STAGING_DB_PASSWORD:?set BRIO_STAGING_DB_PASSWORD}" + : "${POSTGRES_CANARY_SUPERUSER_PASSWORD:?set POSTGRES_CANARY_SUPERUSER_PASSWORD}" + : "${POSTGRES_CA_PEM:?set POSTGRES_CA_PEM}" + : "${POSTGRES_SERVER_CERT_PEM:?set POSTGRES_SERVER_CERT_PEM}" + : "${POSTGRES_SERVER_KEY_PEM:?set POSTGRES_SERVER_KEY_PEM}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${runtime_dir}" ]]; then + echo "Refusing to reuse an existing job-scoped Brio runtime directory." >&2 + exit 1 + fi + install -d -m 0700 "${runtime_dir}" + umask 077 + printf '%s' "${POSTGRES_CANARY_SUPERUSER_PASSWORD}" > "${runtime_dir}/postgres-superuser-password" + printf '%s' "${BRIO_STAGING_DB_PASSWORD}" > "${runtime_dir}/brio-staging-app-password" + printf '%s' "${BRIO_STAGING_BACKUP_DB_PASSWORD}" > "${runtime_dir}/brio-staging-backup-password" + printf '%s' "${POSTGRES_CA_PEM}" > "${runtime_dir}/postgres-ca.pem" + printf '%s' "${POSTGRES_SERVER_CERT_PEM}" > "${runtime_dir}/postgres-server-cert.pem" + printf '%s' "${POSTGRES_SERVER_KEY_PEM}" > "${runtime_dir}/postgres-server-key.pem" + printf '%s' "${BRIO_BACKUP_RECIPIENT_CERT_PEM}" > "${runtime_dir}/brio-backup-recipient-cert.pem" + chmod 0600 "${runtime_dir}"/* + + - name: Materialize job-scoped VIF credential + if: inputs.environment == 'production' + shell: bash + env: + DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} + run: | + set -euo pipefail + : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" + runtime_dir="${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${runtime_dir}" ]] || { echo "Refusing to reuse a job-scoped VIF runtime directory." >&2; exit 1; } + install -d -m 0700 "${runtime_dir}" + umask 077 + printf '%s' "${DEPLOY_VIF_DB_PASSWORD}" > "${runtime_dir}/vif-db-password" + chmod 0600 "${runtime_dir}/vif-db-password" - name: Prepare deployment bundle shell: bash @@ -38,36 +113,58 @@ jobs: DEPLOY_VIF_DB_NETWORK: ${{ secrets.DEPLOY_VIF_DB_NETWORK }} DEPLOY_VIF_DB_NAME: ${{ secrets.DEPLOY_VIF_DB_NAME }} DEPLOY_VIF_DB_USER: ${{ secrets.DEPLOY_VIF_DB_USER }} - DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} + DEPLOY_BRIO_STAGING_DB_NETWORK: ${{ secrets.DEPLOY_BRIO_STAGING_DB_NETWORK }} run: | set -euo pipefail + umask 077 deploy_env="${{ inputs.environment }}" : "${DEPLOY_CATWLK_DB_NETWORK:?set DEPLOY_CATWLK_DB_NETWORK environment secret}" : "${DEPLOY_LE_PETIT_COIN_DB_NETWORK:?set DEPLOY_LE_PETIT_COIN_DB_NETWORK environment secret}" + if [[ "${deploy_env}" == "canary" ]]; then + : "${DEPLOY_BRIO_STAGING_DB_NETWORK:?set DEPLOY_BRIO_STAGING_DB_NETWORK canary environment secret}" + if [[ "${DEPLOY_BRIO_STAGING_DB_NETWORK}" != "makepad_brio_staging_db" ]]; then + echo "DEPLOY_BRIO_STAGING_DB_NETWORK must be makepad_brio_staging_db." >&2 + exit 1 + fi + fi if [[ "${deploy_env}" == "production" ]]; then : "${DEPLOY_VIF_DB_NETWORK:?set DEPLOY_VIF_DB_NETWORK production environment secret}" - : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" DEPLOY_VIF_DB_NAME="${DEPLOY_VIF_DB_NAME:-vif}" DEPLOY_VIF_DB_USER="${DEPLOY_VIF_DB_USER:-vif}" fi - bundle_root="${RUNNER_TEMP}/bundle" - mkdir -p "${bundle_root}/config" "${bundle_root}/scripts" "${bundle_root}/envs/${{ inputs.environment }}" + bundle_root="${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -e "${bundle_root}" ]]; then + echo "Refusing to reuse an existing job-scoped deployment bundle: ${bundle_root}" >&2 + exit 1 + fi + mkdir -p "${bundle_root}/bootstrap" "${bundle_root}/config" "${bundle_root}/scripts" "${bundle_root}/envs/${{ inputs.environment }}" cp compose.yml "${bundle_root}/compose.yml" cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh" cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh" + cp scripts/run-brio-encrypted-backup.sh "${bundle_root}/scripts/run-brio-encrypted-backup.sh" + cp scripts/run-brio-encrypted-backup-loop.sh "${bundle_root}/scripts/run-brio-encrypted-backup-loop.sh" + cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh" + cp scripts/deploy-brio-canary-postgres.sh "${bundle_root}/scripts/deploy-brio-canary-postgres.sh" + cp scripts/brio-db-transaction.sh "${bundle_root}/scripts/brio-db-transaction.sh" + cp scripts/ensure-brio-tmp-cleaner.sh "${bundle_root}/scripts/ensure-brio-tmp-cleaner.sh" + cp bootstrap/brio-staging-app.sql "${bundle_root}/bootstrap/brio-staging-app.sql" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <> "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <> "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <&2 exit 1 fi - bundle_root="${RUNNER_TEMP}/bundle" - remote_port=${REMOTE_PORT:-22} - ssh_opts=(-o StrictHostKeyChecking=accept-new -p "${remote_port}") - scp_opts=(-o StrictHostKeyChecking=accept-new -P "${remote_port}") - remote_target="${REMOTE_USER}@${REMOTE_HOST}" - - ssh "${ssh_opts[@]}" "${remote_target}" mkdir -p "${REMOTE_DIR}/config" "${REMOTE_DIR}/scripts" "${REMOTE_DIR}/envs/${{ inputs.environment }}" - scp "${scp_opts[@]}" "${bundle_root}/compose.yml" "${remote_target}:${REMOTE_DIR}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/config/runtrace-pg_hba.conf" "${remote_target}:${REMOTE_DIR}/config/runtrace-pg_hba.conf" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup.sh" - scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup-loop.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup-loop.sh" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.db" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.deploy" - - ssh "${ssh_opts[@]}" "${remote_target}" bash -se -- "${REMOTE_DIR}" "${STACK_NAME}" "${{ inputs.environment }}" <<'EOF' - set -euo pipefail - remote_dir=$1 - stack_name=$2 - deploy_env=$3 - env_deploy="${remote_dir}/envs/${deploy_env}/.env.deploy" - db_env="${remote_dir}/envs/${deploy_env}/.env.db" - db_network=$(grep '^MAKEPAD_POSTGRES_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - le_petit_coin_db_network=$(grep '^MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_root_user=$(grep '^POSTGRES_USER=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_root_password_file=$(grep '^MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_tls_cert_config=$(grep '^MAKEPAD_POSTGRES_TLS_CERT_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_tls_key_secret=$(grep '^MAKEPAD_POSTGRES_TLS_KEY_SECRET=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_runtrace_hba_config=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) - runtrace_backup_path=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) - runtrace_backup_password_file=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) - postgres_ca_cert_file=$(grep '^MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) - vif_enabled=0 - if [[ "${deploy_env}" == "production" ]]; then - vif_enabled=1 - vif_db_network=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_name=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NAME=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_user=$(grep '^MAKEPAD_POSTGRES_VIF_DB_USER=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_password=$(grep '^MAKEPAD_POSTGRES_VIF_DB_PASSWORD=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - fi - : "${db_network:?MAKEPAD_POSTGRES_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${le_petit_coin_db_network:?MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${postgres_image:?POSTGRES_IMAGE is missing or empty in ${db_env}}" - : "${postgres_root_user:?POSTGRES_USER is missing or empty in ${db_env}}" - : "${postgres_root_password_file:?MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" - : "${postgres_tls_cert_config:?MAKEPAD_POSTGRES_TLS_CERT_CONFIG is missing or empty in ${db_env}}" - : "${postgres_tls_key_secret:?MAKEPAD_POSTGRES_TLS_KEY_SECRET is missing or empty in ${db_env}}" - : "${postgres_runtrace_hba_config:?MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG is missing or empty in ${db_env}}" - if [[ "${deploy_env}" == "production" ]]; then - : "${runtrace_backup_path:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH is missing or empty in ${db_env}}" - : "${runtrace_backup_password_file:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" - : "${postgres_ca_cert_file:?MAKEPAD_POSTGRES_CA_CERT_HOST_PATH is missing or empty in ${db_env}}" - fi - if [[ ! -s "${postgres_root_password_file}" ]]; then - echo "PostgreSQL superuser password file is missing or empty: ${postgres_root_password_file}" >&2 - exit 1 - fi - if ! docker config inspect "${postgres_tls_cert_config}" >/dev/null 2>&1; then - echo "PostgreSQL TLS certificate config does not exist: ${postgres_tls_cert_config}" >&2 + if [[ ! "${REMOTE_DIR}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+$ ]] \ + || [[ "${REMOTE_DIR}" == *"/../"* || "${REMOTE_DIR}" == *"/.." || "${REMOTE_DIR}" == *"/./"* || "${REMOTE_DIR}" == *"/." || "${REMOTE_DIR}" == *"//"* ]]; then + echo "DEPLOY_REMOTE_DIR must be a normalized application path below /srv or /opt." >&2 exit 1 fi - if ! docker secret inspect "${postgres_tls_key_secret}" >/dev/null 2>&1; then - echo "PostgreSQL TLS private-key secret does not exist: ${postgres_tls_key_secret}" >&2 + bundle_root="${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + private_key_file="${ssh_directory}/id_ed25519" + known_hosts_file="${ssh_directory}/known_hosts" + [[ -r "${private_key_file}" && -r "${known_hosts_file}" ]] || { + echo "Job-scoped SSH material is missing or unreadable." >&2 exit 1 - fi - if [[ "${deploy_env}" == "production" ]]; then - if [[ ! -d "${runtrace_backup_path}" || -L "${runtrace_backup_path}" ]]; then - echo "Runtrace backup path must be a pre-provisioned non-symlink directory: ${runtrace_backup_path}" >&2 - exit 1 - fi - backup_directory_mode=$(stat -c '%a' "${runtrace_backup_path}") - backup_directory_uid=$(stat -c '%u' "${runtrace_backup_path}") - if [[ "${backup_directory_mode}" != "700" || "${backup_directory_uid}" != "70" ]]; then - echo "Runtrace backup path must be owned by uid 70 with mode 0700: ${runtrace_backup_path}" >&2 - exit 1 - fi - if [[ ! -s "${runtrace_backup_password_file}" || -L "${runtrace_backup_password_file}" ]]; then - echo "Runtrace backup credential must be a non-empty, non-symlink file: ${runtrace_backup_password_file}" >&2 - exit 1 - fi - backup_password_mode=$(stat -c '%a' "${runtrace_backup_password_file}") - backup_password_uid=$(stat -c '%u' "${runtrace_backup_password_file}") - if [[ "${backup_password_mode}" != "400" || "${backup_password_uid}" != "70" ]]; then - echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2 - exit 1 - fi - if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then - echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2 - exit 1 - fi - postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") - if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then - echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 - exit 1 - fi - fi - hba_path="${remote_dir}/config/runtrace-pg_hba.conf" - hba_sha256=$(sha256sum "${hba_path}" | awk '{print $1}') - if docker config inspect "${postgres_runtrace_hba_config}" >/dev/null 2>&1; then - deployed_hba_sha256=$(docker config inspect "${postgres_runtrace_hba_config}" --format '{{index .Spec.Labels "content-sha256"}}') - if [[ "${deployed_hba_sha256}" != "${hba_sha256}" ]]; then - echo "PostgreSQL HBA config ${postgres_runtrace_hba_config} does not match the repository policy. Create a new versioned config name and update MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG." >&2 - exit 1 - fi - else - docker config create --label "content-sha256=${hba_sha256}" "${postgres_runtrace_hba_config}" "${hba_path}" >/dev/null - fi - if [[ "${vif_enabled}" == "1" ]]; then - : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" - : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" - : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" - fi - - ensure_encrypted_overlay_network() { - local network_name=$1 - if docker network inspect "${network_name}" >/dev/null 2>&1; then - local driver scope encrypted - driver=$(docker network inspect "${network_name}" --format '{{.Driver}}') - scope=$(docker network inspect "${network_name}" --format '{{.Scope}}') - encrypted=$(docker network inspect "${network_name}" --format '{{index .Options "encrypted"}}') - if [[ "${driver}" != "overlay" || "${scope}" != "swarm" || "${encrypted}" != "true" ]]; then - echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted, then rerun this deployment." >&2 - exit 1 - fi - return - fi - docker network create --driver overlay --attachable --opt encrypted "${network_name}" >/dev/null } + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -p "${remote_port}") + scp_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${known_hosts_file}" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${private_key_file}" -P "${remote_port}") + remote_target="${REMOTE_USER}@${REMOTE_HOST}" + remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_bundle_q %q "${remote_bundle}" + printf -v remote_parent_q %q "${REMOTE_DIR}/.deploy" + # The normalized, job-scoped paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "install -d -m 0755 ${remote_parent_q} && if [ -e ${remote_bundle_q} ]; then echo 'Refusing existing remote deployment bundle.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + scp "${scp_opts[@]}" -r "${bundle_root}/." "${remote_target}:${remote_bundle}/" + # Install the host-side expiry guard before any job credential is transferred. + printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" + printf -v db_env_q %q "${remote_bundle}/envs/${{ inputs.environment }}/.env.db" + # The job-scoped paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0755 ${cleaner_q} && ${cleaner_q} ${db_env_q}" - ensure_encrypted_overlay_network "${db_network}" - ensure_encrypted_overlay_network "${le_petit_coin_db_network}" - if [[ "${vif_enabled}" == "1" ]]; then - ensure_encrypted_overlay_network "${vif_db_network}" - export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" + remote_script="${remote_bundle}/scripts/deploy-postgres-stack.sh" + if [[ "${{ inputs.environment }}" == "canary" ]]; then + runtime_dir="${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime_dir="/tmp/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_runtime_q %q "${remote_runtime_dir}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "if [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote runtime directory.' >&2; exit 1; fi && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}"/* "${remote_target}:${remote_runtime_dir}/" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0755 ${remote_bundle_q}/scripts/brio-db-transaction.sh && chmod 0600 ${remote_runtime_q}/postgres-superuser-password ${remote_runtime_q}/brio-staging-app-password ${remote_runtime_q}/brio-staging-backup-password ${remote_runtime_q}/postgres-ca.pem ${remote_runtime_q}/postgres-server-cert.pem ${remote_runtime_q}/postgres-server-key.pem ${remote_runtime_q}/brio-backup-recipient-cert.pem" + remote_script="${remote_bundle}/scripts/deploy-brio-canary-postgres.sh" + else + runtime_dir="${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_runtime_dir="/tmp/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v remote_runtime_q %q "${remote_runtime_dir}" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "if [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote VIF runtime directory.' >&2; exit 1; fi && install -d -m 0700 ${remote_runtime_q}" + scp "${scp_opts[@]}" "${runtime_dir}/vif-db-password" "${remote_target}:${remote_runtime_dir}/vif-db-password" + # The fixed job path is escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${remote_target}" "chmod 0600 ${remote_runtime_q}/vif-db-password" fi - - export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" - export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" - docker compose \ - --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ - --env-file "${env_deploy}" \ - -f "${remote_dir}/compose.yml" \ - -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ - config > "${remote_dir}/stack.yml" - - docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" - - if [[ "${vif_enabled}" != "1" ]]; then - exit 0 + printf -v remote_script_q %q "${remote_script}" + printf -v stack_name_q %q "${STACK_NAME}" + printf -v deploy_env_q %q "${{ inputs.environment }}" + # Values are intentionally expanded locally and shell-escaped with %q. + # shellcheck disable=SC2029 + if [[ "${{ inputs.environment }}" == "canary" ]]; then + ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${remote_runtime_q}" + else + ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${deploy_env_q} ${remote_runtime_q}" fi - postgres_ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --network "${vif_db_network}" \ - -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres -c "select 1" >/dev/null 2>&1; then - postgres_ready=1 - break - fi - sleep 2 - done - if [[ "${postgres_ready}" != "1" ]]; then - echo "Postgres did not become reachable via makepad-postgres-vif on ${vif_db_network} after 60 seconds." >&2 + - name: Remove remote job-scoped deployment material + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} + REMOTE_DIR: ${{ secrets.DEPLOY_REMOTE_DIR }} + run: | + set -euo pipefail + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_directory}/id_ed25519" && -r "${ssh_directory}/known_hosts" ]] || exit 0 + if [[ ! "${REMOTE_DIR}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+$ ]] \ + || [[ "${REMOTE_DIR}" == *"/../"* || "${REMOTE_DIR}" == *"/.." || "${REMOTE_DIR}" == *"/./"* || "${REMOTE_DIR}" == *"/." || "${REMOTE_DIR}" == *"//"* ]]; then + echo "Refusing cleanup with invalid DEPLOY_REMOTE_DIR." >&2 exit 1 fi + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_directory}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_directory}/id_ed25519" -p "${remote_port}") + if [[ "${{ inputs.environment }}" == "canary" ]]; then + runtime_dir="/tmp/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + else + runtime_dir="/tmp/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + fi + remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf -v runtime_dir_q %q "${runtime_dir}" + printf -v remote_bundle_q %q "${remote_bundle}" + # The fixed job paths are escaped locally with printf %q. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${runtime_dir_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${runtime_dir_q} ]; then rm -f ${runtime_dir_q}/postgres-superuser-password ${runtime_dir_q}/brio-staging-app-password ${runtime_dir_q}/brio-staging-backup-password ${runtime_dir_q}/postgres-ca.pem ${runtime_dir_q}/postgres-server-cert.pem ${runtime_dir_q}/postgres-server-key.pem ${runtime_dir_q}/brio-backup-recipient-cert.pem ${runtime_dir_q}/vif-db-password; if [ -f ${runtime_dir_q}/RECOVERY_REQUIRED ] && [ ! -L ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${runtime_dir_q} -depth -delete; fi; elif [ -e ${runtime_dir_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" - docker run --rm --network "${vif_db_network}" \ - -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ - "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ - -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres \ - -v ON_ERROR_STOP=1 \ - -v vif_db="${vif_db_name}" \ - -v vif_user="${vif_db_user}" \ - -v vif_password="${vif_db_password}" <<'SQL' - SELECT format('CREATE ROLE %I LOGIN', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec - SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec - SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec - SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') - WHERE EXISTS ( - SELECT 1 - FROM pg_database d - JOIN pg_roles r ON r.oid = d.datdba - WHERE d.datname = :'vif_db' - AND r.rolname <> :'vif_user' - ) \gexec - SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec - SQL - EOF + - name: Remove job-scoped deployment material + if: always() + shell: bash + run: | + set -euo pipefail + for cleanup_target in \ + "${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + "${RUNNER_TEMP}/postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"; do + case "${cleanup_target}" in + "${RUNNER_TEMP}"/postgres-deploy-*|"${RUNNER_TEMP}"/postgres-brio-canary-runtime-*|"${RUNNER_TEMP}"/postgres-brio-vif-runtime-*) + if [[ -L "${cleanup_target}" ]]; then + echo "Refusing to remove cleanup symlink: ${cleanup_target}" >&2 + exit 1 + elif [[ -d "${cleanup_target}" ]]; then + find "${cleanup_target}" -mindepth 1 -delete + rmdir "${cleanup_target}" + elif [[ -e "${cleanup_target}" ]]; then + echo "Expected a cleanup directory: ${cleanup_target}" >&2 + exit 1 + fi + ;; + *) echo "Refusing to remove unexpected cleanup path: ${cleanup_target}" >&2; exit 1 ;; + esac + done diff --git a/.github/workflows/release-brio-identity-db.yml b/.github/workflows/release-brio-identity-db.yml new file mode 100644 index 0000000..ecdcf97 --- /dev/null +++ b/.github/workflows/release-brio-identity-db.yml @@ -0,0 +1,224 @@ +name: Release Brio Identity Database +run-name: Release Brio DB deployment ${{ inputs.postgres_deployment_run_id }}/${{ inputs.postgres_deployment_run_attempt }} + +on: + workflow_dispatch: + inputs: + postgres_deployment_run_id: + description: Exact completed Deploy Brio Identity Database run ID + required: true + type: string + postgres_deployment_run_attempt: + description: Exact completed deployment run attempt + required: true + type: string + +concurrency: + group: brio-identity-database-release-orchestrator + cancel-in-progress: false + +permissions: + contents: read + +jobs: + attest: + name: protected-cross-repository-attestation + runs-on: [self-hosted, linux, x64, makepad] + environment: release-brio-identity-db + timeout-minutes: 45 + steps: + - name: Require protected main release context + shell: bash + env: + POSTGRES_RUN_ID: ${{ inputs.postgres_deployment_run_id }} + POSTGRES_RUN_ATTEMPT: ${{ inputs.postgres_deployment_run_attempt }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "Makepad-fr/postgres" && "${GITHUB_REF}" == "refs/heads/main" ]] || { + echo "The Brio database release orchestrator is restricted to Makepad-fr/postgres main." >&2 + exit 1 + } + [[ "${POSTGRES_RUN_ID}" =~ ^[1-9][0-9]*$ && "${POSTGRES_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] || { + echo "Deployment run and attempt must be positive integers." >&2 + exit 1 + } + + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + + - name: Validate deployment evidence and dispatch exact Keycloak verifier + id: verify + shell: bash + env: + RELEASE_ORCHESTRATOR_TOKEN: ${{ secrets.KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN }} + POSTGRES_RUN_ID: ${{ inputs.postgres_deployment_run_id }} + POSTGRES_RUN_ATTEMPT: ${{ inputs.postgres_deployment_run_attempt }} + run: | + set -euo pipefail + : "${RELEASE_ORCHESTRATOR_TOKEN:?release-brio-identity-db must define KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN}" + release_token=${RELEASE_ORCHESTRATOR_TOKEN} + unset RELEASE_ORCHESTRATOR_TOKEN + work="${RUNNER_TEMP}/brio-db-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${work}" && ! -L "${work}" ]] || { echo "Refusing to reuse release workspace." >&2; exit 1; } + install -d -m 0700 "${work}" + umask 077 + api=https://api.github.com + headers=(--header 'Accept: application/vnd.github+json' --header 'X-GitHub-Api-Version: 2022-11-28') + github_api() { + # Supply authentication through curl's stdin-only config. The + # bearer value is never written to the persistent self-hosted + # runner filesystem or exposed in argv. + printf 'header = "Authorization: Bearer %s"\n' "${release_token}" | \ + curl --config - "$@" + } + fetch_complete_listing() { + local endpoint=$1 collection=$2 destination=$3 page=1 collected=0 returned total page_file + local pages="${work}/pages-${collection}-$RANDOM" + install -d -m 0700 "${pages}" + while ((page <= 1000)); do + page_file="${pages}/$(printf '%04d' "${page}").json" + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}${endpoint}&page=${page}" > "${page_file}" + read -r returned total < <(python3 - "${page_file}" "${collection}" <<'PY' + import json, sys + value=json.load(open(sys.argv[1], encoding="utf-8")); rows=value.get(sys.argv[2]); total=value.get("total_count") + if not isinstance(rows, list) or isinstance(total, bool) or not isinstance(total, int) or total < 0: + raise SystemExit("GitHub listing page has an invalid shape") + print(len(rows), total) + PY + ) + collected=$((collected + returned)) + ((collected <= total)) || { echo "GitHub listing grew or duplicated while paginating." >&2; return 1; } + if ((collected == total)); then break; fi + ((returned > 0)) || { echo "GitHub listing ended before total_count." >&2; return 1; } + page=$((page + 1)) + done + ((collected == total && page <= 1000)) || { echo "GitHub listing exceeded the pagination safety bound." >&2; return 1; } + python3 - "${destination}" "${collection}" "${pages}"/*.json <<'PY' + import json, pathlib, sys + destination=pathlib.Path(sys.argv[1]); field=sys.argv[2]; pages=[json.load(open(path, encoding="utf-8")) for path in sys.argv[3:]] + totals={page.get("total_count") for page in pages}; rows=[row for page in pages for row in page.get(field, [])] + ids=[row.get("id") for row in rows] + if len(totals) != 1 or totals != {len(rows)} or any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in ids) or len(ids) != len(set(ids)): + raise SystemExit("GitHub paginated listing is incomplete, changed, or duplicated") + destination.write_text(json.dumps({"total_count":len(rows),field:rows}, sort_keys=True, separators=(",", ":"))) + PY + find "${pages}" -mindepth 1 -delete + rmdir "${pages}" + } + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/runs/${POSTGRES_RUN_ID}" > "${work}/postgres-run.json" + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/workflows/deploy-brio-identity-db.yml" > "${work}/postgres-workflow.json" + fetch_complete_listing "/repos/Makepad-fr/postgres/actions/runs/${POSTGRES_RUN_ID}/artifacts?per_page=100" \ + artifacts "${work}/postgres-artifacts.json" + read -r deployment_artifact_id postgres_head_sha < <( + python3 scripts/verify-brio-release-evidence.py postgres-run \ + "${work}/postgres-run.json" "${work}/postgres-workflow.json" "${work}/postgres-artifacts.json" \ + "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" + ) + github_api --fail --silent --show-error --location "${headers[@]}" \ + "${api}/repos/Makepad-fr/postgres/actions/artifacts/${deployment_artifact_id}/zip" \ + > "${work}/postgres-evidence.zip" + python3 scripts/verify-brio-release-evidence.py postgres-evidence \ + "${work}/postgres-evidence.zip" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" + + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/git/ref/heads/main" > "${work}/keycloak-main.json" + keycloak_sha=$(python3 scripts/verify-brio-release-evidence.py keycloak-main "${work}/keycloak-main.json") + dispatch_started=$(date -u +%Y-%m-%dT%H:%M:%SZ) + python3 - "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" "${keycloak_sha}" \ + > "${work}/dispatch.json" <<'PY' + import json, sys + print(json.dumps({"ref":"main","inputs":{ + "postgres_deployment_run_id":sys.argv[1], + "postgres_deployment_run_attempt":sys.argv[2], + "postgres_deployment_head_sha":sys.argv[3], + "keycloak_release_sha":sys.argv[4], + }}, separators=(",", ":"))) + PY + code=$(github_api --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --request POST "${headers[@]}" --data-binary "@${work}/dispatch.json" \ + "${api}/repos/Makepad-fr/keycloak/actions/workflows/verify-brio-database.yml/dispatches") + [[ "${code}" == 204 ]] || { echo "Keycloak verifier dispatch returned HTTP ${code}." >&2; exit 1; } + + verifier_run_id= + for _ in $(seq 1 60); do + fetch_complete_listing "/repos/Makepad-fr/keycloak/actions/workflows/verify-brio-database.yml/runs?branch=main&event=workflow_dispatch&per_page=100" \ + workflow_runs "${work}/keycloak-runs.json" + verifier_run_id=$(python3 scripts/verify-brio-release-evidence.py verifier-run-select \ + "${work}/keycloak-runs.json" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${keycloak_sha}" "${dispatch_started}") + [[ -z "${verifier_run_id}" ]] || break + sleep 10 + done + [[ -n "${verifier_run_id}" ]] || { echo "The exact Keycloak verifier run was not found." >&2; exit 1; } + + verifier_status= + verifier_conclusion= + for _ in $(seq 1 180); do + github_api --fail --silent --show-error "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/actions/runs/${verifier_run_id}" > "${work}/keycloak-run.json" + read -r verifier_status verifier_conclusion < <(python3 - "${work}/keycloak-run.json" <<'PY' + import json, sys + run=json.load(open(sys.argv[1], encoding="utf-8")) + print(run.get("status") or "missing", run.get("conclusion") or "pending") + PY + ) + [[ "${verifier_status}" != completed ]] || break + sleep 10 + done + [[ "${verifier_status}" == completed && "${verifier_conclusion}" == success ]] || { + echo "Keycloak verifier did not complete successfully." >&2 + exit 1 + } + python3 scripts/verify-brio-release-evidence.py verifier-run \ + "${work}/keycloak-run.json" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${keycloak_sha}" "${verifier_run_id}" + fetch_complete_listing "/repos/Makepad-fr/keycloak/actions/runs/${verifier_run_id}/artifacts?per_page=100" \ + artifacts "${work}/keycloak-artifacts.json" + attestation_artifact_id=$(python3 scripts/verify-brio-release-evidence.py attestation-artifact \ + "${work}/keycloak-artifacts.json" "${verifier_run_id}" 1) + github_api --fail --silent --show-error --location "${headers[@]}" \ + "${api}/repos/Makepad-fr/keycloak/actions/artifacts/${attestation_artifact_id}/zip" \ + > "${work}/keycloak-attestation.zip" + python3 scripts/verify-brio-release-evidence.py attestation \ + "${work}/keycloak-attestation.zip" "${POSTGRES_RUN_ID}" "${POSTGRES_RUN_ATTEMPT}" "${postgres_head_sha}" \ + "${verifier_run_id}" 1 "${keycloak_sha}" + unset release_token + { + echo "verifier_run_id=${verifier_run_id}" + echo "keycloak_sha=${keycloak_sha}" + echo "postgres_sha=${postgres_head_sha}" + } >> "${GITHUB_OUTPUT}" + + - name: Record validated external attestation + shell: bash + run: | + set -euo pipefail + { + echo '### Brio identity database release attested' + echo + echo "- PostgreSQL deployment: \`${{ inputs.postgres_deployment_run_id }}/${{ inputs.postgres_deployment_run_attempt }}\`" + echo "- PostgreSQL commit: \`${{ steps.verify.outputs.postgres_sha }}\`" + echo "- Keycloak verifier run: \`${{ steps.verify.outputs.verifier_run_id }}/1\`" + echo "- Keycloak commit: \`${{ steps.verify.outputs.keycloak_sha }}\`" + echo '- Artifact source: exact Keycloak verifier output (not synthesized by this workflow)' + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Remove release material + if: always() + shell: bash + run: | + set -euo pipefail + work="${RUNNER_TEMP}/brio-db-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + case "${work}" in "${RUNNER_TEMP}"/brio-db-release-*) ;; *) exit 1 ;; esac + if [[ -L "${work}" ]]; then + echo "Refusing to remove a symlinked release workspace." >&2 + exit 1 + elif [[ -d "${work}" ]]; then + find "${work}" -mindepth 1 -delete + rmdir "${work}" + elif [[ -e "${work}" ]]; then + echo "Release workspace has an unexpected type." >&2 + exit 1 + fi diff --git a/.github/workflows/verify-keycloak-cohort-restores.yml b/.github/workflows/verify-keycloak-cohort-restores.yml new file mode 100644 index 0000000..c161bb1 --- /dev/null +++ b/.github/workflows/verify-keycloak-cohort-restores.yml @@ -0,0 +1,265 @@ +name: Verify Keycloak Cohort Restore Compatibility + +on: + workflow_dispatch: + inputs: + keycloak_release_sha: + description: Exact protected Keycloak main SHA whose pinned 26.7.3 runtime must start all five restored databases + required: true + type: string + +permissions: + contents: read + +concurrency: + group: postgres-keycloak-cohort-restore + cancel-in-progress: false + +jobs: + verify: + name: restore-five-databases-and-start-keycloak + runs-on: [self-hosted, linux, x64, makepad] + environment: keycloak-cohort-restore + timeout-minutes: 120 + steps: + - name: Enforce protected immutable release inputs + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + KEYCLOAK_SOURCE_TOKEN: ${{ secrets.KEYCLOAK_COHORT_SOURCE_TOKEN }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == Makepad-fr/postgres && "${GITHUB_REF}" == refs/heads/main ]] + [[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ && "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + [[ "${GITHUB_SHA}" =~ ^[a-f0-9]{40}$ && "${KEYCLOAK_RELEASE_SHA}" =~ ^[a-f0-9]{40}$ ]] + : "${KEYCLOAK_SOURCE_TOKEN:?set the dedicated Keycloak Contents-read token}" + observed_main=$(GH_TOKEN="${KEYCLOAK_SOURCE_TOKEN}" gh api \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + repos/Makepad-fr/keycloak/git/ref/heads/main --jq .object.sha) + [[ "${observed_main}" == "${KEYCLOAK_RELEASE_SHA}" ]] || { + echo "The requested Keycloak release is not the exact current protected main SHA." >&2 + exit 1 + } + + - name: Check out protected PostgreSQL producer + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + ref: ${{ github.sha }} + fetch-depth: 1 + + - name: Check out exact Keycloak release source + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + repository: Makepad-fr/keycloak + token: ${{ secrets.KEYCLOAK_COHORT_SOURCE_TOKEN }} + persist-credentials: false + ref: ${{ inputs.keycloak_release_sha }} + path: _keycloak + fetch-depth: 1 + + - name: Verify exact protected sources and runtime pin + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + [[ "$(git rev-parse HEAD)" == "${GITHUB_SHA}" ]] + [[ "$(git -C _keycloak rev-parse HEAD)" == "${KEYCLOAK_RELEASE_SHA}" ]] + grep -Fxq 'KEYCLOAK_UPSTREAM_VERSION=26.7.3' _keycloak/envs/production/.env.keycloak + grep -Fxq 'KEYCLOAK_IMAGE=dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' \ + _keycloak/envs/production/.env.keycloak + + - name: Ensure interrupted cohort material expires on the release host + shell: bash + run: | + set -euo pipefail + ./scripts/ensure-brio-tmp-cleaner.sh envs/canary/.env.db + systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer + systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer + + - name: Configure isolated SSH and registry state + shell: bash + env: + SSH_PRIVATE_KEY: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS }} + DHI_USERNAME: ${{ secrets.DHI_REGISTRY_USERNAME }} + DHI_PASSWORD: ${{ secrets.DHI_REGISTRY_PASSWORD }} + run: | + set -euo pipefail + : "${SSH_PRIVATE_KEY:?}" "${SSH_KNOWN_HOSTS:?}" "${DHI_USERNAME:?}" "${DHI_PASSWORD:?}" + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ ! -e "${job_root}" && ! -L "${job_root}" ]] + install -d -m 0700 "${job_root}/ssh" "${job_root}/backups" "${job_root}/docker" + umask 077 + printf '%s\n' "${SSH_PRIVATE_KEY}" >"${job_root}/ssh/id_ed25519" + printf '%s\n' "${SSH_KNOWN_HOSTS}" >"${job_root}/ssh/known_hosts" + chmod 0600 "${job_root}/ssh/"* + printf '%s' "${DHI_PASSWORD}" | docker --config "${job_root}/docker" login dhi.io --username "${DHI_USERNAME}" --password-stdin >/dev/null + + - name: Capture exact live cohort backups + shell: bash + env: + REMOTE_HOST: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_HOST }} + REMOTE_PORT: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PORT }} + REMOTE_USER: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_USER }} + run: | + set -euo pipefail + : "${REMOTE_HOST:?}" "${REMOTE_USER:?}" + [[ "${REMOTE_USER}" != root ]] + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${job_root}/ssh/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${job_root}/ssh/id_ed25519" -p "${remote_port}") + target="${REMOTE_USER}@${REMOTE_HOST}" + helper_digest=$(sha256sum scripts/capture-keycloak-cohort-backups.sh | cut -d' ' -f1) + cleaner_digest=$(sha256sum scripts/clean-keycloak-cohort-resources.sh | cut -d' ' -f1) + # The reviewed digests are intentionally expanded by this client. + # shellcheck disable=SC2029 + [[ $(ssh "${ssh_opts[@]}" "${target}" "probe ${helper_digest} ${cleaner_digest}") == cohort-capture-contract-ok ]] + # The SSH key is restricted by an operator-installed forced command. + # Only exact probe/capture/fetch/cleanup verbs and this reviewed helper + # digest can reach Docker on the database host. + # Run identity and digest are intentionally client-expanded forced-command arguments. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" "capture ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${helper_digest} ${cleaner_digest}" + for database in keycloak_catwlk keycloak_makepad keycloak_runtrace keycloak_vestiaire keycloak_vif; do + partial="${job_root}/backups/.${database}.dump.partial" + # The forced-command fetch tuple is intentionally client-expanded. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${target}" \ + "fetch ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${database}.dump ${helper_digest} ${cleaner_digest}" >"${partial}" + [[ -s "${partial}" && ! -L "${partial}" ]] + mv -T "${partial}" "${job_root}/backups/${database}.dump" + done + [[ $(find "${job_root}/backups" -mindepth 1 -maxdepth 1 -type f -name '*.dump' | wc -l) -eq 5 ]] + + - name: Build exact checked-out Catwlk provider runtime + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + base='dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' + tag="makepad/keycloak-catwlk-cohort:${KEYCLOAK_RELEASE_SHA}" + manifest_runtime=$(python3 - <<'PY' + import json + value=json.load(open("_keycloak/instances/catwlk/manifest.json", encoding="utf-8"))["runtime"] + assert value == {"source_dir":"providers/catwlk-email-router","dockerfile":"providers/catwlk-email-router/Dockerfile","image":"makepad/keycloak-catwlk"} + print(value["dockerfile"]) + PY + ) + DOCKER_BUILDKIT=1 docker build \ + --build-arg "KEYCLOAK_BASE_IMAGE=${base}" \ + --label "org.makepad.keycloak.base-image=${base}" \ + --label "org.makepad.keycloak.source-sha=${KEYCLOAK_RELEASE_SHA}" \ + --label org.makepad.keycloak.runtime=catwlk-custom-provider \ + --file "_keycloak/${manifest_runtime}" --tag "${tag}" _keycloak >/dev/null + image_id=$(docker image inspect "${tag}" --format '{{.Id}}') + [[ "${image_id}" =~ ^sha256:[a-f0-9]{64}$ ]] + docker run --rm --network none --read-only --entrypoint sh "${tag}" -euc \ + 'test -s /opt/keycloak/providers/catwlk-keycloak-email-router.jar' + printf '%s\n' "${tag}" > "/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/catwlk-image-tag" + + - name: Restore every database and start its reviewed Keycloak runtime + shell: bash + run: | + set -euo pipefail + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + export DOCKER_CONFIG="${job_root}/docker" + docker pull 'dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' >/dev/null + docker pull 'postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' >/dev/null + catwlk_image=$(<"${job_root}/catwlk-image-tag") + scripts/restore-keycloak-cohort-backups.sh "${job_root}/backups" "${job_root}/results" _keycloak "${catwlk_image}" + + - name: Build and validate canonical cohort evidence + shell: bash + env: + KEYCLOAK_RELEASE_SHA: ${{ inputs.keycloak_release_sha }} + run: | + set -euo pipefail + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + evidence_dir="${job_root}/evidence" + install -d -m 0700 "${evidence_dir}" + python3 - "${job_root}/results/instances.json" "${evidence_dir}/keycloak-cohort-restore-evidence.json" \ + "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT}" "${GITHUB_SHA}" "${KEYCLOAK_RELEASE_SHA}" <<'PY' + import json + import pathlib + import sys + + results = json.loads(pathlib.Path(sys.argv[1]).read_text()) + payload = { + "schema": "makepad.keycloak-cohort-restore-evidence.v2", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/verify-keycloak-cohort-restores.yml", + "postgres_run_id": int(sys.argv[3]), + "postgres_run_attempt": int(sys.argv[4]), + "postgres_head_sha": sys.argv[5], + "postgres_ref": "refs/heads/main", + "keycloak_release_sha": sys.argv[6], + "keycloak_base_image": "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "catwlk_runtime_image_id": results["catwlk_runtime_image_id"], + "fingerprint_schema": "makepad.keycloak-config-fingerprint.v2", + "keycloak_upstream_version": "26.7.3", + "result": "restored-databases-compatible", + "instances": results["instances"], + } + pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + PY + chmod 0600 "${evidence_dir}/keycloak-cohort-restore-evidence.json" + python3 scripts/verify-keycloak-cohort-evidence.py \ + "${evidence_dir}/keycloak-cohort-restore-evidence.json" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --head-sha "${GITHUB_SHA}" --keycloak-release-sha "${KEYCLOAK_RELEASE_SHA}" + [[ $(find "${evidence_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n') == keycloak-cohort-restore-evidence.json ]] + + - name: Publish immutable five-database compatibility evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/postgres-keycloak-cohort-${{ github.run_id }}-${{ github.run_attempt }}/evidence/keycloak-cohort-restore-evidence.json + if-no-files-found: error + retention-days: 35 + + - name: Remove remote and local cohort material + if: always() + shell: bash + env: + REMOTE_HOST: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_HOST }} + REMOTE_PORT: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_PORT }} + REMOTE_USER: ${{ secrets.KEYCLOAK_COHORT_DB_SSH_USER }} + run: | + set -euo pipefail + cleanup_status=0 + job_root="/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ -r "${job_root}/ssh/id_ed25519" && -r "${job_root}/ssh/known_hosts" && -n "${REMOTE_HOST}" && -n "${REMOTE_USER}" ]]; then + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${job_root}/ssh/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${job_root}/ssh/id_ed25519" -p "${remote_port}") + if [[ -f scripts/capture-keycloak-cohort-backups.sh && -f scripts/clean-keycloak-cohort-resources.sh ]]; then + helper_digest=$(sha256sum scripts/capture-keycloak-cohort-backups.sh | cut -d' ' -f1) + cleaner_digest=$(sha256sum scripts/clean-keycloak-cohort-resources.sh | cut -d' ' -f1) + # The forced-command cleanup tuple is intentionally client-expanded. + # shellcheck disable=SC2029 + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" \ + "cleanup ${GITHUB_RUN_ID} ${GITHUB_RUN_ATTEMPT} ${helper_digest} ${cleaner_digest}" || cleanup_status=1 + else + echo "Reviewed capture/cleaner sources are unavailable for constrained remote cleanup; the host TTL remains authoritative." >&2 + cleanup_status=1 + fi + fi + if [[ -f "${job_root}/catwlk-image-tag" && ! -L "${job_root}/catwlk-image-tag" ]]; then + catwlk_image=$(<"${job_root}/catwlk-image-tag") + if [[ "${catwlk_image}" == "makepad/keycloak-catwlk-cohort:${{ inputs.keycloak_release_sha }}" ]]; then + docker image rm "${catwlk_image}" >/dev/null 2>&1 || cleanup_status=1 + else + echo "Refusing to remove an unexpected Catwlk image tag." >&2 + cleanup_status=1 + fi + fi + if [[ -d "${job_root}" && ! -L "${job_root}" ]]; then + find "${job_root}" -mindepth 1 -delete || cleanup_status=1 + rmdir "${job_root}" || cleanup_status=1 + elif [[ -e "${job_root}" || -L "${job_root}" ]]; then + echo "Refusing cleanup of an unsafe local cohort path." >&2 + cleanup_status=1 + fi + exit "${cleanup_status}" diff --git a/README.md b/README.md index edac409..9feb6ad 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,23 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `envs/canary/.env.db`: canary PostgreSQL settings - `envs/production/compose.yml`: production Swarm overrides - `envs/production/.env.db`: production PostgreSQL settings -- `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the Vif, Makepad, Vestiaire, and Runtrace Keycloak databases +- `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the existing Vif, Makepad, Vestiaire, and Runtrace Keycloak databases - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database +- `bootstrap/brio-staging-app.sql`: idempotent SQL bootstrap for the Brio staging application database +- `bootstrap/keycloak-brio-staging.sql`: targeted idempotent bootstrap for Brio's Keycloak database - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets +- `scripts/run-brio-encrypted-backup.sh`: streaming CMS-encrypted backup for one allowlisted Brio database +- `scripts/verify-brio-encrypted-restore.sh`: destructive two-database Brio restore verification +- `scripts/deploy-postgres-stack.sh`: checked-in remote Swarm preflight, deployment, convergence, and database-provisioning entrypoint +- `scripts/deploy-brio-canary-postgres.sh`: prevalidated, snapshot-backed canary host/Swarm transaction, Brio bootstrap, access probes, and backup verification +- `scripts/deploy-brio-identity-db-host.sh`: guarded standalone DB-VM snapshot, durable recovery evidence, rollback, HBA update, Brio identity bootstrap, access probes, and backup verification +- `scripts/ensure-brio-tmp-cleaner.sh`: persistent host guard that expires abandoned Brio deployment directories after three hours while preserving recovery-marked runs +- `scripts/install-keycloak-cohort-capture-host.sh`: idempotent root installer for the digest-bound cohort SSH forced command and persistent file/Docker-resource cleaner +- `scripts/clean-keycloak-cohort-resources.sh`: fail-closed cleanup of expired labeled cohort containers, networks, and dump directories +- `scripts/test-brio-deployment-failures.sh`: isolated behavioral fault tests for promotion, stack, signal, rollback, evidence-retention, and symlink failures ## Networks @@ -30,17 +41,22 @@ Production also joins the VIF-specific external overlay network: - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` +Canary additionally joins Brio's staging-only, application-owned network: + +- `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` (`makepad_brio_staging_db`) + The manual deploy workflow sources these Compose variables from environment secrets with this mapping: - `${MAKEPAD_POSTGRES_DB_NETWORK}` <- `DEPLOY_CATWLK_DB_NETWORK` - `${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK}` <- `DEPLOY_LE_PETIT_COIN_DB_NETWORK` - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK` production only +- `${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK}` <- `DEPLOY_BRIO_STAGING_DB_NETWORK` canary only -Every database network must be an attachable Swarm overlay created with `--opt encrypted`. The deploy workflow creates new networks with encryption and fails closed when an existing network is not encrypted. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and `--opt encrypted`, then redeploy PostgreSQL and the dependent stacks. +Every database network must be an attachable Swarm overlay created with `--opt encrypted=true`; Brio's dedicated network must additionally be `--internal`. PostgreSQL owns that shared Brio database network and labels it with `com.makepad.owner=Makepad-fr/postgres`, environment `staging`, instance `brio`, and purpose `database`; Brio consumes it but must not create or relabel it. The explicit encryption value matters: Docker records a valueless `--opt encrypted` as an empty option rather than the required `true`. The deploy workflow creates new networks with those properties and fails closed when an existing network does not match. To migrate an existing network, schedule a maintenance window, stop its dependent stacks, remove and recreate the network with the same name and required options, then redeploy PostgreSQL and the dependent stacks. Application network topology is owned by the consuming application repositories. New Keycloak instances keep their own DB-facing Docker networks in the Keycloak repository and connect to this PostgreSQL server through the configured DB endpoint. -When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. Canary does not attach the VIF network. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead. The production override publishes PostgreSQL port 5432 in host mode so certificate-verified clients on the Keycloak and application VMs retain that endpoint while PostgreSQL remains pinned to the database node. +When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. Brio staging attaches only through `makepad_brio_staging_db` and verifies the alias `makepad-postgres-brio-staging`. Canary does not attach the VIF network. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead. The production override publishes PostgreSQL port 5432 in host mode so certificate-verified clients on the Keycloak and application VMs retain that endpoint while PostgreSQL remains pinned to the database node. ## Node Labels @@ -52,18 +68,72 @@ docker node update --label-add infra.makepad.postgres=true ## Deployment -Use the manual GitHub Actions workflow in this repository. +Use the manual GitHub Actions workflow in this repository for Swarm canary and +ordinary shared-stack deployments. Brio's topology is deliberately split: +`brio_staging` belongs to the canary application Swarm, while +`keycloak_brio_staging` is provisioned only by the separate `Deploy Brio +Identity Database` workflow on the standalone database VM. The production +Swarm override contains no Brio identity backup service and must never be used +to bootstrap or back up the Brio Keycloak database. + +CI and deployment workflows use the existing Makepad self-hosted Linux runner +with the exact `self-hosted`, `Linux`, `X64`, and `makepad` labels. Pull-request +jobs reject forks before a runner is assigned, check out the exact candidate +head, receive no protected environment, and run with read-only repository +permissions. Deployment workflows reject every Git ref except `main`; configure +their GitHub environments with the same deployment-branch restriction and +required reviewers. + +Swarm deployments share one target-wide concurrency group across canary and +production. Every run uploads to a unique +`/.deploy/postgres--` bundle; no workflow writes +a shared remote `stack.yml`. Before credentials are uploaded, both deployment +paths install or validate a restricted, restartable host cleaner. It removes +only abandoned top-level `/tmp/postgres-brio-*` directories older than three +hours, covering runner interruption or loss of the SSH session in addition to +the workflows' immediate `always()` cleanup. The dedicated database VM currently runs standalone Docker Compose rather than -joining the application Swarm. On that host, deploy the same TLS and backup -policy with `compose.host.yml` after provisioning the certificate, key, CA, -password files, backup directory, and committed HBA policy: +joining the application Swarm. Do not run the following recovery-level command +for the ordinary Brio identity rollout; the protected workflow supplies and +validates the reviewed inputs: ```bash +: "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set to the DB certificate SAN hostname}" docker compose --env-file envs/production/.env.db -f compose.host.yml config docker compose --env-file envs/production/.env.db -f compose.host.yml up -d --pull always --remove-orphans --wait ``` +The standalone DB-VM deployment additionally requires +`MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST` to be exported as the exact DB +VM host or IP present in the PostgreSQL server certificate SAN. The encrypted +identity backup refuses any connection mode other than `verify-full`. +The current host private key contract is numeric owner/group `70:70`, mode +`0400`; the deploy preflight requires that exact verified live form without +copying the key into CI. +The workflow also refuses a Swarm node, pins Compose project `postgres`, and +requires the existing `postgres-postgres-1` container to have the exact Compose +project/service labels, host networking, pinned image, healthy state, and +`/var/lib/makepad/postgres:/var/lib/postgresql/data` read-write bind. It requires an exact restart +acknowledgement and confirmation of a current successful encrypted restore +test, and renders the Keycloak application HBA rule to one verified egress +`/32`. Candidate Compose, HBA, SQL, and scripts are staged and prevalidated. +Immediately before the first managed-file mutation, the deploy snapshots the +exact prior Compose inputs, HBA, scripts, affected backup credentials, and the +complete standalone Brio identity backup directory. Any +error or `HUP`/`INT`/`TERM` after that boundary restores the snapshot, recreates +the prior Compose project, and requires the exact PostgreSQL target to return +healthy. The candidate promotion recreates the fixed Compose project so the +updated bind-mounted HBA is loaded, then verifies TLS identity, plaintext rejection, +cross-database rejection, read-only backup access, and a newly published CMS +backup. The rollback is disarmed only after all those local probes and the fresh +encrypted backup pass. Local DB-VM probes and the host-network backup container set +`PGHOSTADDR=127.0.0.1` while retaining the certificate host in `PGHOST`, so +transport is deterministically local while `verify-full` still checks the +declared DNS or IP SAN. The identity backup HBA rule is correspondingly limited +to `127.0.0.1/32`; only the Keycloak application role receives the separately +rendered egress `/32` rule. + The host deployment preserves the existing host-network endpoint used by Keycloak while requiring TLS and SCRAM for `runtrace` and `keycloak_runtrace`. Other databases keep their existing SCRAM transport policy. @@ -74,32 +144,317 @@ Required environment secrets: - `DEPLOY_SSH_PORT` - `DEPLOY_SSH_USER` - `DEPLOY_SSH_PRIVATE_KEY` +- `DEPLOY_SSH_KNOWN_HOSTS` - `DEPLOY_REMOTE_DIR` - `DEPLOY_STACK_NAME` - `DEPLOY_CATWLK_DB_NETWORK` - `DEPLOY_LE_PETIT_COIN_DB_NETWORK` +Canary additionally requires: + +- `DEPLOY_BRIO_STAGING_DB_NETWORK` set exactly to `makepad_brio_staging_db`; + the workflow rejects alternate names so Brio and PostgreSQL cannot drift onto + disconnected look-alike networks +- `POSTGRES_CANARY_SUPERUSER_PASSWORD` +- `POSTGRES_CA_PEM` +- `POSTGRES_SERVER_CERT_PEM`, whose SAN includes + `makepad-postgres-brio-staging` +- `POSTGRES_SERVER_KEY_PEM`, matching that certificate +- `BRIO_STAGING_DB_PASSWORD` +- `BRIO_STAGING_BACKUP_DB_PASSWORD` +- `BRIO_BACKUP_RECIPIENT_CERT_PEM`, containing only the public recovery + certificate + +The canary workflow materializes these only in mode-0700 job directories with +mode-0600 files, transfers them to one job-scoped remote `/tmp` directory, +and removes local and remote job material in `always()` cleanup steps. Before +any mutation it validates exact managed destinations (including every existing +path component), immutable object digests, all network contracts, the current +stack identity, and both Compose and Swarm renderings. It atomically publishes +a root-owned transaction journal under +`/var/lib/makepad/postgres-recovery/brio-canary/-` containing the +exact managed-file, database, ACL/default-privilege, backup-directory, and +Swarm service pre-state. Replacements are staged beside their destinations and +promoted with same-filesystem renames. Any later error or signal restores the +database and host snapshot, rolls back each changed pre-existing service once +and verifies its exact prior Spec hash, and removes only objects labeled with +the current deployment ID. A SIGKILL leaves the journal for mandatory recovery +at the beginning of the next deployment. A failed compensation leaves +`RECOVERY_REQUIRED`; normal workflow and TTL cleanup preserve it. Passwords +are read from mounted files inside short-lived containers and never placed in +Docker or `psql` command arguments. + +Canary acceptance uses a hardened one-shot backup container and does not force +a second update of the PostgreSQL or backup Swarm service. Because `docker +stack deploy` does not prune a service removed from a Compose file, a legacy +`${stack}_keycloak_brio_staging_backup` service causes deployment to fail +closed. Inventory its exact Spec and retire that exact service only through a +separately reviewed operation after the standalone `keycloak_brio_staging` +backup has been accepted; never use a broad stack prune for this migration. + +The protected `staging-brio-identity-db` GitHub environment requires: + +- secrets `BRIO_IDENTITY_DB_DEPLOY_SSH_HOST`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_PORT`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_USER`, + `BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY`, and + `BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS` +- secrets `KEYCLOAK_BRIO_STAGING_DB_PASSWORD`, + `KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD`, and + `BRIO_BACKUP_RECIPIENT_CERT_PEM` +- variable `BRIO_IDENTITY_DB_HOSTNAME`, equal to the server-certificate SAN + used by Keycloak for the standalone DB endpoint; the current value is + `65.21.134.125`, which is present as an IP SAN in the deployed certificate +- variable `BRIO_KEYCLOAK_DB_SOURCE_CIDR`, equal to Keycloak's verified single + database egress `/32` (currently `88.99.209.165/32`); the Brio WireGuard + tunnel is for Keycloak-to-MailDev traffic and is not a database route + +Protect that environment with required reviewers. Dispatch `Deploy Brio +Identity Database` from `main`, type +`restart-standalone-postgres-for-brio-staging`, and confirm the restore gate +only after recording a successful restore of the current encrypted backup. +The SSH account must be non-root but has Docker access, which is root-equivalent +and therefore restricted to this reviewed deployment path. + +All Brio workflow-supplied deployment, release, and CI credentials have one +canonical Proton Pass item and are copied only into the named, protected +GitHub environment. They are not repository or organization secrets, runner +configuration, or files on a persistent runner. Existing shared PostgreSQL +production credentials keep their established Proton items; this rollout does +not rename or duplicate them. Non-secret deployment constants remain protected +GitHub environment variables. The exact Brio inventory is: + +| Canonical Proton Pass item | Protected GitHub environment | Exact mirrored fields | +| --- | --- | --- | +| `Hetzner Database Server makepad` | `canary`, `production`, `staging-brio-identity-db`, and `keycloak-cohort-restore` | canonical SSH fields `DEPLOY_SSH_HOST`, `DEPLOY_SSH_PORT`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_PRIVATE_KEY`, `DEPLOY_SSH_KNOWN_HOSTS`; mirror the same reviewed values under the workflow aliases `BRIO_IDENTITY_DB_DEPLOY_SSH_HOST`, `BRIO_IDENTITY_DB_DEPLOY_SSH_PORT`, `BRIO_IDENTITY_DB_DEPLOY_SSH_USER`, `BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY`, `BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, `KEYCLOAK_COHORT_DB_SSH_USER`, `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, and `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS` only in their named environments | +| `Brio Staging - PostgreSQL` | `canary` and `staging-brio-identity-db` | secrets `POSTGRES_CANARY_SUPERUSER_PASSWORD`, `BRIO_STAGING_DB_PASSWORD`, `BRIO_STAGING_BACKUP_DB_PASSWORD`, `KEYCLOAK_BRIO_STAGING_DB_PASSWORD`, and `KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD` | +| `Brio Staging - PKI and Backup Keys` | `canary` and `staging-brio-identity-db` | secrets `POSTGRES_CA_PEM`, `POSTGRES_SERVER_CERT_PEM`, `POSTGRES_SERVER_KEY_PEM`, and public recipient certificate `BRIO_BACKUP_RECIPIENT_CERT_PEM` | +| `PostgreSQL · Brio identity release orchestrator` | `release-brio-identity-db` | secret `KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN` | +| `PostgreSQL · Keycloak cohort source reader` | `keycloak-cohort-restore` | secret `KEYCLOAK_COHORT_SOURCE_TOKEN` | +| `Makepad Docker Hardened Images` | `keycloak-cohort-restore` | canonical fields `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT`, mirrored as secrets `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD` | + +The `canary`, `production`, `staging-brio-identity-db`, and +`keycloak-cohort-restore` environments also hold reviewed non-secret constants +such as `DEPLOY_REMOTE_DIR`, stack/network names, +`BRIO_IDENTITY_DB_HOSTNAME`, and `BRIO_KEYCLOAK_DB_SOURCE_CIDR`. Store them as +environment variables, not duplicated password-vault secrets. + +Use `pass-cli` from an approved administrator workstation and stream secret +values over standard input: + +```bash +pass-cli item view --item-title '' --field '' \ + | gh secret set '' --env '' --repo 'Makepad-fr/postgres' +``` + +Never place values in command arguments, temporary files, shell history, +Actions logs, or issue text. Mirror non-secret variables with the same reviewed +reconciliation session, compare their GitHub read-back, and record only item +IDs, field names, timestamps, and non-secret fingerprints in the deployment +change record. Every listed environment, including `production`, must have +exactly one custom branch deployment policy whose type is `branch` and whose +name is exactly `main`; GitHub's generic "protected branches" option is not an +equivalent restriction. A release is blocked if an item or field is missing, +if that exact policy or required reviewers are absent, or if GitHub differs +from the reviewed Proton version. + +Audit all five policies without changing provider state: + +```bash +python3 scripts/reconcile-github-environment-main-policy.py audit +``` + +Reconcile one environment only after reviewing its current protection rules. +The helper preserves the current wait timer and required reviewers, creates +the exact `main` branch rule before removing broader custom rules, uses bounded +fail-closed pagination, and verifies the provider read-back. Applying requires +an explicit repository/environment confirmation; for production use: + +```bash +python3 scripts/reconcile-github-environment-main-policy.py apply \ + --environment production \ + --confirm Makepad-fr/postgres:production:exact-main +``` + +Run this only from an administrator workstation whose `gh` session has +environment-administration permission. The helper never reads or writes +environment secrets. + +If automatic standalone rollback cannot re-establish the exact healthy target, +the deploy script first deletes all incoming job credentials, then retains a +root-owned mode-0700 recovery bundle under +`/var/lib/makepad/postgres-recovery/brio-identity/-` and leaves a +non-secret `RECOVERY_REQUIRED` marker in the run-scoped `/tmp` directory. The +workflow and three-hour cleaner deliberately skip that marked directory. +Operators must inspect and resolve the retained evidence from the DB VM, then +remove both exact run directories only after recovery is complete. The bundle +may contain the former managed backup credential and must never be copied into +Actions artifacts or ordinary logs. + +The database-VM workflow's local probes do not prove the public route from the +Keycloak host. Release is deliberately two-phase. A successful `Deploy Brio +Identity Database` run stops after uploading exactly one 35-day artifact named +`brio-db-deployment-evidence--` containing the single canonical +`brio-db-deployment-evidence.json` file. That artifact says only that the +standalone host deployment is ready; it never claims Keycloak-path success. + +A reviewer then dispatches the separate protected `Release Brio Identity +Database` workflow with that exact PostgreSQL run ID and attempt. The protected +`release-brio-identity-db` environment supplies the dedicated +`KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN`. Its canonical credential must first be +stored in Proton Pass item `PostgreSQL · Brio identity release orchestrator`, +then mirrored only to that protected environment. Use a dedicated fine-grained +token or short-lived installation-token broker restricted to exactly +`Makepad-fr/postgres` and `Makepad-fr/keycloak`, with Metadata read, Contents +read, and Actions read/write; it must have no Administration, Environments, +Secrets, Members, Packages, Deployments, or organization-runner permission and +must never be an ambient maintainer PAT. Record the App installation or token +owner and a non-secret fingerprint in the same Proton item. The orchestrator independently requires the +PostgreSQL run to be completed successfully on `main`, fully paginates and validates the exact +artifact name, size, one-file ZIP shape, schema, commit, run, and attempt, +resolves the current exact Keycloak `main` SHA, and dispatches +`Makepad-fr/keycloak/.github/workflows/verify-brio-database.yml`. It accepts +only the exact completed verifier run and its single +`brio-db-path-attestation--` artifact; it never creates, +copies, or synthesizes an attestation itself. +The bearer credential is supplied to each API request through curl's stdin-only +configuration and is never materialized in the persistent runner filesystem. + +The protected `Verify Brio Identity Database Path` workflow has the run name +`Verify Brio DB path for PostgreSQL run ` and publishes the +`brio-db-path-ok` evidence. Its dedicated Keycloak runner verifies +`sslmode=verify-full` to `65.21.134.125`, the certificate IP SAN, exact +database/role, TLS 1.2 or 1.3, and server-observed source `88.99.209.165`; no +Keycloak database credential is granted to the PostgreSQL runner. + +Pull-request and protected-main checks run through the repository's native +`CI` workflow on the existing Makepad runner. The workflow never uses +`pull_request_target`, never grants write permissions, and never exposes a +deployment environment to pull-request code. + +## Keycloak 26.7.3 cohort restore evidence + +Before the five-realm Keycloak release, dispatch protected workflow `Verify +Keycloak Cohort Restore Compatibility` with the exact lowercase current +Keycloak protected-main SHA. There is no mutable rollout repository variable. +The workflow resolves `Makepad-fr/keycloak` main independently, checks out that +exact release, and verifies its pinned +`dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f` +runtime and upstream version `26.7.3`. + +The protected release runner captures fresh custom-format, no-owner, +no-privilege dumps of exactly `keycloak_catwlk`, `keycloak_makepad`, +`keycloak_runtrace`, `keycloak_vestiaire`, and +`keycloak_vif` from the exact healthy production Compose container. Every dump +is structurally inspected. Each is then restored into a fresh internal Docker +network and disposable PostgreSQL instance. Catwlk uses the custom DHI-derived +provider image built from the exact checked-out Keycloak release; the other four +instances use the pinned base image. Each runtime must become ready and the v2 +secret-safe fingerprints for realm settings/themes/SMTP, authentication flows, +roles/composites, clients/scopes/mappers, identity providers, components, and +required actions must remain stable. Canonical sorted rows are streamed directly +into SHA-256; raw configuration values are never written or logged. The check +fails when a required persistence table disappears. Because Keycloak's schema is +internal, a reviewed schema change requires coordinated fingerprint/evidence +schema revisions rather than a silent fallback. +No dump is uploaded as an Actions artifact, and remote and local copies are +deleted in the always-cleanup step. + +Success uploads exactly one artifact named +`keycloak-cohort-restore-evidence--` containing only canonical +`keycloak-cohort-restore-evidence.json`, schema +`makepad.keycloak-cohort-restore-evidence.v2`. It binds the exact PostgreSQL +workflow/run/attempt/main SHA, exact Keycloak release SHA/base image/version, +the immutable locally built Catwlk image ID, fingerprint schema, and the sorted +five-instance list. Each entry contains its slug, database, fresh backup SHA-256, +exact runtime identity, category and combined hashes, and `passed` restore, +Keycloak-startup, and configuration-regression statuses. The Keycloak deployment +consumer must resolve that exact completed +successful main-branch workflow run and artifact; operator assertions are not +evidence. + +The `keycloak-cohort-restore` environment is protected to `main` with required +reviewers. Provision its long-lived fields in Proton Pass first, then mirror +only to that environment: + +| Proton Pass item | Protected environment fields | +| --- | --- | +| `PostgreSQL · Keycloak cohort source reader` | `KEYCLOAK_COHORT_SOURCE_TOKEN`, dedicated token/App broker restricted to `Makepad-fr/keycloak` with Metadata and Contents read only | +| `Hetzner Database Server makepad` | canonical `DEPLOY_SSH_*` fields mirrored to `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, and `KEYCLOAK_COHORT_DB_SSH_USER`; dedicated non-root Docker-capable DB capture account only | +| `Makepad Docker Hardened Images` | canonical `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT` fields mirrored to `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD`, with read-only pull access to the exact reviewed Keycloak image | + +The source token has no Actions write, Checks, Administration, Environments, +Secrets, Deployments, or organization permissions. The DB capture account has +no interactive command. Before adding its GitHub secret, an operator runs +`scripts/install-keycloak-cohort-capture-host.sh +` as root on the database host. The installer writes +a root-owned `authorized_keys` forced command, disables forwarding and TTYs, +validates sshd, installs the reviewed capture helper, and enables the persistent +cohort cleanup timer. The workflow first proves the installed helper and +cleaner's exact SHA-256 digests, active timer, and successful last cleaner result, +then may issue only validated +`probe`/`capture`/`fetch`/`cleanup` commands. Checked-out code is never copied to +or executed on the database host. + +Run `scripts/install-keycloak-cohort-cleaner.sh` as root on the protected release +runner host as well. The persistent timer removes expired exactly labeled cohort +containers and networks plus `/tmp/postgres-keycloak-cohort-*` directories after +three hours, including after host downtime. Disposable database and Keycloak +passwords are mode-0400 mounted files rather than Docker configuration +environment values. The ordinary `always()` step still removes local and remote +material immediately after a run. + Production additionally requires: - `DEPLOY_VIF_DB_NETWORK` - `DEPLOY_VIF_DB_PASSWORD` Production can override the VIF database and role names with `DEPLOY_VIF_DB_NAME` and `DEPLOY_VIF_DB_USER`; both default to `vif`. +The VIF password is written only to a mode-0600 file inside a mode-0700, +job-scoped runtime directory. It is never persisted in `.env.deploy` or passed +through a `psql -v` argument; the stdin bootstrap reads it with `\getenv`. `DEPLOY_SSH_USER` must be a non-root deployment account with the Docker permissions needed to create overlay networks and deploy the stack. The workflow rejects `DEPLOY_SSH_USER=root`. -Before the first deployment, provision the PostgreSQL superuser password as a non-empty root-owned file on the database node. The production default path is `/etc/makepad/secrets/postgres-superuser-password`; canary uses `/etc/makepad/secrets/postgres-canary-superuser-password`. Keep the file outside the repository, set mode `0600`, and override `MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH` only when the host secret manager materializes it elsewhere. PostgreSQL receives the value through `POSTGRES_PASSWORD_FILE`, and deployment helpers mount the same file read-only instead of placing the password in command arguments or tracked environment files. - -Provision a private-CA-issued PostgreSQL server certificate before deployment. Its SANs must include every hostname clients verify, including `makepad-postgres` and the DB VM hostname used by Keycloak. Keep the unencrypted private key outside git and create versioned Swarm objects on the database manager: +Before an ordinary non-Brio deployment, provision the PostgreSQL superuser +password as a non-empty root-owned mode-0600 file on the database node. The +production default is `/etc/makepad/secrets/postgres-superuser-password`. +Canary's `/etc/makepad/secrets/postgres-canary-superuser-password` is instead +provisioned by its protected workflow environment. PostgreSQL receives the +value through `POSTGRES_PASSWORD_FILE`; helpers mount it read-only instead of +placing it in command arguments or tracked environment files. + +Provision a private-CA-issued PostgreSQL server certificate before deployment. +Its SANs must include every hostname clients verify, including +`makepad-postgres`, `makepad-postgres-brio-staging`, and the DB VM hostname used +by Keycloak. Keep the unencrypted private key outside git. Canary automatically +creates and content-labels its required `v2` Swarm objects from protected +environment secrets, or rejects an existing name whose label/content differs. +The following commands describe the equivalent recovery operation and the +production object: ```sh docker config create makepad_postgres_tls_cert_v1 /secure/path/server.crt docker secret create makepad_postgres_tls_key_v1 /secure/path/server.key +docker config create makepad_postgres_canary_tls_cert_v2 /secure/path/canary-server.crt +docker secret create makepad_postgres_canary_tls_key_v2 /secure/path/canary-server.key ``` -The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace` and `keycloak_runtrace` and requires SCRAM authentication over TLS for both; unrelated shared databases retain their current SCRAM transport policy during migration. - -The workflow deploys only the PostgreSQL stack. It validates the password file before deployment. If one of the configured database networks does not exist yet, it is created as an encrypted overlay on the manager before deployment. +The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace, Brio, and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `brio_staging`, and `keycloak_brio_staging` and requires SCRAM authentication over TLS for those databases. Each Brio application and backup role is also rejected from every database except its named target; unrelated shared databases retain their current SCRAM transport policy during migration. +The Brio HBA policy uses fresh immutable `makepad_postgres_canary_runtrace_hba_v3` +and `makepad_postgres_runtrace_hba_v3` object names; deployed `v2` objects are +historical and must never be replaced or relabelled in place. + +The workflow copies checked-in deployment entrypoints and deploys only the +PostgreSQL stack. Canary first validates the password and CA files, certificate +chain, certificate/key match, seven-day expiry margin, and exact +`makepad-postgres-brio-staging` SAN. A mismatch fails before stack deployment. +If a configured database network is absent it is created as an encrypted +overlay; existing network drift fails closed. After exact-image convergence, +the canary entrypoint runs the idempotent app bootstrap, proves the alias and +`sslmode=verify-full` connection, proves plaintext and cross-database access are +rejected, proves the backup role is read-only, then requires a new checksummed +CMS-encrypted backup before the workflow succeeds. ## Runtrace Backup And Restore @@ -127,11 +482,106 @@ scripts/verify-runtrace-restore.sh /var/lib/makepad/postgres-backups/runtrace/.dump.cms`, checksummed metadata, and a health marker +are atomically published. Connections require `sslmode=verify-full`, backups +run every six hours, and timestamp directories are retained for exactly 35 +days. + +Create the X.509 encryption recipient and its private key in an offline recovery +environment. Install only the public recipient certificate on both database +hosts at `/etc/makepad/tls/backups/brio-recipient.crt`; the recovery private key +must never be copied to a database host, backup container, repository, CI +secret, or ordinary application secret store. Provision the service inputs: + +```bash +# Canary application database host. +sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/brio-staging +sudo install -o 999 -g 999 -m 0400 /secure/path/brio-staging-backup-role-password \ + /etc/makepad/secrets/postgres-brio-app-backup-password + +# Production identity database host. +sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/keycloak-brio-staging +sudo install -o 999 -g 999 -m 0400 /secure/path/keycloak-brio-staging-backup-role-password \ + /etc/makepad/secrets/postgres-brio-identity-backup-password + +# Public encryption material only; retain the matching private key offline. +sudo install -d -o root -g root -m 0755 /etc/makepad/tls/backups +sudo install -o root -g root -m 0444 /secure/path/brio-recipient.crt \ + /etc/makepad/tls/backups/brio-recipient.crt +``` + +The canary Swarm and standalone DB-VM deploy preflights copy the tracked backup +scripts, reject symlinked inputs, require each backup directory to be owned by +uid 999 with mode 0700, require each database credential to be owned by uid 999 +with mode 0400, and require a root-owned, non-writable public recipient +certificate that remains valid for at least seven days and can create a CMS +AES-256-GCM envelope. +Each mounted credential is the password for its database-specific backup role; +never place a PostgreSQL superuser or application-owner password in either +backup credential file. The backup command refuses any role other than the +expected `brio_staging_backup` or `keycloak_brio_staging_backup` identity. + +The host paths above are local staging artifacts, not disaster-recovery +storage. Replicate every completed timestamp directory and `last-success.json` +from each host to independently administered off-host storage without +decrypting it. Preserve the 35-day encrypted retention window there and alert +before either health marker exceeds two backup intervals. + +Restore verification requires the newest application and identity timestamp +directories, an external copy of the public certificate and private key, two +explicit non-production libpq services, and a mode-0700 temporary directory. +Both libpq service names must end in `_restore_test` as an additional guard +against selecting an ordinary runtime service. +The temporary directory should be a dedicated tmpfs because it is the only +place where the verifier writes plaintext. The verifier checks the envelopes +and checksums before decrypting, validates each custom archive, restores with a +single transaction and exit-on-error, and requires `schema_migrations` plus +`communities` for Brio and `realm` for Keycloak: + +```bash +export PGSERVICEFILE=/secure/restore/postgres-restore-services.conf +export BRIO_APP_RESTORE_SERVICE=brio_app_restore_test +export BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test +export BRIO_RESTORE_RECIPIENT_CERT=/offline-recovery/brio-recipient.crt +export BRIO_RESTORE_RECIPIENT_KEY=/offline-recovery/brio-recipient.key +export BRIO_RESTORE_TEMP_ROOT=/run/brio-restore-tmpfs +export BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets +scripts/verify-brio-encrypted-restore.sh \ + /off-host/brio-staging/ \ + /off-host/keycloak-brio-staging/ +``` + +Keep the service file and recovery private key non-symlinked and accessible only +to the restore operator. Record the source timestamps, artifact checksums, +recipient-certificate fingerprint, target names, duration, result, and operator. +Off-host replication and a successful recorded restore of both databases remain +external release gates; repository tests cannot attest that those operational +steps occurred. + ## Application Databases Create one database and one dedicated user per application. -Vif, Makepad, Vestiaire, and Runtrace Keycloak use these databases and roles: +Vif, Makepad, Vestiaire, Runtrace, and Brio staging Keycloak use these databases and roles: | Application | Database | Role | | --- | --- | --- | @@ -139,6 +589,7 @@ Vif, Makepad, Vestiaire, and Runtrace Keycloak use these databases and roles: | Makepad | `keycloak_makepad` | `keycloak_makepad_app` | | Vestiaire | `keycloak_vestiaire` | `keycloak_vestiaire_app` | | Runtrace Keycloak | `keycloak_runtrace` | `keycloak_runtrace_app` | +| Brio staging Keycloak | `keycloak_brio_staging` | `keycloak_brio_staging_app` | Runtrace application persistence uses: @@ -152,7 +603,15 @@ OpenPanel application persistence uses: | --- | --- | --- | | OpenPanel app | `openpanel` | `openpanel_app` | -Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must be a PostgreSQL superuser connection URI for the target server, usually using the `postgres` role, because the bootstrap creates roles, sets passwords, creates databases, and assigns database ownership. For example: `postgres://postgres@:5432/postgres?sslmode=disable`. +Brio staging application persistence and backups use: + +| Purpose | Database | Role | +| --- | --- | --- | +| Brio staging app | `brio_staging` | `brio_staging_app` | +| Brio staging read-only backup | `brio_staging` | `brio_staging_backup` | +| Brio Keycloak read-only backup | `keycloak_brio_staging` | `keycloak_brio_staging_backup` | + +Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must be a PostgreSQL superuser connection URI for the target server, usually using the `postgres` role, because the bootstrap creates roles, sets passwords, creates databases, and assigns database ownership. Run it on the database host over a Unix-domain socket, for example: `postgresql:///postgres?host=%2Fvar%2Frun%2Fpostgresql&user=postgres`. If remote administration is unavoidable, use the certificate-SAN hostname with `sslmode=verify-full`, the issuing CA, and a protected libpq password source. Both targeted Brio bootstraps inspect their own session and refuse a remote plaintext administrator session before changing any role or password. ```bash : "${POSTGRES_ADMIN_URL:?set POSTGRES_ADMIN_URL to a PostgreSQL superuser connection URI}" @@ -162,6 +621,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must : "${KEYCLOAK_RUNTRACE_DB_PASSWORD:?set KEYCLOAK_RUNTRACE_DB_PASSWORD to a generated password}" : "${RUNTRACE_DB_PASSWORD:?set RUNTRACE_DB_PASSWORD to a generated password}" : "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}" +: "${BRIO_STAGING_DB_PASSWORD:?set BRIO_STAGING_DB_PASSWORD to a generated password}" +: "${BRIO_STAGING_BACKUP_DB_PASSWORD:?set BRIO_STAGING_BACKUP_DB_PASSWORD to a distinct generated password}" psql "$POSTGRES_ADMIN_URL" \ -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ @@ -179,11 +640,28 @@ psql "$POSTGRES_ADMIN_URL" \ -v keycloak_runtrace_app_password="$KEYCLOAK_RUNTRACE_DB_PASSWORD" \ -f bootstrap/keycloak-runtrace-app.sql +: "${KEYCLOAK_BRIO_STAGING_DB_PASSWORD:?set KEYCLOAK_BRIO_STAGING_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD:?set KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD to a distinct generated password}" +psql "$POSTGRES_ADMIN_URL" \ + -v keycloak_brio_staging_app_password="$KEYCLOAK_BRIO_STAGING_DB_PASSWORD" \ + -v keycloak_brio_staging_backup_password="$KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD" \ + -f bootstrap/keycloak-brio-staging.sql + psql "$POSTGRES_ADMIN_URL" \ -v openpanel_app_password="$OPENPANEL_DB_PASSWORD" \ -f bootstrap/openpanel-app.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v brio_staging_app_password="$BRIO_STAGING_DB_PASSWORD" \ + -v brio_staging_backup_password="$BRIO_STAGING_BACKUP_DB_PASSWORD" \ + -f bootstrap/brio-staging-app.sql ``` +The backup roles have no ownership or write privileges, default to read-only +transactions, and receive only schema usage plus `SELECT` on current and future +tables and sequences in their own database. Keep all four Brio credentials +distinct. + The current production Keycloak environments connect with the DB VM host: ```text @@ -191,8 +669,10 @@ postgres://keycloak_vif_app:@:5432/keycloak_vif?sslmode=disa postgres://keycloak_makepad_app:@:5432/keycloak_makepad?sslmode=disable postgres://keycloak_vestiaire_app:@:5432/keycloak_vestiaire?sslmode=disable postgres://keycloak_runtrace_app:@:5432/keycloak_runtrace?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt +postgres://keycloak_brio_staging_app:@:5432/keycloak_brio_staging?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt postgres://runtrace_app:@:5432/runtrace?sslmode=verify-full&sslrootcert=/etc/runtrace/postgres/ca.crt postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable +postgres://brio_staging_app:@makepad-postgres-brio-staging:5432/brio_staging?sslmode=verify-full&sslrootcert=/etc/brio/postgres/ca.crt ``` Stacks deployed through this repository's shared overlay network should use the `makepad-postgres` alias instead: @@ -219,14 +699,37 @@ The production VIF app uses its app-specific overlay alias and deploy-time provi postgres://vif:@makepad-postgres-vif:5432/vif?sslmode=disable ``` +Brio staging uses only its isolated encrypted overlay and certificate-matching alias: + +```text +postgres://brio_staging_app:@makepad-postgres-brio-staging:5432/brio_staging?sslmode=verify-full&sslrootcert=/etc/brio/postgres/ca.crt +``` + If production overrides `DEPLOY_VIF_DB_NAME` or `DEPLOY_VIF_DB_USER`, use those values in the connection URI. ## Validation +Run the static deployment checks and the disposable PostgreSQL 16 bootstrap test: + +```sh +./scripts/validate-postgres-config.sh +./scripts/test-brio-bootstrap.sh +./scripts/test-brio-encrypted-backup.sh +./scripts/test-brio-encrypted-restore.sh +./scripts/test-brio-deploy-guards.sh +./scripts/test-brio-deployment-contracts.sh +./scripts/test-brio-deployment-failures.sh +``` + Run the local static checks before opening a deployment PR: ```bash bash scripts/validate-postgres-config.sh bash scripts/test-runtrace-tls-policy.sh bash scripts/test-runtrace-backup.sh +bash scripts/test-brio-encrypted-backup.sh +bash scripts/test-brio-encrypted-restore.sh +bash scripts/test-brio-deploy-guards.sh +bash scripts/test-brio-deployment-contracts.sh +bash scripts/test-brio-deployment-failures.sh ``` diff --git a/bootstrap/brio-staging-app.sql b/bootstrap/brio-staging-app.sql new file mode 100644 index 0000000..84e011b --- /dev/null +++ b/bootstrap/brio-staging-app.sql @@ -0,0 +1,82 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. This creates the staging-only +-- Brio application role/database pair without embedding credentials. + +\if :{?brio_staging_app_password} +\else + \echo 'missing required psql variable: brio_staging_app_password' + SELECT 1 / 0; +\endif + +\if :{?brio_staging_backup_password} +\else + \echo 'missing required psql variable: brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'brio_staging_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS brio_staging_app_password_is_nonempty \gset +\if :brio_staging_app_password_is_nonempty +\else + \echo 'empty required psql variable: brio_staging_app_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'brio_staging_backup_password'), '') IS NULL THEN 'false' ELSE 'true' END AS brio_staging_backup_password_is_nonempty \gset +\if :brio_staging_backup_password_is_nonempty +\else + \echo 'empty required psql variable: brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE + WHEN inet_client_addr() IS NULL OR coalesce((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false) + THEN 'true' ELSE 'false' +END AS brio_staging_bootstrap_transport_is_secure \gset +\if :brio_staging_bootstrap_transport_is_secure +\else + \echo 'Brio staging bootstrap refuses a remote plaintext PostgreSQL session' + SELECT 1 / 0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('brio-staging-app-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'brio_staging_app') THEN + CREATE ROLE brio_staging_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'brio_staging_backup') THEN + CREATE ROLE brio_staging_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; +END; +$$; +ALTER ROLE brio_staging_app LOGIN PASSWORD :'brio_staging_app_password'; +ALTER ROLE brio_staging_app NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS; +ALTER ROLE brio_staging_backup LOGIN PASSWORD :'brio_staging_backup_password'; +ALTER ROLE brio_staging_backup NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2; +SELECT 'CREATE DATABASE brio_staging OWNER brio_staging_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'brio_staging') \gexec +SELECT 'ALTER DATABASE brio_staging OWNER TO brio_staging_app' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'brio_staging' + AND r.rolname <> 'brio_staging_app' +) \gexec +REVOKE ALL ON DATABASE brio_staging FROM PUBLIC; +REVOKE ALL ON DATABASE brio_staging FROM brio_staging_backup; +GRANT CONNECT ON DATABASE brio_staging TO brio_staging_app; +GRANT CONNECT ON DATABASE brio_staging TO brio_staging_backup; +ALTER ROLE brio_staging_backup IN DATABASE brio_staging SET default_transaction_read_only TO on; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('brio-staging-app-bootstrap')); + +\connect brio_staging +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO brio_staging_backup; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO brio_staging_backup; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE brio_staging_app IN SCHEMA public GRANT SELECT ON TABLES TO brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE brio_staging_app IN SCHEMA public GRANT SELECT ON SEQUENCES TO brio_staging_backup; diff --git a/bootstrap/keycloak-brio-staging.sql b/bootstrap/keycloak-brio-staging.sql new file mode 100644 index 0000000..6c7e3a6 --- /dev/null +++ b/bootstrap/keycloak-brio-staging.sql @@ -0,0 +1,83 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. This creates only the +-- staging-only Brio Keycloak role/database pair and never rotates credentials +-- for another Keycloak realm. + +\if :{?keycloak_brio_staging_app_password} +\else + \echo 'missing required psql variable: keycloak_brio_staging_app_password' + SELECT 1 / 0; +\endif + +\if :{?keycloak_brio_staging_backup_password} +\else + \echo 'missing required psql variable: keycloak_brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'keycloak_brio_staging_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_brio_staging_app_password_is_nonempty \gset +\if :keycloak_brio_staging_app_password_is_nonempty +\else + \echo 'empty required psql variable: keycloak_brio_staging_app_password' + SELECT 1 / 0; +\endif + +SELECT CASE WHEN NULLIF(btrim(:'keycloak_brio_staging_backup_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_brio_staging_backup_password_is_nonempty \gset +\if :keycloak_brio_staging_backup_password_is_nonempty +\else + \echo 'empty required psql variable: keycloak_brio_staging_backup_password' + SELECT 1 / 0; +\endif + +SELECT CASE + WHEN inet_client_addr() IS NULL OR coalesce((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false) + THEN 'true' ELSE 'false' +END AS keycloak_brio_staging_bootstrap_transport_is_secure \gset +\if :keycloak_brio_staging_bootstrap_transport_is_secure +\else + \echo 'Brio Keycloak bootstrap refuses a remote plaintext PostgreSQL session' + SELECT 1 / 0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('keycloak-brio-staging-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_brio_staging_app') THEN + CREATE ROLE keycloak_brio_staging_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_brio_staging_backup') THEN + CREATE ROLE keycloak_brio_staging_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + END IF; +END; +$$; +ALTER ROLE keycloak_brio_staging_app LOGIN PASSWORD :'keycloak_brio_staging_app_password'; +ALTER ROLE keycloak_brio_staging_app NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS; +ALTER ROLE keycloak_brio_staging_backup LOGIN PASSWORD :'keycloak_brio_staging_backup_password'; +ALTER ROLE keycloak_brio_staging_backup NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2; +SELECT 'CREATE DATABASE keycloak_brio_staging OWNER keycloak_brio_staging_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_brio_staging') \gexec +SELECT 'ALTER DATABASE keycloak_brio_staging OWNER TO keycloak_brio_staging_app' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'keycloak_brio_staging' + AND r.rolname <> 'keycloak_brio_staging_app' +) \gexec +REVOKE ALL ON DATABASE keycloak_brio_staging FROM PUBLIC; +REVOKE ALL ON DATABASE keycloak_brio_staging FROM keycloak_brio_staging_backup; +GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_app; +GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_backup; +ALTER ROLE keycloak_brio_staging_backup IN DATABASE keycloak_brio_staging SET default_transaction_read_only TO on; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('keycloak-brio-staging-bootstrap')); + +\connect keycloak_brio_staging +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO keycloak_brio_staging_backup; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO keycloak_brio_staging_backup; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO keycloak_brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT SELECT ON TABLES TO keycloak_brio_staging_backup; +ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT SELECT ON SEQUENCES TO keycloak_brio_staging_backup; diff --git a/compose.host.yml b/compose.host.yml index 6ab3ba7..618b606 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -81,3 +81,53 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + keycloak_brio_staging_backup: + image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} + user: "999:999" + network_mode: host + command: + - /usr/local/bin/run-brio-encrypted-backup-loop.sh + environment: + PGHOST: ${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST:?set MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST to the DB certificate SAN host or IP} + PGHOSTADDR: 127.0.0.1 + PGPORT: "5432" + PGUSER: keycloak_brio_staging_backup + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password + BRIO_BACKUP_DATABASE: keycloak_brio_staging + BRIO_BACKUP_ROOT: /backups + BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt + BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} + BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} + BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH:-/var/lib/makepad/postgres-backups/keycloak-brio-staging}:/backups" + - "${MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-brio-identity-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:-/etc/makepad/tls/backups/brio-recipient.crt}:/etc/postgresql/brio-backup-recipient.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-brio-encrypted-backup.sh}:/usr/local/bin/run-brio-encrypted-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-brio-encrypted-backup-loop.sh}:/usr/local/bin/run-brio-encrypted-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/compose.yml b/compose.yml index 2cc8077..03ca83b 100644 --- a/compose.yml +++ b/compose.yml @@ -54,6 +54,10 @@ configs: file: ./scripts/run-runtrace-backup.sh runtrace_backup_loop_script: file: ./scripts/run-runtrace-backup-loop.sh + brio_encrypted_backup_script: + file: ./scripts/run-brio-encrypted-backup.sh + brio_encrypted_backup_loop_script: + file: ./scripts/run-brio-encrypted-backup-loop.sh secrets: postgres_tls_key: diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 55be5a8..a1d759e 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -3,6 +3,16 @@ local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject +hostnossl brio_staging all all reject +hostnossl keycloak_brio_staging all all reject hostssl runtrace all all scram-sha-256 hostssl keycloak_runtrace all all scram-sha-256 +hostssl brio_staging brio_staging_app all scram-sha-256 +hostssl brio_staging brio_staging_backup all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_backup 127.0.0.1/32 scram-sha-256 +host all brio_staging_app all reject +host all brio_staging_backup all reject +host all keycloak_brio_staging_app all reject +host all keycloak_brio_staging_backup all reject host all all all scram-sha-256 diff --git a/envs/canary/.env.db b/envs/canary/.env.db index 43296e7..c5b2127 100644 --- a/envs/canary/.env.db +++ b/envs/canary/.env.db @@ -1,9 +1,23 @@ # Canary PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +BRIO_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres-canary MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-canary-superuser-password -MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v1 -MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v1 +MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_canary_tls_cert_v2 +MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_canary_tls_key_v2 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v3 +MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt +MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=/var/lib/makepad/postgres-backups/brio-staging +MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-app-backup-password +MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=/etc/makepad/tls/backups/brio-recipient.crt +MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 +MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G +MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 +MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION=128M +MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE=20m +MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES=5 diff --git a/envs/canary/compose.yml b/envs/canary/compose.yml index ffe261f..6d31ec4 100644 --- a/envs/canary/compose.yml +++ b/envs/canary/compose.yml @@ -18,6 +18,7 @@ services: update_config: parallelism: 1 order: stop-first + failure_action: rollback rollback_config: parallelism: 1 order: stop-first @@ -28,6 +29,76 @@ services: reservations: cpus: "${MAKEPAD_POSTGRES_CPU_RESERVATION:-0.25}" memory: ${MAKEPAD_POSTGRES_MEMORY_RESERVATION:-256M} + networks: + brio_staging: + aliases: + - makepad-postgres-brio-staging + + brio_staging_backup: + image: ${BRIO_BACKUP_IMAGE:?set BRIO_BACKUP_IMAGE} + user: "999:999" + command: + - /usr/local/bin/run-brio-encrypted-backup-loop.sh + environment: + PGHOST: makepad-postgres-brio-staging + PGPORT: "5432" + PGUSER: brio_staging_backup + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_BACKUP_PASSWORD_FILE: /run/secrets/postgres_backup_password + BRIO_BACKUP_DATABASE: brio_staging + BRIO_BACKUP_ROOT: /backups + BRIO_BACKUP_RECIPIENT_CERT: /etc/postgresql/brio-backup-recipient.crt + BRIO_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS:-21600} + BRIO_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS:-300} + BRIO_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "/usr/local/bin/run-brio-encrypted-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + networks: + - brio_staging + volumes: + - "${MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH:?set MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH}:/backups" + - "${MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_CA_CERT_HOST_PATH}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH:?set MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH}:/etc/postgresql/brio-backup-recipient.crt:ro" + configs: + - source: brio_encrypted_backup_script + target: /usr/local/bin/run-brio-encrypted-backup.sh + mode: 0555 + - source: brio_encrypted_backup_loop_script + target: /usr/local/bin/run-brio-encrypted-backup-loop.sh + mode: 0555 + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + deploy: + replicas: 1 + placement: + constraints: + - node.labels.infra.makepad.postgres == true + restart_policy: + condition: on-failure + delay: 30s + resources: + limits: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT:-1.0}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT:-1G} + reservations: + cpus: "${MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION:-0.1}" + memory: ${MAKEPAD_POSTGRES_BACKUP_MEMORY_RESERVATION:-128M} networks: db: @@ -36,3 +107,6 @@ networks: le_petit_coin: external: true name: ${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK} + brio_staging: + external: true + name: ${MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK:?set MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK} diff --git a/envs/production/.env.db b/envs/production/.env.db index fbe71c5..acf10b9 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -1,18 +1,34 @@ # Production PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 +BRIO_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-superuser-password +MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH=/etc/makepad/tls/postgres/server.crt +MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH=/etc/makepad/secrets/postgres-server.key +MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH=/srv/makepad/postgres/config/runtrace-pg_hba.conf MAKEPAD_POSTGRES_TLS_CERT_CONFIG=makepad_postgres_tls_cert_v1 MAKEPAD_POSTGRES_TLS_KEY_SECRET=makepad_postgres_tls_key_v1 -MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v1 +MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v3 MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=/etc/makepad/tls/postgres/ca.crt MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=/var/lib/makepad/postgres-backups/runtrace MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password MAKEPAD_POSTGRES_RUNTRACE_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_RUNTRACE_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH=/var/lib/makepad/postgres-backups/keycloak-brio-staging +MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-brio-identity-backup-password +MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=/etc/makepad/tls/backups/brio-recipient.crt +MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-runtrace-backup.sh +MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-runtrace-backup-loop.sh +MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-brio-encrypted-backup.sh +MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH=/srv/makepad/postgres/scripts/run-brio-encrypted-backup-loop.sh +MAKEPAD_POSTGRES_BRIO_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BRIO_BACKUP_RETENTION_DAYS=35 +# Standalone compose.host.yml additionally requires MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST +# to be exported as the DB VM host or IP present in the PostgreSQL server certificate SAN. MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 diff --git a/host/systemd/makepad-keycloak-cohort-cleaner.service b/host/systemd/makepad-keycloak-cohort-cleaner.service new file mode 100644 index 0000000..2dd19e7 --- /dev/null +++ b/host/systemd/makepad-keycloak-cohort-cleaner.service @@ -0,0 +1,14 @@ +[Unit] +Description=Remove expired Keycloak cohort restore material and Docker resources +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +User=root +Group=root +ExecStart=/usr/local/libexec/makepad/clean-keycloak-cohort-resources +NoNewPrivileges=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/tmp /var/run/docker.sock diff --git a/host/systemd/makepad-keycloak-cohort-cleaner.timer b/host/systemd/makepad-keycloak-cohort-cleaner.timer new file mode 100644 index 0000000..44ef37a --- /dev/null +++ b/host/systemd/makepad-keycloak-cohort-cleaner.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Periodically reconcile Keycloak cohort restore resources + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min +RandomizedDelaySec=2min +Persistent=true +Unit=makepad-keycloak-cohort-cleaner.service + +[Install] +WantedBy=timers.target diff --git a/scripts/brio-db-transaction.sh b/scripts/brio-db-transaction.sh new file mode 100755 index 0000000..bb6753d --- /dev/null +++ b/scripts/brio-db-transaction.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 3)); then + echo "Usage: brio-db-transaction.sh " >&2 + exit 2 +fi + +operation=$1 +scope=$2 +journal_dir=$3 + +case "${operation}" in prepare|restore|fingerprint) ;; *) echo "Unsupported database transaction operation." >&2; exit 2 ;; esac +case "${scope}" in + brio) + database=brio_staging + app_role=brio_staging_app + backup_role=brio_staging_backup + ;; + keycloak) + database=keycloak_brio_staging + app_role=keycloak_brio_staging_app + backup_role=keycloak_brio_staging_backup + ;; + *) echo "Unsupported Brio database transaction scope." >&2; exit 2 ;; +esac + +[[ "${journal_dir}" == /* && -d "${journal_dir}" && ! -L "${journal_dir}" ]] || { + echo "The database transaction journal must be a non-symlinked absolute directory." >&2 + exit 2 +} +: "${PGUSER:?PGUSER is required}" "${PGHOST:?PGHOST is required}" "${PGPASSWORD_FILE:?PGPASSWORD_FILE is required}" +[[ -s "${PGPASSWORD_FILE}" && ! -L "${PGPASSWORD_FILE}" ]] || { + echo "PGPASSWORD_FILE must be a non-empty regular file." >&2 + exit 2 +} +export PGPASSWORD +PGPASSWORD=$(cat "${PGPASSWORD_FILE}") +export PGAPPNAME=brio-db-transaction PSQL_HISTORY=/dev/null + +psql_base=(psql -X --no-password --set ON_ERROR_STOP=1 --quiet) + +database_exists() { + [[ $("${psql_base[@]}" --tuples-only --no-align --dbname postgres \ + --command "SELECT count(*) FROM pg_database WHERE datname = '${database}'") == 1 ]] +} + +emit_fingerprint() { + local db_exists=false role + database_exists && db_exists=true + printf 'scope\t%s\n' "${scope}" + for role in "${app_role}" "${backup_role}"; do + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'role', rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, + rolcanlogin, rolreplication, rolconnlimit, coalesce(rolvaliduntil::text, ''), + rolbypassrls, coalesce(rolpassword, '') + FROM pg_authid WHERE rolname = '${role}';" + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'global-setting', r.rolname, setting + FROM pg_db_role_setting s JOIN pg_roles r ON r.oid = s.setrole + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE s.setdatabase = 0 AND r.rolname = '${role}' ORDER BY setting;" + done + printf 'database-exists\t%s\n' "${db_exists}" + if [[ "${db_exists}" == true ]]; then + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname postgres --command " + SELECT 'database', d.datname, r.rolname, d.datallowconn, d.datconnlimit, + coalesce(d.datacl::text, '') + FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = '${database}'; + SELECT 'setting', role_name, setting + FROM ( + SELECT r.rolname AS role_name, unnest(s.setconfig) AS setting + FROM pg_db_role_setting s + JOIN pg_roles r ON r.oid = s.setrole + JOIN pg_database d ON d.oid = s.setdatabase + WHERE d.datname = '${database}' AND r.rolname IN ('${app_role}', '${backup_role}') + ) q ORDER BY role_name, setting;" + "${psql_base[@]}" --tuples-only --no-align --field-separator $'\t' --dbname "${database}" --command " + SELECT 'namespace', n.nspname, coalesce(n.nspacl::text, '') + FROM pg_namespace n WHERE n.nspname = 'public'; + SELECT 'relation', n.nspname, c.relname, c.relkind, coalesce(c.relacl::text, '') + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r','p','v','m','f','S') + ORDER BY n.nspname, c.relname, c.relkind; + SELECT 'default-acl', owner_name, schema_name, d.defaclobjtype, + grantee_name, x.privilege_type, x.is_grantable + FROM pg_default_acl d + JOIN pg_roles owner_role ON owner_role.oid = d.defaclrole + LEFT JOIN pg_namespace n ON n.oid = d.defaclnamespace + CROSS JOIN LATERAL aclexplode(d.defaclacl) x + LEFT JOIN pg_roles grantee_role ON grantee_role.oid = x.grantee + CROSS JOIN LATERAL (VALUES(owner_role.rolname, coalesce(n.nspname, ''), + CASE WHEN x.grantee = 0 THEN 'PUBLIC' ELSE grantee_role.rolname END)) + names(owner_name, schema_name, grantee_name) + WHERE owner_role.rolname = '${app_role}' AND coalesce(n.nspname, 'public') = 'public' + ORDER BY owner_name, schema_name, d.defaclobjtype, grantee_name, x.privilege_type, x.is_grantable;" + fi +} + +write_role_restore() { + local role=$1 output=$2 exists + exists=$("${psql_base[@]}" --tuples-only --no-align --dbname postgres \ + --command "SELECT count(*) FROM pg_authid WHERE rolname = '${role}'") + if [[ "${exists}" == 0 ]]; then + printf 'DROP ROLE IF EXISTS %s;\n' "${role}" >> "${output}" + return + fi + "${psql_base[@]}" --tuples-only --no-align --dbname postgres --command " + SELECT format( + 'ALTER ROLE %I WITH %s %s %s %s %s %s CONNECTION LIMIT %s %s PASSWORD %L VALID UNTIL %L;', + rolname, + CASE WHEN rolsuper THEN 'SUPERUSER' ELSE 'NOSUPERUSER' END, + CASE WHEN rolinherit THEN 'INHERIT' ELSE 'NOINHERIT' END, + CASE WHEN rolcreaterole THEN 'CREATEROLE' ELSE 'NOCREATEROLE' END, + CASE WHEN rolcreatedb THEN 'CREATEDB' ELSE 'NOCREATEDB' END, + CASE WHEN rolcanlogin THEN 'LOGIN' ELSE 'NOLOGIN' END, + CASE WHEN rolreplication THEN 'REPLICATION' ELSE 'NOREPLICATION' END, + rolconnlimit, + CASE WHEN rolbypassrls THEN 'BYPASSRLS' ELSE 'NOBYPASSRLS' END, + rolpassword, + coalesce(rolvaliduntil::text, 'infinity')) + FROM pg_authid WHERE rolname = '${role}'; + SELECT format('UPDATE pg_authid SET rolvaliduntil = NULL WHERE rolname = %L;', rolname) + FROM pg_authid WHERE rolname = '${role}' AND rolvaliduntil IS NULL; + SELECT format('ALTER ROLE %I RESET ALL;', rolname) + FROM pg_authid WHERE rolname = '${role}'; + SELECT format('ALTER ROLE %I SET %I TO %L;', r.rolname, + split_part(setting, '=', 1), substr(setting, strpos(setting, '=') + 1)) + FROM pg_db_role_setting s JOIN pg_roles r ON r.oid = s.setrole + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE s.setdatabase = 0 AND r.rolname = '${role}' ORDER BY setting;" >> "${output}" +} + +prepare_journal() { + local restore_tmp="${journal_dir}/.restore.sql.tmp" fingerprint_tmp="${journal_dir}/.prestate.fingerprint.tmp" + local restore_file="${journal_dir}/restore.sql" fingerprint_file="${journal_dir}/prestate.fingerprint" + for path in "${restore_file}" "${fingerprint_file}" "${restore_tmp}" "${fingerprint_tmp}"; do + [[ ! -e "${path}" && ! -L "${path}" ]] || { echo "Refusing to overwrite database journal material." >&2; exit 1; } + done + umask 077 + { + printf '%s\n' '\set ON_ERROR_STOP on' 'SET client_min_messages = warning;' + if database_exists; then + printf '\\connect %s\n' "${database}" + # These catalogs are the exact objects changed by the Brio bootstrap. The + # fixed OID/name predicates make the compensation fail closed on drift. + "${psql_base[@]}" --tuples-only --no-align --dbname "${database}" --command " + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_namespace WHERE oid = %s AND nspname = %L) THEN RAISE EXCEPTION ''namespace identity drift''; END IF; END \$do\$; UPDATE pg_namespace SET nspacl = %s WHERE oid = %s AND nspname = %L;', + oid, nspname, + CASE WHEN nspacl IS NULL THEN 'NULL' ELSE quote_literal(nspacl::text) || '::aclitem[]' END, + oid, nspname) + FROM pg_namespace WHERE nspname = 'public'; + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_class WHERE oid = %s AND relnamespace = %s AND relname = %L AND relkind = %L) THEN RAISE EXCEPTION ''relation identity drift''; END IF; END \$do\$; UPDATE pg_class SET relacl = %s WHERE oid = %s AND relnamespace = %s AND relname = %L AND relkind = %L;', + c.oid, c.relnamespace, c.relname, c.relkind, + CASE WHEN c.relacl IS NULL THEN 'NULL' ELSE quote_literal(c.relacl::text) || '::aclitem[]' END, + c.oid, c.relnamespace, c.relname, c.relkind) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r','p','v','m','f','S') + ORDER BY c.oid; + SELECT format('ALTER DEFAULT PRIVILEGES FOR ROLE %I IN SCHEMA public REVOKE ALL ON %s FROM %I;', + '${app_role}', CASE defaclobjtype WHEN 'r' THEN 'TABLES' ELSE 'SEQUENCES' END, '${backup_role}') + FROM (VALUES ('r'::\"char\"), ('S'::\"char\")) kinds(defaclobjtype); + SELECT format('ALTER DEFAULT PRIVILEGES FOR ROLE %I IN SCHEMA public GRANT %s ON %s TO %I%s;', + owner_role.rolname, string_agg(DISTINCT x.privilege_type, ', ' ORDER BY x.privilege_type), + CASE d.defaclobjtype WHEN 'r' THEN 'TABLES' ELSE 'SEQUENCES' END, + '${backup_role}', CASE WHEN bool_and(x.is_grantable) THEN ' WITH GRANT OPTION' ELSE '' END) + FROM pg_default_acl d + JOIN pg_roles owner_role ON owner_role.oid = d.defaclrole + JOIN pg_namespace n ON n.oid = d.defaclnamespace + CROSS JOIN LATERAL aclexplode(d.defaclacl) x + JOIN pg_roles grantee_role ON grantee_role.oid = x.grantee + WHERE owner_role.rolname = '${app_role}' AND n.nspname = 'public' + AND grantee_role.rolname = '${backup_role}' AND d.defaclobjtype IN ('r','S') + GROUP BY owner_role.rolname, d.defaclobjtype, x.is_grantable + ORDER BY d.defaclobjtype, x.is_grantable;" + printf '%s\n' '\connect postgres' + "${psql_base[@]}" --tuples-only --no-align --dbname postgres --command " + SELECT format('ALTER DATABASE %I OWNER TO %I;', d.datname, r.rolname) + FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = '${database}'; + SELECT format('ALTER DATABASE %I %s; ALTER DATABASE %I CONNECTION LIMIT %s;', + datname, CASE WHEN datallowconn THEN 'ALLOW_CONNECTIONS true' ELSE 'ALLOW_CONNECTIONS false' END, + datname, datconnlimit) + FROM pg_database WHERE datname = '${database}'; + SELECT format( + 'DO \$do\$ BEGIN IF NOT EXISTS (SELECT FROM pg_database WHERE oid = %s AND datname = %L) THEN RAISE EXCEPTION ''database identity drift''; END IF; END \$do\$; UPDATE pg_database SET datacl = %s WHERE oid = %s AND datname = %L;', + oid, datname, + CASE WHEN datacl IS NULL THEN 'NULL' ELSE quote_literal(datacl::text) || '::aclitem[]' END, + oid, datname) + FROM pg_database WHERE datname = '${database}'; + SELECT format('ALTER ROLE %I IN DATABASE %I RESET ALL;', r.rolname, d.datname) + FROM pg_roles r CROSS JOIN pg_database d + WHERE r.rolname IN ('${app_role}', '${backup_role}') AND d.datname = '${database}' + ORDER BY r.rolname; + SELECT format('ALTER ROLE %I IN DATABASE %I SET %I TO %L;', r.rolname, d.datname, + split_part(setting, '=', 1), substr(setting, strpos(setting, '=') + 1)) + FROM pg_db_role_setting s + JOIN pg_roles r ON r.oid = s.setrole + JOIN pg_database d ON d.oid = s.setdatabase + CROSS JOIN LATERAL unnest(s.setconfig) setting + WHERE r.rolname IN ('${app_role}', '${backup_role}') AND d.datname = '${database}' + ORDER BY r.rolname, setting;" + else + printf "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid();\n" "${database}" + printf 'DROP DATABASE IF EXISTS %s;\n' "${database}" + fi + } > "${restore_tmp}" + write_role_restore "${app_role}" "${restore_tmp}" + write_role_restore "${backup_role}" "${restore_tmp}" + emit_fingerprint > "${fingerprint_tmp}" + chmod 0600 "${restore_tmp}" "${fingerprint_tmp}" + mv -T "${restore_tmp}" "${restore_file}" + mv -T "${fingerprint_tmp}" "${fingerprint_file}" + sync -f "${journal_dir}" 2>/dev/null || sync +} + +restore_journal() { + [[ -s "${journal_dir}/restore.sql" && ! -L "${journal_dir}/restore.sql" \ + && -s "${journal_dir}/prestate.fingerprint" && ! -L "${journal_dir}/prestate.fingerprint" ]] || { + echo "The database transaction journal is incomplete." >&2 + exit 1 + } + "${psql_base[@]}" --dbname postgres --file "${journal_dir}/restore.sql" >/dev/null + local restored="${journal_dir}/.restored.fingerprint.tmp" + emit_fingerprint > "${restored}" + if ! cmp -s "${journal_dir}/prestate.fingerprint" "${restored}"; then + echo "Database compensation did not restore the exact prior Brio state." >&2 + # The fingerprint deliberately contains PostgreSQL SCRAM verifiers so the + # recovery check can prove exact credential restoration. Never emit either + # side of a mismatch: CI and remote deployment logs are not secret stores. + rm -f -- "${restored}" + exit 1 + fi + rm -f -- "${restored}" +} + +case "${operation}" in + prepare) prepare_journal ;; + restore) restore_journal ;; + fingerprint) emit_fingerprint ;; +esac diff --git a/scripts/capture-keycloak-cohort-backups.sh b/scripts/capture-keycloak-cohort-backups.sh new file mode 100755 index 0000000..20e6e1a --- /dev/null +++ b/scripts/capture-keycloak-cohort-backups.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +if (($# != 1)); then + echo "usage: capture-keycloak-cohort-backups.sh /tmp/postgres-keycloak-cohort--" >&2 + exit 2 +fi + +output_dir=$1 +run_id=${COHORT_CAPTURE_RUN_ID:-} +run_attempt=${COHORT_CAPTURE_RUN_ATTEMPT:-} +postgres_image='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +container=postgres-postgres-1 +[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ ]] || { + echo "Exact positive workflow run and attempt are required." >&2 + exit 2 +} +[[ "${output_dir}" == "/tmp/postgres-keycloak-cohort-${run_id}-${run_attempt}" ]] || { + echo "Backup output must be the exact run-scoped /tmp path." >&2 + exit 2 +} +[[ -d /tmp && ! -L /tmp && ! -e "${output_dir}" && ! -L "${output_dir}" ]] || { + echo "Refusing an unsafe or reused cohort backup output path." >&2 + exit 1 +} +for command_name in docker sha256sum; do + command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; } +done + +cleanup_on_failure() { + local status=$? + trap - EXIT HUP INT TERM + if ((status != 0)) && [[ -d "${output_dir}" && ! -L "${output_dir}" ]]; then + find "${output_dir}" -mindepth 1 -delete + rmdir "${output_dir}" + fi + exit "${status}" +} +trap cleanup_on_failure EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +umask 077 +install -d -m 0700 "${output_dir}" + +inspect=$(docker inspect "${container}" --format '{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.service"}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}') +IFS='|' read -r compose_project compose_service observed_image state health <<<"${inspect}" +[[ "${compose_project}" == postgres && "${compose_service}" == postgres && "${observed_image}" == "${postgres_image}" \ + && "${state}" == running && "${health}" == healthy ]] || { + echo "The exact healthy production PostgreSQL container is not present." >&2 + exit 1 +} + +databases=( + keycloak_catwlk + keycloak_makepad + keycloak_runtrace + keycloak_vestiaire + keycloak_vif +) +for database in "${databases[@]}"; do + exists=$(docker exec "${container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + exec psql -h 127.0.0.1 -U postgres -d postgres -v ON_ERROR_STOP=1 -Atqc \ + "SELECT count(*) FROM pg_database WHERE datname = '\''$1'\'' AND datallowconn" + ' sh "${database}") + [[ "${exists}" == 1 ]] || { echo "Required database ${database} is missing or disallows connections." >&2; exit 1; } + partial="${output_dir}/.${database}.dump.partial" + final="${output_dir}/${database}.dump" + docker exec "${container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + exec pg_dump -h 127.0.0.1 -U postgres --format=custom --compress=6 \ + --no-owner --no-privileges --dbname "$1" + ' sh "${database}" >"${partial}" + [[ -s "${partial}" && ! -L "${partial}" ]] || { echo "Empty backup for ${database}." >&2; exit 1; } + docker run --rm --read-only --network none --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${partial},dst=/backup.dump,readonly" \ + "${postgres_image}" pg_restore --list /backup.dump >/dev/null + chmod 0600 "${partial}" + mv -T "${partial}" "${final}" +done + +expected=$(printf '%s\n' "${databases[@]/%/.dump}" | sort) +observed=$(find "${output_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | sort) +[[ "${observed}" == "${expected}" ]] || { echo "Cohort backup directory has an unexpected entry set." >&2; exit 1; } +sha256sum "${output_dir}"/*.dump >/dev/null +trap - EXIT HUP INT TERM +echo "Captured and structurally validated the exact five Keycloak databases." diff --git a/scripts/clean-keycloak-cohort-resources.sh b/scripts/clean-keycloak-cohort-resources.sh new file mode 100755 index 0000000..0775639 --- /dev/null +++ b/scripts/clean-keycloak-cohort-resources.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +[[ "$(id -u)" -eq 0 ]] || { echo "The cohort resource cleaner must run as root." >&2; exit 1; } +for command_name in date docker find; do command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; }; done + +now=$(date +%s) +contract=makepad-keycloak-cohort-restore-v1 + +remove_expired_container() { + local identifier=$1 details name observed_contract expires + details=$(docker container inspect "${identifier}" --format '{{.Name}}|{{index .Config.Labels "makepad.cleanup.contract"}}|{{index .Config.Labels "makepad.cleanup.expires-epoch"}}') + IFS='|' read -r name observed_contract expires <<<"${details}" + name=${name#/} + [[ "${name}" =~ ^pg-kc-(db|app)-[1-9][0-9]*-[1-9][0-9]*-(catwlk|makepad|runtrace|vestiaire|vif)$ \ + && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { + echo "Refusing malformed labeled cohort container ${identifier}." >&2 + return 1 + } + ((expires > now)) || docker rm -f "${identifier}" >/dev/null +} + +remove_expired_network() { + local identifier=$1 details name observed_contract expires + details=$(docker network inspect "${identifier}" --format '{{.Name}}|{{index .Labels "makepad.cleanup.contract"}}|{{index .Labels "makepad.cleanup.expires-epoch"}}') + IFS='|' read -r name observed_contract expires <<<"${details}" + [[ "${name}" =~ ^pg-kc-[1-9][0-9]*-[1-9][0-9]*-(catwlk|makepad|runtrace|vestiaire|vif)$ \ + && "${observed_contract}" == "${contract}" && "${expires}" =~ ^[1-9][0-9]*$ ]] || { + echo "Refusing malformed labeled cohort network ${identifier}." >&2 + return 1 + } + ((expires > now)) || docker network rm "${identifier}" >/dev/null +} + +while IFS= read -r identifier; do [[ -z "${identifier}" ]] || remove_expired_container "${identifier}"; done \ + < <(docker container ls -aq --filter "label=makepad.cleanup.contract=${contract}") +while IFS= read -r identifier; do [[ -z "${identifier}" ]] || remove_expired_network "${identifier}"; done \ + < <(docker network ls -q --filter "label=makepad.cleanup.contract=${contract}") + +find /tmp -mindepth 1 -maxdepth 1 -type d -name 'postgres-keycloak-cohort-*' -mmin +180 \ + -exec sh -euc 'for directory do + [ ! -L "$directory" ] || continue + [ ! -f "$directory/RECOVERY_REQUIRED" ] || continue + find "$directory" -depth -delete + done' sh {} + diff --git a/scripts/deploy-brio-canary-postgres.sh b/scripts/deploy-brio-canary-postgres.sh new file mode 100755 index 0000000..e3326b2 --- /dev/null +++ b/scripts/deploy-brio-canary-postgres.sh @@ -0,0 +1,831 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 3)); then + echo "Usage: deploy-brio-canary-postgres.sh " >&2 + exit 2 +fi + +remote_dir=$1 +stack_name=$2 +runtime_dir=$3 +db_env="${remote_dir}/envs/canary/.env.db" +env_deploy="${remote_dir}/envs/canary/.env.deploy" +rollback_armed=0 +stack_mutated=0 +stack_preexisting=0 +db_mutated=0 +postgres_image= +validation_image= +declare -a missing_configs=() +declare -a missing_secrets=() +declare -a missing_networks=() +deployment_id=${runtime_dir##*/postgres-brio-canary-runtime-} +recovery_root=/var/lib/makepad/postgres-recovery/brio-canary +journal_dir="${recovery_root}/${deployment_id}" +failure_injection=${BRIO_DEPLOY_FAILURE_INJECTION:-} + +if [[ -n "${failure_injection}" ]]; then + [[ "${BRIO_DEPLOY_TEST_MODE:-}" == "isolated-container" && -f /.dockerenv ]] || { + echo "Failure injection is permitted only in the isolated deployment test container." >&2 + exit 2 + } + case "${failure_injection}" in + after-managed-file-promotion|term-after-managed-file-promotion|kill-after-managed-file-promotion|after-stack-deploy|after-bootstrap|kill-after-bootstrap|after-app-probe|after-plaintext-probe|after-nontarget-probe|after-backup-role-probe|after-backup-verification|rollback-restore) ;; + *) echo "Unsupported canary deployment failure injection." >&2; exit 2 ;; + esac +fi + +if [[ ! "${remote_dir}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+/\.deploy/postgres-[0-9]+-[0-9]+$ ]] \ + || [[ "${remote_dir}" == *"/../"* || "${remote_dir}" == *"/.." || "${remote_dir}" == *"/./"* || "${remote_dir}" == *"/." || "${remote_dir}" == *"//"* ]]; then + echo "remote-dir must be a unique /srv or /opt .deploy/postgres-- path." >&2 + exit 2 +fi +[[ "${runtime_dir}" =~ ^/tmp/postgres-brio-canary-runtime-[0-9]+-[0-9]+$ ]] || { echo "runtime-secret-dir must be a job-scoped /tmp/postgres-brio-canary-runtime-- path." >&2; exit 2; } +case "${stack_name}" in ''|*[!a-zA-Z0-9_-]*) echo "stack-name contains unsupported characters." >&2; exit 2 ;; esac + +cleanup_runtime() { + local name + for name in postgres-superuser-password brio-staging-app-password brio-staging-backup-password postgres-ca.pem postgres-server-cert.pem postgres-server-key.pem brio-backup-recipient-cert.pem; do + [[ ! -f "${runtime_dir}/${name}" || -L "${runtime_dir}/${name}" ]] || rm -f -- "${runtime_dir:?}/${name}" + done + if [[ -f "${runtime_dir}/RECOVERY_REQUIRED" && ! -L "${runtime_dir}/RECOVERY_REQUIRED" ]]; then + return + fi + if [[ -e "${runtime_dir}/RECOVERY_REQUIRED" || -L "${runtime_dir}/RECOVERY_REQUIRED" ]]; then + echo "Unsafe recovery marker type; preserving the canary runtime for operator inspection." >&2 + return 1 + fi + rm -f -- "${runtime_dir}/candidate-stack.yml" "${runtime_dir}/prior-services.list" + rm -f -- "${runtime_dir}/prior-service-spec-hashes.list" + if [[ -d "${runtime_dir}/prior-service-specs" && ! -L "${runtime_dir}/prior-service-specs" ]]; then + find "${runtime_dir}/prior-service-specs" -depth -delete + fi + rmdir -- "${runtime_dir}" 2>/dev/null || true +} + +handle_exit() { + local status=$? + trap - EXIT HUP INT TERM + if [[ "${rollback_armed}" == "1" ]]; then + if ! rollback_canary; then + journal_marker "${journal_dir}" RECOVERY_REQUIRED || true + { + printf 'deployment_id=%s\n' "${runtime_dir##*/postgres-brio-canary-runtime-}" + printf 'reason=%s\n' 'automatic-canary-rollback-failed' + } > "${runtime_dir}/RECOVERY_REQUIRED" + chmod 0600 "${runtime_dir}/RECOVERY_REQUIRED" + echo "Automatic canary rollback failed; root-protected snapshot retained in ${journal_dir}." >&2 + status=1 + fi + fi + cleanup_runtime || status=1 + exit "${status}" +} +trap handle_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +read_setting() { + local name=$1 file=$2 value + value=$(grep -E "^${name}=" "${file}" | tail -n 1 | cut -d= -f2-) + [[ -n "${value}" ]] || { echo "${name} is missing or empty in ${file}." >&2; exit 1; } + printf '%s' "${value}" +} + +run_db_transaction() { + local operation=$1 source_journal=${2:-${journal_dir}} helper + if [[ "${operation}" == prepare ]]; then helper="${remote_dir}/scripts/brio-db-transaction.sh"; else helper="${source_journal}/brio-db-transaction.sh"; fi + [[ -f "${helper}" && ! -L "${helper}" ]] || { echo "Canary database compensation helper is unavailable." >&2; return 1; } + # The Brio-only network is deliberately allowed to be absent on a first + # deployment. Journal preparation and recovery therefore use the already + # validated shared database network and its certificate SAN. + docker run --rm --network "${db_network}" \ + --mount "type=bind,src=${helper},dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + --mount "type=bind,src=${source_journal},dst=/journal" \ + --mount "type=bind,src=${superuser_host_file},dst=/run/secrets/postgres_superuser_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + -e "PGUSER=${postgres_user}" -e PGHOST=makepad-postgres \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e PGPASSWORD_FILE=/run/secrets/postgres_superuser_password \ + "${postgres_image}" /usr/local/bin/brio-db-transaction.sh "${operation}" brio /journal/database +} + +journal_marker() { + local source_journal=$1 marker=$2 + case "${marker}" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; + *) echo "Refusing unsupported canary journal marker." >&2; return 1 ;; + esac + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 1 ;; esac + [ -d /journal ] && [ ! -L /journal ] + tmp="/journal/.${MARKER}.tmp" + [ ! -e "$tmp" ] && [ ! -L "$tmp" ] + printf "%s\n" "$MARKER" > "$tmp" + chmod 0600 "$tmp" + mv -fT "$tmp" "/journal/$MARKER" + sync -f /journal 2>/dev/null || sync + ' +} + +remove_durable_journal() { + local source_journal=$1 + docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "JOURNAL=${source_journal}" "${validation_image}" sh -euc ' + case "$JOURNAL" in /var/lib/makepad/postgres-recovery/brio-canary/*) identifier=${JOURNAL##*/} ;; *) exit 1 ;; esac + case "$identifier" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$identifier" in *-*) ;; *) exit 1 ;; esac + target="/managed-var-lib${JOURNAL#/var/lib/makepad}" + [ -d "$target" ] && [ ! -L "$target" ] + find "$target" -depth -delete + sync -f /managed-var-lib/postgres-recovery/brio-canary 2>/dev/null || sync + ' +} + +journal_has_marker() { + local source_journal=$1 marker=$2 + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal,readonly" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|STACK_MUTATION_ARMED|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 2 ;; esac + [ -f "/journal/$MARKER" ] && [ ! -L "/journal/$MARKER" ] + ' >/dev/null 2>&1 +} + +rollback_canary() { + local rollback_status=0 service_name previous_spec name expected_hash current_hash + echo "Canary deployment failed after the mutation boundary; restoring exact database, Swarm specs, and managed host files." >&2 + if [[ "${db_mutated}" == 1 ]] || journal_has_marker "${journal_dir}" DATABASE_MUTATION_ARMED; then + run_db_transaction restore "${journal_dir}" || rollback_status=1 + fi + if [[ "${failure_injection}" == rollback-restore ]]; then + rollback_status=1 + else + if [[ "${stack_mutated}" == 1 ]] || journal_has_marker "${journal_dir}" STACK_MUTATION_ARMED; then + while IFS= read -r service_name; do + if ! docker run --rm --mount "type=bind,src=${journal_dir},dst=/journal,readonly" \ + -e "SERVICE=${service_name}" "${validation_image}" sh -euc 'grep -Fxq "$SERVICE" /journal/swarm/prior-services.list'; then + docker service rm "${service_name}" >/dev/null || rollback_status=1 + fi + done < <(docker stack services "${stack_name}" --format '{{.Name}}' 2>/dev/null || true) + while IFS='|' read -r service_name expected_hash; do + docker service inspect "${service_name}" >/dev/null 2>&1 || { rollback_status=1; continue; } + current_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + if [[ "${current_hash}" != "${expected_hash}" ]]; then + previous_spec=$(docker service inspect "${service_name}" --format '{{if .PreviousSpec}}present{{else}}absent{{end}}') + if [[ "${previous_spec}" == present ]]; then + docker service rollback --detach=false "${service_name}" >/dev/null || rollback_status=1 + else + rollback_status=1 + fi + fi + current_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + [[ "${current_hash}" == "${expected_hash}" ]] || rollback_status=1 + done < <(docker run --rm --mount "type=bind,src=${journal_dir},dst=/journal,readonly" \ + "${validation_image}" cat /journal/swarm/prior-service-spec-hashes.list) + fi + docker run --rm \ + --mount type=bind,src=/etc,dst=/host/etc \ + --mount type=bind,src=/var/lib,dst=/host/var/lib \ + --mount "type=bind,src=${journal_dir}/rollback,dst=/rollback,readonly" \ + "${validation_image}" sh -euc ' + backup=/host/var/lib/makepad/postgres-backups/brio-staging + if [ -d "$backup" ] && [ ! -L "$backup" ]; then + for child in "$backup"/* "$backup"/.*; do + [ -e "$child" ] || [ -L "$child" ] || continue + name=${child##*/}; case "$name" in .|..|latest|last-success.json) continue ;; esac + if ! grep -Fxq "$name" /rollback/backup-entries.list; then + case "$name" in 20??????T??????Z|.20??????T??????Z.partial) ;; *) echo "Unsafe unexpected backup entry: $name" >&2; exit 1 ;; esac + [ ! -L "$child" ] || { echo "Unexpected backup entry is a symlink." >&2; exit 1; } + find "$child" -depth -delete + fi + done + rm -f -- "$backup/latest" "$backup/last-success.json" + if [ -f /rollback/prior-latest-target ]; then + target=$(cat /rollback/prior-latest-target) + case "$target" in 20??????T??????Z) ;; *) exit 1 ;; esac + ln -s "$target" "$backup/latest" + fi + if [ -f /rollback/prior-last-success.json ]; then cp -a /rollback/prior-last-success.json "$backup/last-success.json"; fi + fi + while IFS= read -r path; do + case "$path" in + etc/makepad/secrets/postgres-canary-superuser-password|etc/makepad/tls/postgres/ca.crt|etc/makepad/secrets/postgres-brio-app-backup-password|etc/makepad/tls/backups/brio-recipient.crt) rm -f -- "/host/$path" ;; + var/lib/makepad/postgres-backups/brio-staging) + if ! rmdir -- "/host/$path" 2>/dev/null; then echo "New backup directory is not empty." >&2; exit 1; fi ;; + *) echo "Unexpected absent rollback path." >&2; exit 1 ;; + esac + done < /rollback/absent.list + tar --numeric-owner -xpf /rollback/managed.tar -C /host + ' || rollback_status=1 + fi + for object_kind in secret config network; do + while IFS= read -r name; do + [[ -n "${name}" ]] || continue + if docker "${object_kind}" inspect "${name}" >/dev/null 2>&1; then + if [[ "${object_kind}" == network ]]; then + owner=$(docker network inspect "${name}" --format '{{index .Labels "makepad.brio.deployment-id"}}') + else + owner=$(docker "${object_kind}" inspect "${name}" --format '{{index .Spec.Labels "makepad.brio.deployment-id"}}') + fi + [[ "${owner}" == "${deployment_id}" ]] && docker "${object_kind}" rm "${name}" >/dev/null || rollback_status=1 + fi + done < <(docker "${object_kind}" ls --filter "label=makepad.brio.deployment-id=${deployment_id}" --format '{{.Name}}' 2>/dev/null || true) + done + if [[ "${rollback_status}" == 0 ]]; then journal_marker "${journal_dir}" ROLLED_BACK || rollback_status=1; fi + if [[ "${rollback_status}" == 0 ]]; then remove_durable_journal "${journal_dir}" || rollback_status=1; fi + return "${rollback_status}" +} + +recover_incomplete_journals() { + local current_id="${deployment_id}" current_journal="${journal_dir}" saved_injection="${failure_injection}" + local listing state identifier + listing=$(docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib,readonly \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-canary + [ ! -e "$root" ] && exit 0 + [ -d "$root" ] && [ ! -L "$root" ] + for candidate in "$root"/* "$root"/.*; do + [ -e "$candidate" ] || continue + name=${candidate##*/}; case "$name" in .|..) continue ;; .*.staging) echo "INCOMPLETE:$name"; continue ;; esac + case "$name" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "UNSAFE:$name"; continue ;; esac + case "$name" in *-*) ;; *) echo "UNSAFE:$name"; continue ;; esac + [ -d "$candidate" ] && [ ! -L "$candidate" ] || { echo "UNSAFE:$name"; continue; } + if [ -f "$candidate/COMMITTED" ] && [ ! -L "$candidate/COMMITTED" ]; then echo "COMMITTED:$name" + elif [ -f "$candidate/ROLLED_BACK" ] && [ ! -L "$candidate/ROLLED_BACK" ]; then echo "ROLLED_BACK:$name" + elif [ -f "$candidate/IN_PROGRESS" ] && [ ! -L "$candidate/IN_PROGRESS" ]; then echo "PENDING:$name" + else echo "UNSAFE:$name"; fi + done + ') + while IFS=: read -r state identifier; do + [[ -n "${state}" ]] || continue + case "${state}" in + COMMITTED|ROLLED_BACK) remove_durable_journal "${recovery_root}/${identifier}" ;; + PENDING) + deployment_id=${identifier} + journal_dir="${recovery_root}/${identifier}" + db_mutated=0; stack_mutated=0 + journal_has_marker "${journal_dir}" DATABASE_MUTATION_ARMED && db_mutated=1 + journal_has_marker "${journal_dir}" STACK_MUTATION_ARMED && stack_mutated=1 + failure_injection= + if ! rollback_canary; then + journal_marker "${journal_dir}" RECOVERY_REQUIRED || true + echo "Interrupted canary transaction could not be recovered." >&2 + return 1 + fi + ;; + INCOMPLETE|UNSAFE) echo "Unsafe or incomplete canary journal detected: ${identifier}" >&2; return 1 ;; + *) return 1 ;; + esac + done <<< "${listing}" + deployment_id=${current_id} + journal_dir=${current_journal} + failure_injection=${saved_injection} + db_mutated=0; stack_mutated=0 +} + +postgres_image=$(read_setting POSTGRES_IMAGE "${db_env}") +validation_image=$(read_setting BRIO_BACKUP_IMAGE "${db_env}") +postgres_user=$(read_setting POSTGRES_USER "${db_env}") +superuser_host_file=$(read_setting MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH "${db_env}") +ca_host_file=$(read_setting MAKEPAD_POSTGRES_CA_CERT_HOST_PATH "${db_env}") +backup_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH "${db_env}") +recipient_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH "${db_env}") +backup_host_dir=$(read_setting MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH "${db_env}") +tls_cert_config=$(read_setting MAKEPAD_POSTGRES_TLS_CERT_CONFIG "${db_env}") +tls_key_secret=$(read_setting MAKEPAD_POSTGRES_TLS_KEY_SECRET "${db_env}") +hba_config=$(read_setting MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG "${db_env}") +brio_network=$(read_setting MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK "${env_deploy}") +db_network=$(read_setting MAKEPAD_POSTGRES_DB_NETWORK "${env_deploy}") +le_petit_coin_network=$(read_setting MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK "${env_deploy}") + +if [[ "${brio_network}" != "makepad_brio_staging_db" ]]; then + echo "Canary provisioning requires makepad_brio_staging_db." >&2 + exit 1 +fi +[[ "${superuser_host_file}" == "/etc/makepad/secrets/postgres-canary-superuser-password" \ + && "${ca_host_file}" == "/etc/makepad/tls/postgres/ca.crt" \ + && "${backup_host_file}" == "/etc/makepad/secrets/postgres-brio-app-backup-password" \ + && "${recipient_host_file}" == "/etc/makepad/tls/backups/brio-recipient.crt" \ + && "${backup_host_dir}" == "/var/lib/makepad/postgres-backups/brio-staging" \ + && "${tls_cert_config}" == "makepad_postgres_canary_tls_cert_v2" \ + && "${tls_key_secret}" == "makepad_postgres_canary_tls_key_v2" \ + && "${hba_config}" == "makepad_postgres_canary_runtrace_hba_v3" ]] || { + echo "Canary environment does not match the exact reviewed managed-path and immutable-object contract." >&2 + exit 1 +} +for network_name in "${db_network}" "${le_petit_coin_network}" "${brio_network}"; do + [[ "${network_name}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$ ]] || { + echo "Canary network name is not normalized: ${network_name}" >&2 + exit 1 + } +done +[[ "${db_network}" != "${le_petit_coin_network}" && "${db_network}" != "${brio_network}" \ + && "${le_petit_coin_network}" != "${brio_network}" ]] || { + echo "Canary database networks must be distinct." >&2 + exit 1 +} + +assert_no_symlink_components() { + local path=$1 current='' component + local -a components=() + [[ "${path}" == /* && "${path}" != *"//"* && "${path}" != *"/./"* \ + && "${path}" != */. && "${path}" != *"/../"* && "${path}" != */.. ]] || { + echo "Managed path is not a normalized absolute path: ${path}" >&2 + return 1 + } + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + [[ -n "${component}" && "${component}" != "." && "${component}" != ".." ]] || { + echo "Managed path contains an unsafe component: ${path}" >&2 + return 1 + } + current="${current}/${component}" + [[ ! -L "${current}" ]] || { + echo "Managed path contains a symlink component: ${current}" >&2 + return 1 + } + done +} +for managed_path in "${superuser_host_file}" "${ca_host_file}" "${backup_host_file}" "${recipient_host_file}" "${backup_host_dir}" \ + /var/lib/makepad/postgres-recovery "${recovery_root}"; do + assert_no_symlink_components "${managed_path}" +done +for managed_parent in \ + /etc/makepad/secrets \ + /etc/makepad/tls/postgres \ + /etc/makepad/tls/backups \ + /var/lib/makepad/postgres-backups; do + [[ -d "${managed_parent}" && ! -L "${managed_parent}" ]] || { + echo "Required exact managed parent is missing, not a directory, or a symlink: ${managed_parent}" >&2 + exit 1 + } +done +for managed_file in "${superuser_host_file}" "${ca_host_file}" "${backup_host_file}" "${recipient_host_file}"; do + [[ ! -e "${managed_file}" || -f "${managed_file}" ]] || { + echo "Managed file destination has an unexpected type: ${managed_file}" >&2 + exit 1 + } +done +[[ ! -e "${backup_host_dir}" || -d "${backup_host_dir}" ]] || { + echo "Managed backup destination has an unexpected type: ${backup_host_dir}" >&2 + exit 1 +} + +require_runtime_file() { + local path="${runtime_dir}/$1" mode + if [[ ! -s "${path}" || -L "${path}" ]]; then + echo "Required job-scoped input is missing, empty, or a symlink: $1" >&2 + exit 1 + fi + mode=$(stat -c '%a' "${path}") + if [[ "${mode}" != "600" ]]; then + echo "Job-scoped input must have mode 0600: $1" >&2 + exit 1 + fi +} + +runtime_mode=$(stat -c '%a' "${runtime_dir}") +[[ ! -L "${runtime_dir}" && "${runtime_mode}" == "700" ]] || { + echo "Job-scoped runtime directory must be a non-symlink with mode 0700." >&2 + exit 1 +} +for input in \ + postgres-superuser-password \ + brio-staging-app-password \ + brio-staging-backup-password \ + postgres-ca.pem \ + postgres-server-cert.pem \ + postgres-server-key.pem \ + brio-backup-recipient-cert.pem; do + require_runtime_file "${input}" +done +for password_file in postgres-superuser-password brio-staging-app-password brio-staging-backup-password; do + if [[ $(awk 'END { print NR }' "${runtime_dir}/${password_file}") -ne 1 ]] || grep -q $'\r' "${runtime_dir}/${password_file}"; then + echo "Password input must contain one line and no carriage return: ${password_file}" >&2 + exit 1 + fi +done +if cmp -s "${runtime_dir}/postgres-superuser-password" "${runtime_dir}/brio-staging-app-password" \ + || cmp -s "${runtime_dir}/postgres-superuser-password" "${runtime_dir}/brio-staging-backup-password" \ + || cmp -s "${runtime_dir}/brio-staging-app-password" "${runtime_dir}/brio-staging-backup-password"; then + echo "PostgreSQL superuser, Brio application, and Brio backup credentials must all be distinct." >&2 + exit 1 +fi + +command -v openssl >/dev/null 2>&1 || { echo "openssl is required." >&2; exit 1; } +openssl x509 -in "${runtime_dir}/postgres-ca.pem" -noout -checkend 604800 >/dev/null +openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -noout -checkend 604800 >/dev/null +openssl verify -purpose sslserver -CAfile "${runtime_dir}/postgres-ca.pem" "${runtime_dir}/postgres-server-cert.pem" >/dev/null +openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -noout -checkhost makepad-postgres-brio-staging >/dev/null +cert_key_hash=$(openssl x509 -in "${runtime_dir}/postgres-server-cert.pem" -pubkey -noout | openssl pkey -pubin -outform DER | sha256sum | cut -d' ' -f1) +private_key_hash=$(openssl pkey -in "${runtime_dir}/postgres-server-key.pem" -pubout -outform DER | sha256sum | cut -d' ' -f1) +[[ "${cert_key_hash}" == "${private_key_hash}" ]] || { echo "PostgreSQL TLS certificate and private key do not match." >&2; exit 1; } +if grep -q -- 'PRIVATE KEY' "${runtime_dir}/brio-backup-recipient-cert.pem" \ + || ! openssl x509 -in "${runtime_dir}/brio-backup-recipient-cert.pem" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${runtime_dir}/brio-backup-recipient-cert.pem" -out /dev/null; then + echo "Brio backup recipient must be a valid public encryption certificate with at least seven days remaining." >&2 + exit 1 +fi + +prevalidate_swarm_config() { + local name=$1 source=$2 kind=$3 digest deployed_digest actual_file + digest=$(sha256sum "${source}" | cut -d' ' -f1) + if docker config inspect "${name}" >/dev/null 2>&1; then + deployed_digest=$(docker config inspect "${name}" --format '{{index .Spec.Labels "content-sha256"}}') + [[ "${deployed_digest}" == "${digest}" ]] || { + echo "${kind} config ${name} has different content; create a new versioned name." >&2 + return 1 + } + actual_file=$(mktemp) + docker config inspect "${name}" --format '{{printf "%s" .Spec.Data}}' > "${actual_file}" + if [[ $(sha256sum "${actual_file}" | cut -d' ' -f1) != "${digest}" ]]; then + rm -f "${actual_file}" + echo "${kind} config ${name} content does not match its label." >&2 + return 1 + fi + rm -f "${actual_file}" + else + missing_configs+=("${name}|${source}|${digest}") + fi +} + +prevalidate_swarm_secret() { + local name=$1 source=$2 digest deployed_digest + digest=$(sha256sum "${source}" | cut -d' ' -f1) + if docker secret inspect "${name}" >/dev/null 2>&1; then + deployed_digest=$(docker secret inspect "${name}" --format '{{index .Spec.Labels "content-sha256"}}') + [[ "${deployed_digest}" == "${digest}" ]] || { + echo "TLS private-key secret ${name} cannot be replaced in place; create a new versioned name." >&2 + return 1 + } + else + missing_secrets+=("${name}|${source}|${digest}") + fi +} + +prevalidate_network() { + local name=$1 required_internal=$2 details + if docker network inspect "${name}" >/dev/null 2>&1; then + details=$(docker network inspect "${name}" --format '{{.Driver}} {{.Scope}} {{.Internal}} {{.Attachable}} {{index .Options "encrypted"}}') + if [[ "${required_internal}" == "true" ]]; then + [[ "${details}" == "overlay swarm true true true" ]] || { + echo "Network ${name} must be an internal, attachable, encrypted Swarm overlay; got ${details}." >&2 + return 1 + } + else + [[ "${details}" == "overlay swarm false true true" ]] || { + echo "Network ${name} must be an external, attachable, encrypted Swarm overlay; got ${details}." >&2 + return 1 + } + fi + else + missing_networks+=("${name}|${required_internal}") + fi +} + +# Validate every immutable object, network, and rendered stack before crossing +# any host or Swarm mutation boundary. +[[ -f "${remote_dir}/compose.yml" && ! -L "${remote_dir}/compose.yml" \ + && -f "${remote_dir}/envs/canary/compose.yml" && ! -L "${remote_dir}/envs/canary/compose.yml" \ + && -f "${remote_dir}/config/runtrace-pg_hba.conf" && ! -L "${remote_dir}/config/runtrace-pg_hba.conf" \ + && -x "${remote_dir}/scripts/deploy-postgres-stack.sh" && ! -L "${remote_dir}/scripts/deploy-postgres-stack.sh" \ + && -x "${remote_dir}/scripts/brio-db-transaction.sh" && ! -L "${remote_dir}/scripts/brio-db-transaction.sh" ]] || { + echo "Canary bundle inputs are missing or symlinked." >&2 + exit 1 +} + +# Recovery and first-deployment journal capture use the pre-existing shared +# PostgreSQL network, because the Brio-only network may not exist yet. Validate +# that transport boundary before a recovery helper can receive a superuser +# credential or connect to the database alias. +prevalidate_network "${db_network}" false +docker network inspect "${db_network}" >/dev/null 2>&1 || { + echo "The pre-existing shared database network ${db_network} is required for crash-safe journal preparation." >&2 + exit 1 +} + +# A SIGKILL can leave immutable objects created by the interrupted run. Recover +# its root-owned journal before inventorying those objects so the current +# attempt cannot mistake interrupted-run state for its own validated pre-state. +recover_incomplete_journals + +prevalidate_swarm_config "${tls_cert_config}" "${runtime_dir}/postgres-server-cert.pem" "TLS certificate" +prevalidate_swarm_secret "${tls_key_secret}" "${runtime_dir}/postgres-server-key.pem" +prevalidate_swarm_config "${hba_config}" "${remote_dir}/config/runtrace-pg_hba.conf" "HBA policy" +prevalidate_network "${le_petit_coin_network}" false +prevalidate_network "${brio_network}" true + +export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" +export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_network}" +export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_network}" +docker compose \ + --env-file "${db_env}" \ + --env-file "${env_deploy}" \ + -f "${remote_dir}/compose.yml" \ + -f "${remote_dir}/envs/canary/compose.yml" \ + config > "${runtime_dir}/candidate-stack.yml" +docker stack config --compose-file "${runtime_dir}/candidate-stack.yml" >/dev/null + +: > "${runtime_dir}/prior-services.list" +: > "${runtime_dir}/prior-service-spec-hashes.list" +install -d -m 0700 "${runtime_dir}/prior-service-specs" +if docker stack services "${stack_name}" --format '{{.Name}}' > "${runtime_dir}/prior-services.list" 2>/dev/null \ + && [[ -s "${runtime_dir}/prior-services.list" ]]; then + stack_preexisting=1 + while IFS= read -r service_name; do + [[ "${service_name}" == "${stack_name}_"* ]] || { + echo "Existing stack contains an unexpected service identity: ${service_name}" >&2 + exit 1 + } + [[ $(docker service inspect "${service_name}" --format '{{index .Spec.Labels "com.docker.stack.namespace"}}') == "${stack_name}" ]] || { + echo "Existing service ${service_name} does not carry the exact stack namespace label." >&2 + exit 1 + } + if [[ "${service_name}" == "${stack_name}_keycloak_brio_staging_backup" ]]; then + legacy_hash=$(docker service inspect "${service_name}" --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) + echo "Legacy identity backup service ${service_name} is still present (spec sha256 ${legacy_hash})." >&2 + echo "Retire it through a separately reviewed operation only after standalone keycloak_brio_staging backup acceptance; this deployment will not prune it." >&2 + exit 1 + fi + spec_file="${runtime_dir}/prior-service-specs/${service_name}.json" + docker service inspect "${service_name}" --format '{{json .Spec}}' > "${spec_file}" + [[ -s "${spec_file}" && ! -L "${spec_file}" ]] || { echo "Failed to inventory exact prior service spec." >&2; exit 1; } + printf '%s|%s\n' "${service_name}" "$(sha256sum "${spec_file}" | cut -d' ' -f1)" \ + >> "${runtime_dir}/prior-service-spec-hashes.list" + done < "${runtime_dir}/prior-services.list" + grep -Fxq "${stack_name}_postgres" "${runtime_dir}/prior-services.list" || { + echo "Existing canary stack does not contain its expected PostgreSQL service." >&2 + exit 1 + } +fi +[[ "${stack_preexisting}" == 1 ]] || { + echo "Canary Brio provisioning requires a healthy pre-existing PostgreSQL stack so its database state can be journaled before mutation." >&2 + exit 1 +} + +# The durable root-owned transaction journal is staged atomically before the +# first host, Swarm, role, database, ACL, or service mutation. +docker run --rm \ + --mount type=bind,src=/etc,dst=/host/etc,readonly \ + --mount type=bind,src=/var/lib,dst=/host/var/lib,readonly \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${runtime_dir},dst=/runtime,readonly" \ + --mount "type=bind,src=${remote_dir}/scripts/brio-db-transaction.sh,dst=/input/brio-db-transaction.sh,readonly" \ + -e "DEPLOYMENT_ID=${deployment_id}" \ + "${validation_image}" sh -euc ' + case "$DEPLOYMENT_ID" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$DEPLOYMENT_ID" in *-*) ;; *) exit 1 ;; esac + root=/managed-var-lib/postgres-recovery/brio-canary + stage="$root/.${DEPLOYMENT_ID}.staging" + final="$root/$DEPLOYMENT_ID" + for path in /managed-var-lib /managed-var-lib/postgres-recovery "$root"; do [ ! -L "$path" ] || exit 1; done + install -d -o 0 -g 0 -m 0700 /managed-var-lib/postgres-recovery "$root" + [ ! -e "$stage" ] && [ ! -L "$stage" ] && [ ! -e "$final" ] && [ ! -L "$final" ] + install -d -o 0 -g 0 -m 0700 "$stage" "$stage/rollback" "$stage/database" "$stage/swarm" + cat > "$stage/rollback/paths.list" <<"PATHS" +etc/makepad/secrets/postgres-canary-superuser-password +etc/makepad/tls/postgres/ca.crt +etc/makepad/secrets/postgres-brio-app-backup-password +etc/makepad/tls/backups/brio-recipient.crt +var/lib/makepad/postgres-backups/brio-staging +PATHS + : > "$stage/rollback/present.list" + : > "$stage/rollback/absent.list" + while IFS= read -r path; do + current=/host + old_ifs=$IFS; IFS=/; set -- $path; IFS=$old_ifs + for component do current="$current/$component"; [ ! -L "$current" ] || { echo "Snapshot path contains a symlink component." >&2; exit 1; }; done + if [ -e "/host/$path" ]; then printf "%s\n" "$path" >> "$stage/rollback/present.list"; else printf "%s\n" "$path" >> "$stage/rollback/absent.list"; fi + done < "$stage/rollback/paths.list" + tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar" -C /host -T "$stage/rollback/present.list" + backup=/host/var/lib/makepad/postgres-backups/brio-staging + : > "$stage/rollback/backup-entries.list" + if [ -d "$backup" ] && [ ! -L "$backup" ]; then + find "$backup" -mindepth 1 -maxdepth 1 -printf "%f\n" | LC_ALL=C sort > "$stage/rollback/backup-entries.list" + if [ -L "$backup/latest" ]; then readlink "$backup/latest" > "$stage/rollback/prior-latest-target"; fi + if [ -f "$backup/last-success.json" ] && [ ! -L "$backup/last-success.json" ]; then + cp -a "$backup/last-success.json" "$stage/rollback/prior-last-success.json" + fi + fi + cp /runtime/prior-services.list "$stage/swarm/prior-services.list" + cp /runtime/prior-service-spec-hashes.list "$stage/swarm/prior-service-spec-hashes.list" + cp -a /runtime/prior-service-specs "$stage/swarm/specs" + install -o 0 -g 0 -m 0700 /input/brio-db-transaction.sh "$stage/brio-db-transaction.sh" + printf "%s\n" "$DEPLOYMENT_ID" > "$stage/deployment-id" + chown -R 0:0 "$stage" + find "$stage" -type d -exec chmod 0700 {} + + find "$stage" -type f -exec chmod 0600 {} + + sync -f "$stage" 2>/dev/null || sync + ' +journal_stage="${recovery_root}/.${deployment_id}.staging" +run_db_transaction prepare "${journal_stage}" +docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "DEPLOYMENT_ID=${deployment_id}" "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-canary + stage="$root/.${DEPLOYMENT_ID}.staging"; final="$root/$DEPLOYMENT_ID" + [ -s "$stage/database/restore.sql" ] && [ -s "$stage/database/prestate.fingerprint" ] + printf "%s\n" IN_PROGRESS > "$stage/IN_PROGRESS"; chmod 0600 "$stage/IN_PROGRESS" + sync -f "$stage" 2>/dev/null || sync + mv -T "$stage" "$final" + sync -f "$root" 2>/dev/null || sync + ' + +rollback_armed=1 +journal_marker "${journal_dir}" STACK_MUTATION_ARMED +stack_mutated=1 + +for record in "${missing_networks[@]}"; do + IFS='|' read -r name required_internal <<< "${record}" + network_args=(--driver overlay --attachable --opt encrypted=true) + [[ "${required_internal}" != "true" ]] || network_args+=(--internal) + docker network create "${network_args[@]}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" >/dev/null +done +for record in "${missing_configs[@]}"; do + IFS='|' read -r name source digest <<< "${record}" + docker config create --label "content-sha256=${digest}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" "${source}" >/dev/null +done +for record in "${missing_secrets[@]}"; do + IFS='|' read -r name source digest <<< "${record}" + docker secret create --label "content-sha256=${digest}" --label "makepad.brio.deployment-id=${deployment_id}" "${name}" "${source}" >/dev/null +done + +# Stage every replacement next to its final destination, then promote with +# same-filesystem renames. The exact snapshot above compensates any partial +# promotion or any later stack/bootstrap/backup failure. +stage_tag=${deployment_id} +docker run --rm \ + --mount "type=bind,src=${runtime_dir},dst=/runtime,readonly" \ + --mount type=bind,src=/etc,dst=/host/etc \ + --mount type=bind,src=/var/lib,dst=/host/var/lib \ + -e "STAGE_TAG=${stage_tag}" \ + "${postgres_image}" sh -euc ' + case "$STAGE_TAG" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "Invalid stage identifier." >&2; exit 1 ;; esac + case "$STAGE_TAG" in *-*) ;; *) echo "Invalid stage identifier." >&2; exit 1 ;; esac + stage_run=${STAGE_TAG%-*} + stage_attempt=${STAGE_TAG#*-} + case "$stage_run:$stage_attempt" in *[!0-9:]*|:*|*:) echo "Invalid stage identifier." >&2; exit 1 ;; esac + for path in \ + /host/etc /host/etc/makepad /host/etc/makepad/secrets \ + /host/etc/makepad/tls /host/etc/makepad/tls/postgres /host/etc/makepad/tls/backups \ + /host/var /host/var/lib /host/var/lib/makepad /host/var/lib/makepad/postgres-backups; do + [ ! -L "$path" ] || { echo "Managed destination contains a symlink: $path" >&2; exit 1; } + done + super_stage="/host/etc/makepad/secrets/.postgres-canary-superuser-password.${STAGE_TAG}.stage" + ca_stage="/host/etc/makepad/tls/postgres/.ca.crt.${STAGE_TAG}.stage" + backup_stage="/host/etc/makepad/secrets/.postgres-brio-app-backup-password.${STAGE_TAG}.stage" + recipient_stage="/host/etc/makepad/tls/backups/.brio-recipient.crt.${STAGE_TAG}.stage" + super_final=/host/etc/makepad/secrets/postgres-canary-superuser-password + ca_final=/host/etc/makepad/tls/postgres/ca.crt + backup_final=/host/etc/makepad/secrets/postgres-brio-app-backup-password + recipient_final=/host/etc/makepad/tls/backups/brio-recipient.crt + cleanup() { rm -f -- "$super_stage" "$ca_stage" "$backup_stage" "$recipient_stage"; } + trap cleanup EXIT HUP INT TERM + for path in "$super_stage" "$ca_stage" "$backup_stage" "$recipient_stage"; do + [ ! -e "$path" ] && [ ! -L "$path" ] || { echo "Refusing an existing stage path." >&2; exit 1; } + done + for path in "$super_final" "$ca_final" "$backup_final" "$recipient_final"; do + [ ! -L "$path" ] && [ ! -d "$path" ] || { echo "Refusing an unsafe final managed path." >&2; exit 1; } + done + install -o 0 -g 0 -m 0600 /runtime/postgres-superuser-password "$super_stage" + install -o 0 -g 0 -m 0444 /runtime/postgres-ca.pem "$ca_stage" + install -o 999 -g 999 -m 0400 /runtime/brio-staging-backup-password "$backup_stage" + install -o 0 -g 0 -m 0444 /runtime/brio-backup-recipient-cert.pem "$recipient_stage" + mv -fT "$super_stage" "$super_final" + mv -fT "$ca_stage" "$ca_final" + mv -fT "$backup_stage" "$backup_final" + mv -fT "$recipient_stage" "$recipient_final" + install -d -o 999 -g 999 -m 0700 /host/var/lib/makepad/postgres-backups/brio-staging + trap - EXIT HUP INT TERM + ' +[[ $(stat -c '%u:%a' "${superuser_host_file}") == "0:600" ]] || { echo "Canary superuser credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_file}") == "999:400" ]] || { echo "Canary backup credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_dir}") == "999:700" && ! -L "${backup_host_dir}" ]] || { echo "Canary backup directory installation failed its ownership/mode check." >&2; exit 1; } +for public_file in "${ca_host_file}" "${recipient_host_file}"; do + public_mode=$(stat -c '%a' "${public_file}") + if [[ $(stat -c '%u' "${public_file}") != "0" ]] || (( (8#${public_mode} & 8#022) != 0 )); then + echo "Canary public certificate installation failed its ownership/mode check." >&2 + exit 1 + fi +done + +case "${failure_injection}" in + after-managed-file-promotion) echo "Injected failure after canary managed-file promotion." >&2; exit 96 ;; + term-after-managed-file-promotion) kill -TERM "$$" ;; + kill-after-managed-file-promotion) kill -KILL "$$" ;; +esac + +"${remote_dir}/scripts/deploy-postgres-stack.sh" "${remote_dir}" "${stack_name}" canary +[[ "${failure_injection}" != "after-stack-deploy" ]] || { echo "Injected failure before canary bootstrap." >&2; exit 95; } + +journal_marker "${journal_dir}" DATABASE_MUTATION_ARMED +db_mutated=1 +docker run --rm --network "${brio_network}" \ + -v "${superuser_host_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${runtime_dir}/brio-staging-app-password:/run/secrets/brio_app_password:ro" \ + -v "${runtime_dir}/brio-staging-backup-password:/run/secrets/brio_backup_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + -v "${remote_dir}/bootstrap/brio-staging-app.sql:/bootstrap/brio-staging-app.sql:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export PGSSLMODE=verify-full PGSSLROOTCERT=/etc/postgresql/ca.crt + export BRIO_APP_PASSWORD="$(cat /run/secrets/brio_app_password)" + export BRIO_BACKUP_PASSWORD="$(cat /run/secrets/brio_backup_password)" + { + printf "%s\n" "\\getenv brio_staging_app_password BRIO_APP_PASSWORD" "\\getenv brio_staging_backup_password BRIO_BACKUP_PASSWORD" + cat /bootstrap/brio-staging-app.sql + } > /tmp/bootstrap.sql + exec psql -X -v ON_ERROR_STOP=1 -h makepad-postgres-brio-staging -U "$1" -d postgres -f /tmp/bootstrap.sql + ' sh "${postgres_user}" >/dev/null + +case "${failure_injection}" in + after-bootstrap) echo "Injected failure after canary database bootstrap." >&2; exit 94 ;; + kill-after-bootstrap) kill -KILL "$$" ;; +esac + +run_role_query() { + local password_file=$1 role=$2 database=$3 sslmode=$4 query=$5 + docker run --rm --network "${brio_network}" \ + -v "${password_file}:/run/secrets/role_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/role_password)" + export PGSSLMODE="$3" PGSSLROOTCERT=/etc/postgresql/ca.crt + exec psql -X -At -h makepad-postgres-brio-staging -U "$1" -d "$2" -c "$4" + ' sh "${role}" "${database}" "${sslmode}" "${query}" +} + +alias_ip=$(docker run --rm --network "${brio_network}" "${postgres_image}" getent hosts makepad-postgres-brio-staging | awk 'NR == 1 {print $1}') +[[ -n "${alias_ip}" ]] || { echo "Brio database alias does not resolve on its isolated network." >&2; exit 1; } +if [[ $(run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app brio_staging verify-full "select current_database() || ':' || current_user") != "brio_staging:brio_staging_app" ]]; then + echo "Brio application role failed the verify-full identity probe." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-app-probe ]] || { echo "Injected failure after canary app probe." >&2; exit 93; } +if run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app brio_staging disable "select 1" >/dev/null 2>&1; then + echo "Plaintext access to brio_staging was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-plaintext-probe ]] || { echo "Injected failure after canary plaintext probe." >&2; exit 92; } +if run_role_query "${runtime_dir}/brio-staging-app-password" brio_staging_app postgres verify-full "select 1" >/dev/null 2>&1; then + echo "Brio application role was unexpectedly accepted by a non-target database." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-nontarget-probe ]] || { echo "Injected failure after canary non-target probe." >&2; exit 91; } +if [[ $(run_role_query "${runtime_dir}/brio-staging-backup-password" brio_staging_backup brio_staging verify-full "show default_transaction_read_only") != "on" ]]; then + echo "Brio backup role is not read-only." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-backup-role-probe ]] || { echo "Injected failure after canary backup-role probe." >&2; exit 90; } + +previous_latest=$(docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + "${validation_image}" sh -euc 'readlink /backups/latest 2>/dev/null || true') +backup_started_at=$(date +%s) +# A one-shot verifier creates no second Swarm service update, so PreviousSpec +# remains the exact predeployment spec and one rollback can restore it. +docker run --rm --network "${brio_network}" --user 999:999 --read-only \ + --cap-drop ALL --security-opt no-new-privileges:true \ + --tmpfs /tmp:size=64m,mode=0700,uid=999,gid=999 \ + --mount "type=bind,src=${backup_host_dir},dst=/backups" \ + --mount "type=bind,src=${backup_host_file},dst=/run/secrets/postgres_backup_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + --mount "type=bind,src=${recipient_host_file},dst=/etc/postgresql/brio-backup-recipient.crt,readonly" \ + --mount "type=bind,src=${remote_dir}/scripts/run-brio-encrypted-backup.sh,dst=/usr/local/bin/run-brio-encrypted-backup.sh,readonly" \ + -e PGHOST=makepad-postgres-brio-staging -e PGPORT=5432 -e PGUSER=brio_staging_backup \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e POSTGRES_BACKUP_PASSWORD_FILE=/run/secrets/postgres_backup_password \ + -e BRIO_BACKUP_DATABASE=brio_staging -e BRIO_BACKUP_ROOT=/backups \ + -e BRIO_BACKUP_RECIPIENT_CERT=/etc/postgresql/brio-backup-recipient.crt \ + -e BRIO_BACKUP_RETENTION_DAYS=35 \ + "${validation_image}" /usr/local/bin/run-brio-encrypted-backup.sh +backup_verified=0 +for _ in $(seq 1 60); do + if docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + -e "BACKUP_STARTED_AT=${backup_started_at}" -e "PREVIOUS_LATEST=${previous_latest}" \ + "${validation_image}" sh -euc ' + status=/backups/last-success.json + [ -s "$status" ] && [ "$(stat -c %Y "$status")" -ge "$BACKUP_STARTED_AT" ] + grep -q "\"database\":\"brio_staging\"" "$status" + grep -q "\"encrypted\":true" "$status" + latest=$(readlink /backups/latest) + case "$latest" in 20??????T??????Z) ;; *) exit 1 ;; esac + [ "$latest" != "$PREVIOUS_LATEST" ] + directory="/backups/$latest" + [ -s "$directory/brio_staging.dump.cms" ] && [ -s "$directory/SHA256SUMS" ] + (cd "$directory" && sha256sum --check --status SHA256SUMS) + openssl cms -cmsout -inform DER -in "$directory/brio_staging.dump.cms" -noout >/dev/null 2>&1 + '; then + backup_verified=1 + break + fi + sleep 2 +done +[[ "${backup_verified}" == "1" ]] || { echo "A fresh validated encrypted brio_staging backup was not published." >&2; exit 1; } +[[ "${failure_injection}" != after-backup-verification ]] || { echo "Injected failure after canary backup verification." >&2; exit 89; } + +journal_marker "${journal_dir}" COMMITTED +rollback_armed=0 +remove_durable_journal "${journal_dir}" +echo "Brio canary PostgreSQL provisioning, bootstrap, transport policy, alias, and encrypted backup verification passed." diff --git a/scripts/deploy-brio-identity-db-host.sh b/scripts/deploy-brio-identity-db-host.sh new file mode 100755 index 0000000..7b4e032 --- /dev/null +++ b/scripts/deploy-brio-identity-db-host.sh @@ -0,0 +1,806 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 4)); then + echo "Usage: deploy-brio-identity-db-host.sh " >&2 + exit 2 +fi + +bundle_dir=$1 +runtime_dir=$2 +db_hostname=$3 +keycloak_source_cidr=$4 +live_dir=/srv/makepad/postgres +compose_project=postgres +expected_container_name=postgres-postgres-1 +db_env="${bundle_dir}/envs/production/.env.db" +candidate_compose="${bundle_dir}/compose.host.yml" +rollback_armed=0 +validation_image= +prior_postgres_image= +db_mutated=0 +recovery_marker="${runtime_dir}/RECOVERY_REQUIRED" +recovery_root=/var/lib/makepad/postgres-recovery/brio-identity +recovery_id=${runtime_dir##*/postgres-brio-identity-runtime-} +recovery_evidence="${recovery_root}/${recovery_id}" +journal_dir=${recovery_evidence} +failure_injection=${BRIO_DEPLOY_FAILURE_INJECTION:-} + +if [[ -n "${failure_injection}" ]]; then + [[ "${BRIO_DEPLOY_TEST_MODE:-}" == "isolated-container" && -f /.dockerenv ]] || { + echo "Failure injection is permitted only in the isolated deployment test container." >&2 + exit 2 + } + case "${failure_injection}" in + after-managed-file-promotion|term-after-managed-file-promotion|kill-after-managed-file-promotion|after-bootstrap|kill-after-bootstrap|after-app-probe|after-plaintext-probe|after-nontarget-probe|after-backup-role-probe|after-backup-verification|rollback-restore|rollback-recreate) ;; + *) echo "Unsupported identity deployment failure injection." >&2; exit 2 ;; + esac +fi + +[[ "${BRIO_IDENTITY_DB_DEPLOY_CONFIRM:-}" == "restart-standalone-postgres-for-brio-staging" ]] || { + echo "Set BRIO_IDENTITY_DB_DEPLOY_CONFIRM to the exact standalone DB restart acknowledgement." >&2 + exit 2 +} +[[ "${BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED:-}" == "yes" ]] || { + echo "A successful current backup/restore gate must be confirmed before changing the standalone DB VM." >&2 + exit 2 +} +[[ "${bundle_dir}" =~ ^/tmp/postgres-brio-identity-bundle-[0-9]+-[0-9]+$ ]] || { + echo "job-bundle-dir must be /tmp/postgres-brio-identity-bundle--." >&2 + exit 2 +} +[[ "${runtime_dir}" =~ ^/tmp/postgres-brio-identity-runtime-[0-9]+-[0-9]+$ ]] || { + echo "runtime-secret-dir must be /tmp/postgres-brio-identity-runtime--." >&2 + exit 2 +} +[[ "${db_hostname}" == "65.21.134.125" ]] || { + echo "The Brio identity certificate endpoint must be the reviewed standalone DB IP 65.21.134.125." >&2 + exit 2 +} +[[ "${keycloak_source_cidr}" == "88.99.209.165/32" ]] || { + echo "The Keycloak source must be the reviewed exact egress 88.99.209.165/32." >&2 + exit 2 +} + +read_setting() { + local name=$1 file=$2 value + value=$(grep -E "^${name}=" "${file}" | tail -n 1 | cut -d= -f2-) + [[ -n "${value}" ]] || { echo "${name} is missing or empty in ${file}." >&2; exit 1; } + printf '%s' "${value}" +} + +assert_no_symlink_components() { + local path=$1 current='' component + local -a components=() + [[ "${path}" == /* && "${path}" != *"//"* && "${path}" != *"/./"* \ + && "${path}" != */. && "${path}" != *"/../"* && "${path}" != */.. ]] || { + echo "Managed path is not a normalized absolute path: ${path}" >&2 + return 1 + } + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + [[ -n "${component}" && "${component}" != . && "${component}" != .. ]] || return 1 + current="${current}/${component}" + [[ ! -L "${current}" ]] || { + echo "Managed path contains a symlink component: ${current}" >&2 + return 1 + } + done +} + +validate_postgres_target() { + local expected_image=$1 container_id container_name project_label service_label oneoff_label + local network_mode actual_image data_mount running health + container_id=$(docker container inspect "${expected_container_name}" --format '{{.Id}}' 2>/dev/null) || { + echo "Expected standalone container ${expected_container_name} was not found." >&2 + return 1 + } + container_name=$(docker container inspect "${container_id}" --format '{{.Name}}') + project_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.project"}}') + service_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.service"}}') + oneoff_label=$(docker container inspect "${container_id}" --format '{{index .Config.Labels "com.docker.compose.oneoff"}}') + network_mode=$(docker container inspect "${container_id}" --format '{{.HostConfig.NetworkMode}}') + actual_image=$(docker container inspect "${container_id}" --format '{{.Config.Image}}') + data_mount=$(docker container inspect "${container_id}" --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{printf "%s|%s|%t\n" .Type .Source .RW}}{{end}}{{end}}') + running=$(docker container inspect "${container_id}" --format '{{.State.Running}}') + health=$(docker container inspect "${container_id}" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}') + [[ "${container_name}" == "/${expected_container_name}" \ + && "${project_label}" == "${compose_project}" \ + && "${service_label}" == "postgres" \ + && "${oneoff_label}" == "False" \ + && "${network_mode}" == "host" \ + && "${actual_image}" == "${expected_image}" \ + && "${data_mount}" == 'bind|/var/lib/makepad/postgres|true' \ + && "${running}" == "true" && "${health}" == "healthy" ]] || { + echo "The standalone target failed its exact Compose label, image, host-network, data-bind, or health contract." >&2 + return 1 + } + printf '%s' "${container_id}" +} + +restore_snapshot() { + local source_journal=${1:-${journal_dir}} + docker run --rm \ + --mount "type=bind,src=${live_dir},dst=/managed/live" \ + --mount type=bind,src=/etc/makepad,dst=/managed/etc \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${source_journal}/rollback,dst=/rollback,readonly" \ + "${validation_image}" sh -euc ' + backup=/managed-var-lib/postgres-backups/keycloak-brio-staging + parent=${backup%/*} + [ -d "$parent" ] && [ ! -L "$parent" ] + if [ -e "$backup" ] || [ -L "$backup" ]; then + [ -d "$backup" ] && [ ! -L "$backup" ] || { echo "Identity backup rollback target is unsafe." >&2; exit 1; } + find "$backup" -depth -delete + fi + if [ -f /rollback/identity-backups.tar ] && [ ! -L /rollback/identity-backups.tar ]; then + tar -tf /rollback/identity-backups.tar | while IFS= read -r member; do + case "$member" in postgres-backups/keycloak-brio-staging|postgres-backups/keycloak-brio-staging/*) ;; *) exit 1 ;; esac + done + tar --numeric-owner -xpf /rollback/identity-backups.tar -C /managed-var-lib + elif [ ! -f /rollback/identity-backup-absent ]; then + echo "Identity backup snapshot contract is incomplete." >&2 + exit 1 + fi + while IFS= read -r path; do + case "$path" in + live/compose.host.yml|live/envs/production/.env.db|live/config/runtrace-pg_hba.conf|live/bootstrap/keycloak-brio-staging.sql|live/scripts/run-runtrace-backup.sh|live/scripts/run-runtrace-backup-loop.sh|live/scripts/run-brio-encrypted-backup.sh|live/scripts/run-brio-encrypted-backup-loop.sh|etc/secrets/postgres-brio-identity-backup-password|etc/tls/backups/brio-recipient.crt) ;; + *) echo "Refusing an unexpected rollback path." >&2; exit 1 ;; + esac + rm -f -- "/managed/$path" + done < /rollback/absent.list + tar --numeric-owner -xpf /rollback/managed.tar -C /managed + ' +} + +run_db_transaction() { + local operation=$1 source_journal=${2:-${journal_dir}} helper + if [[ "${operation}" == prepare ]]; then + helper="${bundle_dir}/scripts/brio-db-transaction.sh" + else + helper="${source_journal}/brio-db-transaction.sh" + fi + [[ -f "${helper}" && ! -L "${helper}" ]] || { + echo "The durable database compensation helper is unavailable." >&2 + return 1 + } + docker run --rm --network host \ + --mount "type=bind,src=${helper},dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + --mount "type=bind,src=${source_journal},dst=/journal" \ + --mount "type=bind,src=${superuser_host_file},dst=/run/secrets/postgres_superuser_password,readonly" \ + --mount "type=bind,src=${ca_host_file},dst=/etc/postgresql/ca.crt,readonly" \ + -e "PGUSER=${postgres_user}" -e "PGHOST=${db_hostname}" -e PGHOSTADDR=127.0.0.1 \ + -e PGSSLMODE=verify-full -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -e PGPASSWORD_FILE=/run/secrets/postgres_superuser_password \ + "${postgres_image}" /usr/local/bin/brio-db-transaction.sh "${operation}" keycloak /journal/database +} + +journal_marker() { + local source_journal=$1 marker=$2 + case "${marker}" in IN_PROGRESS|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; + *) echo "Refusing an unsupported journal marker." >&2; return 1 ;; + esac + docker run --rm --mount "type=bind,src=${source_journal},dst=/journal" \ + -e "MARKER=${marker}" "${validation_image}" sh -euc ' + case "$MARKER" in IN_PROGRESS|DATABASE_MUTATION_ARMED|COMMITTED|ROLLED_BACK|RECOVERY_REQUIRED) ;; *) exit 1 ;; esac + [ -d /journal ] && [ ! -L /journal ] + tmp="/journal/.${MARKER}.tmp" + [ ! -e "$tmp" ] && [ ! -L "$tmp" ] + printf "%s\n" "$MARKER" > "$tmp" + chmod 0600 "$tmp" + mv -fT "$tmp" "/journal/$MARKER" + sync -f /journal 2>/dev/null || sync + ' +} + +remove_durable_journal() { + local source_journal=$1 + docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "JOURNAL=${source_journal}" "${validation_image}" sh -euc ' + case "$JOURNAL" in /var/lib/makepad/postgres-recovery/brio-identity/*) identifier=${JOURNAL##*/} ;; *) exit 1 ;; esac + case "$identifier" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$identifier" in *-*) ;; *) exit 1 ;; esac + relative=${JOURNAL#/var/lib/makepad} + target="/managed-var-lib$relative" + [ -d "$target" ] && [ ! -L "$target" ] + find "$target" -depth -delete + sync -f /managed-var-lib/postgres-recovery/brio-identity 2>/dev/null || sync + ' +} + +recover_incomplete_journals() { + local current_id="${recovery_id}" current_journal="${journal_dir}" pending saved_injection="${failure_injection}" marker_list + marker_list=$(docker run --rm --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib,readonly \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-identity + [ ! -e "$root" ] && exit 0 + [ -d "$root" ] && [ ! -L "$root" ] + for candidate in "$root"/* "$root"/.*; do + [ -e "$candidate" ] || continue + name=${candidate##*/} + case "$name" in .|..) continue ;; .*.staging) echo "INCOMPLETE-STAGE:$name"; continue ;; esac + case "$name" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) echo "UNSAFE:$name"; continue ;; esac + case "$name" in *-*) ;; *) echo "UNSAFE:$name"; continue ;; esac + [ -d "$candidate" ] && [ ! -L "$candidate" ] || { echo "UNSAFE:$name"; continue; } + if [ -f "$candidate/COMMITTED" ] && [ ! -L "$candidate/COMMITTED" ]; then + echo "COMMITTED:$name" + elif [ -f "$candidate/ROLLED_BACK" ] && [ ! -L "$candidate/ROLLED_BACK" ]; then + echo "ROLLED_BACK:$name" + elif [ -f "$candidate/IN_PROGRESS" ] && [ ! -L "$candidate/IN_PROGRESS" ]; then + echo "PENDING:$name" + else + echo "UNSAFE:$name" + fi + done + ') + while IFS=: read -r state identifier; do + [[ -n "${state}" ]] || continue + case "${state}" in + COMMITTED|ROLLED_BACK) + remove_durable_journal "${recovery_root}/${identifier}" + ;; + PENDING) + pending="${recovery_root}/${identifier}" + recovery_id=${identifier} + journal_dir=${pending} + recovery_evidence=${pending} + prior_postgres_image=$(docker run --rm --mount "type=bind,src=${pending},dst=/journal,readonly" \ + "${validation_image}" sh -euc 'cat /journal/prior-postgres-image') + [[ "${prior_postgres_image}" == *@sha256:* ]] || { echo "Recovery journal has an invalid prior image." >&2; return 1; } + db_mutated=$(docker run --rm --mount "type=bind,src=${pending},dst=/journal,readonly" \ + "${validation_image}" sh -euc 'if [ -f /journal/DATABASE_MUTATION_ARMED ] && [ ! -L /journal/DATABASE_MUTATION_ARMED ]; then echo 1; else echo 0; fi') + failure_injection= + rollback_armed=1 + if ! rollback_deployment; then + # Do not let the outer EXIT trap attempt the same failed recovery a + # second time. Its durable journal and marker are now authoritative. + rollback_armed=0 + preserve_recovery_evidence || true + echo "An interrupted Brio identity database transaction could not be recovered." >&2 + return 1 + fi + rollback_armed=0 + ;; + INCOMPLETE-STAGE|UNSAFE) + echo "Unsafe or incomplete durable Brio identity journal detected: ${identifier}" >&2 + return 1 + ;; + *) echo "Unexpected durable journal state." >&2; return 1 ;; + esac + done <<< "${marker_list}" + recovery_id=${current_id} + journal_dir=${current_journal} + recovery_evidence=${current_journal} + failure_injection=${saved_injection} + db_mutated=0 +} + +rollback_deployment() { + local rollback_status=0 + echo "Deployment failed after the mutation boundary; restoring the exact managed-file snapshot." >&2 + if [[ "${failure_injection}" == "rollback-restore" ]]; then + rollback_status=1 + else + restore_snapshot "${journal_dir}" || rollback_status=1 + fi + export MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST="${db_hostname}" + export MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH="${live_dir}/config/runtrace-pg_hba.conf" + export MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup.sh" + export MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup-loop.sh" + export MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup.sh" + export MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup-loop.sh" + if [[ "${failure_injection}" == "rollback-recreate" ]]; then + rollback_status=1 + else + docker compose --project-name postgres \ + --env-file "${live_dir}/envs/production/.env.db" \ + -f "${live_dir}/compose.host.yml" \ + up -d --remove-orphans --wait --force-recreate || rollback_status=1 + fi + validate_postgres_target "${prior_postgres_image}" >/dev/null || rollback_status=1 + if [[ "${db_mutated}" == "1" || -f "${journal_dir}/DATABASE_MUTATION_ARMED" ]]; then + run_db_transaction restore "${journal_dir}" || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + journal_marker "${journal_dir}" ROLLED_BACK || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + remove_durable_journal "${journal_dir}" || rollback_status=1 + fi + if [[ "${rollback_status}" == "0" ]]; then + echo "Managed files and the prior healthy postgres Compose target were restored." >&2 + else + echo "Automatic rollback did not restore a healthy exact target; operator intervention is required." >&2 + fi + return "${rollback_status}" +} + +preserve_recovery_evidence() { + # The rollback archive can contain the former managed backup credential. Keep + # it outside /tmp, root-owned and unreadable to the deploy account. The + # runtime marker contains only the fixed evidence path and run identifier. + { + printf 'recovery_evidence=%s\n' "${recovery_evidence}" + printf 'deployment_id=%s\n' "${recovery_id}" + } > "${recovery_marker}" + chmod 0600 "${recovery_marker}" + journal_marker "${journal_dir}" RECOVERY_REQUIRED + echo "Root-protected recovery evidence retained at ${recovery_evidence}." >&2 +} + +cleanup_runtime() { + local cleanup_status=0 + # Incoming application/backup passwords and recipient material are always + # removed, including when rollback itself failed. + for name in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + [[ ! -f "${runtime_dir}/${name}" || -L "${runtime_dir}/${name}" ]] || rm -f -- "${runtime_dir:?}/${name}" || cleanup_status=1 + done + if [[ -f "${recovery_marker}" && ! -L "${recovery_marker}" ]]; then + # The protected snapshot and marker are deliberately exempt from normal and + # TTL cleanup until an operator has completed recovery. + rm -f -- "${runtime_dir}/runtrace-pg_hba.conf" "${runtime_dir}/candidate-compose.yml" || cleanup_status=1 + return "${cleanup_status}" + fi + if [[ -e "${recovery_marker}" || -L "${recovery_marker}" ]]; then + echo "Unsafe recovery marker type; preserving the runtime directory for operator inspection." >&2 + return 1 + fi + if [[ -n "${validation_image}" ]] && docker image inspect "${validation_image}" >/dev/null 2>&1; then + docker run --rm --mount "type=bind,src=${runtime_dir},dst=/runtime" "${validation_image}" \ + sh -euc 'for path in /runtime/runtrace-pg_hba.conf /runtime/candidate-compose.yml; do [ ! -e "$path" ] || find "$path" -depth -delete; done' \ + >/dev/null 2>&1 || cleanup_status=1 + fi + rmdir -- "${runtime_dir}" 2>/dev/null || true + return "${cleanup_status}" +} + +handle_exit() { + local status=$? + trap - EXIT HUP INT TERM + if [[ "${rollback_armed}" == "1" ]]; then + if ! rollback_deployment; then + preserve_recovery_evidence || { + echo "CRITICAL: rollback and durable recovery-evidence preservation both failed." >&2 + status=1 + } + status=1 + fi + fi + cleanup_runtime || status=1 + exit "${status}" +} +trap handle_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +swarm_state=$(docker info --format '{{.Swarm.LocalNodeState}}') +[[ "${swarm_state}" == "inactive" ]] || { + echo "Refusing to run the standalone DB-VM deployment on a Docker Swarm node (state: ${swarm_state})." >&2 + exit 1 +} + +for candidate in \ + "${candidate_compose}" \ + "${db_env}" \ + "${bundle_dir}/config/runtrace-pg_hba.conf" \ + "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" \ + "${bundle_dir}/scripts/run-runtrace-backup.sh" \ + "${bundle_dir}/scripts/run-runtrace-backup-loop.sh" \ + "${bundle_dir}/scripts/run-brio-encrypted-backup.sh" \ + "${bundle_dir}/scripts/run-brio-encrypted-backup-loop.sh" \ + "${bundle_dir}/scripts/brio-db-transaction.sh"; do + [[ -f "${candidate}" && ! -L "${candidate}" ]] || { echo "Candidate bundle file is missing or a symlink: ${candidate}" >&2; exit 1; } +done + +postgres_image=$(read_setting POSTGRES_IMAGE "${db_env}") +validation_image=$(read_setting BRIO_BACKUP_IMAGE "${db_env}") +postgres_user=$(read_setting POSTGRES_USER "${db_env}") +data_host_dir=$(read_setting MAKEPAD_POSTGRES_DATA_PATH "${db_env}") +superuser_host_file=$(read_setting MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH "${db_env}") +server_cert_host_file=$(read_setting MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH "${db_env}") +server_key_host_file=$(read_setting MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH "${db_env}") +ca_host_file=$(read_setting MAKEPAD_POSTGRES_CA_CERT_HOST_PATH "${db_env}") +hba_host_file=$(read_setting MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH "${db_env}") +backup_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PASSWORD_FILE_HOST_PATH "${db_env}") +recipient_host_file=$(read_setting MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH "${db_env}") +backup_host_dir=$(read_setting MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_PATH "${db_env}") + +[[ "${postgres_image}" == *@sha256:* && "${validation_image}" == *@sha256:* ]] || { echo "Candidate images must be digest pinned." >&2; exit 1; } +[[ "${data_host_dir}" == "/var/lib/makepad/postgres" \ + && "${superuser_host_file}" == "/etc/makepad/secrets/postgres-superuser-password" \ + && "${server_cert_host_file}" == "/etc/makepad/tls/postgres/server.crt" \ + && "${server_key_host_file}" == "/etc/makepad/secrets/postgres-server.key" \ + && "${ca_host_file}" == "/etc/makepad/tls/postgres/ca.crt" \ + && "${hba_host_file}" == "/srv/makepad/postgres/config/runtrace-pg_hba.conf" \ + && "${backup_host_file}" == "/etc/makepad/secrets/postgres-brio-identity-backup-password" \ + && "${recipient_host_file}" == "/etc/makepad/tls/backups/brio-recipient.crt" \ + && "${backup_host_dir}" == "/var/lib/makepad/postgres-backups/keycloak-brio-staging" ]] || { + echo "Candidate environment does not match the exact standalone DB-VM path contract." >&2 + exit 1 +} +expected_data_bind="\"\${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data\"" +if ! grep -Fq 'network_mode: host' "${candidate_compose}" \ + || ! grep -Fq "${expected_data_bind}" "${candidate_compose}"; then + echo "Candidate Compose does not declare the reviewed host network and exact PostgreSQL data bind." >&2 + exit 1 +fi + +runtime_mode=$(stat -c '%a' "${runtime_dir}") +[[ ! -L "${runtime_dir}" && "${runtime_mode}" == "700" ]] || { echo "Runtime directory must be a non-symlink with mode 0700." >&2; exit 1; } +for input in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + input_path="${runtime_dir}/${input}" + [[ -s "${input_path}" && ! -L "${input_path}" && $(stat -c '%a' "${input_path}") == "600" ]] || { + echo "Required runtime input must be non-empty, non-symlinked, and mode 0600: ${input}" >&2 + exit 1 + } +done +for password_file in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password; do + if [[ $(awk 'END { print NR }' "${runtime_dir}/${password_file}") -ne 1 ]] || grep -q $'\r' "${runtime_dir}/${password_file}"; then + echo "Password input must contain one line and no carriage return: ${password_file}" >&2 + exit 1 + fi +done +cmp -s "${runtime_dir}/keycloak-brio-staging-app-password" "${runtime_dir}/keycloak-brio-staging-backup-password" \ + && { echo "Keycloak Brio application and backup credentials must be distinct." >&2; exit 1; } + +for live_input in \ + "${live_dir}/compose.host.yml" \ + "${live_dir}/envs/production/.env.db" \ + "${live_dir}/config/runtrace-pg_hba.conf" \ + "${live_dir}/scripts/run-runtrace-backup.sh" \ + "${live_dir}/scripts/run-runtrace-backup-loop.sh" \ + "${superuser_host_file}" "${server_cert_host_file}" "${server_key_host_file}" "${ca_host_file}"; do + [[ -s "${live_input}" && ! -L "${live_input}" ]] || { echo "Required existing DB-VM input is unavailable or unsafe: ${live_input}" >&2; exit 1; } +done +for managed_path in \ + "${live_dir}" "${live_dir}/compose.host.yml" "${live_dir}/envs/production" "${live_dir}/envs/production/.env.db" \ + "${live_dir}/config" "${hba_host_file}" "${live_dir}/bootstrap" "${live_dir}/scripts" \ + "${superuser_host_file}" "${server_cert_host_file}" "${server_key_host_file}" "${ca_host_file}" \ + "${backup_host_file}" "${recipient_host_file}" "${backup_host_dir}" \ + /var/lib/makepad/postgres-backups /var/lib/makepad/postgres-recovery; do + assert_no_symlink_components "${managed_path}" +done +for exact_parent in \ + /srv /srv/makepad "${live_dir}" "${live_dir}/envs" "${live_dir}/envs/production" \ + "${live_dir}/config" "${live_dir}/bootstrap" "${live_dir}/scripts" \ + /etc /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres \ + /var /var/lib /var/lib/makepad /var/lib/makepad/postgres-backups; do + [[ -d "${exact_parent}" && ! -L "${exact_parent}" ]] || { + echo "Required managed parent is missing, symlinked, or not a directory: ${exact_parent}" >&2 + exit 1 + } +done +[[ $(stat -c '%u:%a' "${superuser_host_file}") == "0:600" ]] || { echo "Existing PostgreSQL superuser credential must be root-owned with mode 0600." >&2; exit 1; } +for public_file in "${server_cert_host_file}" "${ca_host_file}"; do + public_mode=$(stat -c '%a' "${public_file}") + if [[ $(stat -c '%u' "${public_file}") != "0" ]] || (( (8#${public_mode} & 8#022) != 0 )); then + echo "Existing PostgreSQL public TLS files must be root-owned and not group/world writable." >&2 + exit 1 + fi +done +key_uid=$(stat -c '%u' "${server_key_host_file}") +key_gid=$(stat -c '%g' "${server_key_host_file}") +key_mode=$(stat -c '%a' "${server_key_host_file}") +[[ "${key_uid}:${key_gid}:${key_mode}" == "70:70:400" ]] || { + echo "Existing PostgreSQL server key must retain the verified 70:70 mode-0400 contract." >&2 + exit 1 +} +for optional_managed in "${backup_host_file}" "${recipient_host_file}"; do + [[ ! -L "${optional_managed}" ]] || { echo "Refusing a symlink at a managed identity backup path." >&2; exit 1; } +done +if [[ -L "${backup_host_dir}" || ( -e "${backup_host_dir}" && ! -d "${backup_host_dir}" ) ]]; then + echo "Refusing a symlink or non-directory Brio identity backup path." >&2 + exit 1 +fi + +command -v openssl >/dev/null 2>&1 || { echo "openssl is required." >&2; exit 1; } +openssl x509 -in "${server_cert_host_file}" -noout -checkend 604800 >/dev/null +openssl x509 -in "${server_cert_host_file}" -noout -checkip "${db_hostname}" >/dev/null +openssl verify -purpose sslserver -CAfile "${ca_host_file}" "${server_cert_host_file}" >/dev/null +cert_key_hash=$(openssl x509 -in "${server_cert_host_file}" -pubkey -noout | openssl pkey -pubin -outform DER | sha256sum | cut -d' ' -f1) +private_key_hash=$(docker run --rm --mount "type=bind,src=${server_key_host_file},dst=/runtime/server.key,readonly" \ + "${validation_image}" openssl pkey -in /runtime/server.key -pubout -outform DER | sha256sum | cut -d' ' -f1) +[[ "${cert_key_hash}" == "${private_key_hash}" ]] || { echo "Existing DB-VM server certificate and key do not match." >&2; exit 1; } +if grep -q -- 'PRIVATE KEY' "${runtime_dir}/brio-backup-recipient-cert.pem" \ + || ! openssl x509 -in "${runtime_dir}/brio-backup-recipient-cert.pem" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${runtime_dir}/brio-backup-recipient-cert.pem" -out /dev/null; then + echo "Brio backup recipient is not a valid public encryption certificate." >&2 + exit 1 +fi + +rendered_hba="${runtime_dir}/runtrace-pg_hba.conf" +awk -v cidr="${keycloak_source_cidr}" ' + $1 == "hostssl" && $2 == "keycloak_brio_staging" && $3 == "keycloak_brio_staging_app" && $4 == "all" && $5 == "scram-sha-256" { + print "hostssl keycloak_brio_staging keycloak_brio_staging_app 127.0.0.1/32 scram-sha-256" + print "hostssl keycloak_brio_staging keycloak_brio_staging_app " cidr " scram-sha-256" + next + } + { print } +' "${bundle_dir}/config/runtrace-pg_hba.conf" > "${rendered_hba}" +chmod 0600 "${rendered_hba}" +app_allow_line=$(grep -n -E "^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+${keycloak_source_cidr//./\.}[[:space:]]+scram-sha-256$" "${rendered_hba}" | cut -d: -f1) +backup_allow_line=$(grep -n -E '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_backup[[:space:]]+127\.0\.0\.1/32[[:space:]]+scram-sha-256$' "${rendered_hba}" | cut -d: -f1) +app_reject_line=$(grep -n -E '^host[[:space:]]+all[[:space:]]+keycloak_brio_staging_app[[:space:]]+all[[:space:]]+reject$' "${rendered_hba}" | cut -d: -f1) +backup_reject_line=$(grep -n -E '^host[[:space:]]+all[[:space:]]+keycloak_brio_staging_backup[[:space:]]+all[[:space:]]+reject$' "${rendered_hba}" | cut -d: -f1) +if [[ -z "${app_allow_line}" || -z "${backup_allow_line}" || -z "${app_reject_line}" || -z "${backup_reject_line}" \ + || "${app_allow_line}" -ge "${app_reject_line}" || "${backup_allow_line}" -ge "${backup_reject_line}" \ + || $(grep -Ec '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+127\.0\.0\.1/32[[:space:]]+scram-sha-256$' "${rendered_hba}") -ne 1 \ + || $(grep -Ec "^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+${keycloak_source_cidr//./\.}[[:space:]]+scram-sha-256$" "${rendered_hba}") -ne 1 ]] \ + || grep -Eq '^hostssl keycloak_brio_staging[[:space:]]+keycloak_brio_staging_app[[:space:]]+all[[:space:]]' "${rendered_hba}"; then + echo "Failed to render ordered, exact source-restricted Keycloak Brio HBA rules." >&2 + exit 1 +fi + +export MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST="${db_hostname}" +export MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH="${hba_host_file}" +export MAKEPAD_POSTGRES_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup.sh" +export MAKEPAD_POSTGRES_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-runtrace-backup-loop.sh" +export MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup.sh" +export MAKEPAD_POSTGRES_BRIO_BACKUP_LOOP_SCRIPT_HOST_PATH="${live_dir}/scripts/run-brio-encrypted-backup-loop.sh" +candidate_compose_command=(docker compose --project-name postgres --env-file "${db_env}" -f "${candidate_compose}") +"${candidate_compose_command[@]}" config --quiet +"${candidate_compose_command[@]}" config > "${runtime_dir}/candidate-compose.yml" +if ! grep -Fq 'network_mode: host' "${runtime_dir}/candidate-compose.yml" \ + || ! grep -Fq 'source: /var/lib/makepad/postgres' "${runtime_dir}/candidate-compose.yml" \ + || ! grep -Fq 'target: /var/lib/postgresql/data' "${runtime_dir}/candidate-compose.yml"; then + echo "Rendered candidate Compose failed the host-network or exact data-bind prevalidation." >&2 + exit 1 +fi + +prior_container_id=$(validate_postgres_target "${postgres_image}") +prior_postgres_image=$(docker container inspect "${prior_container_id}" --format '{{.Config.Image}}') +docker pull "${postgres_image}" >/dev/null +docker pull "${validation_image}" >/dev/null + +# A SIGKILL cannot run shell traps. Recover any root-owned transaction journal +# left by an interrupted prior deployment before creating a new mutation. +recover_incomplete_journals + +# Build the complete root-owned journal at a same-filesystem staging path. It +# includes exact managed files and a compensating database-state program before +# the first host, Compose, role, database, ACL, or backup-service mutation. +docker run --rm \ + --mount "type=bind,src=${live_dir},dst=/managed/live,readonly" \ + --mount type=bind,src=/etc/makepad,dst=/managed/etc,readonly \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + --mount "type=bind,src=${bundle_dir}/scripts/brio-db-transaction.sh,dst=/input/brio-db-transaction.sh,readonly" \ + -e "RECOVERY_ID=${recovery_id}" -e "PRIOR_POSTGRES_IMAGE=${prior_postgres_image}" \ + "${validation_image}" sh -euc ' + case "$RECOVERY_ID" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$RECOVERY_ID" in *-*) ;; *) exit 1 ;; esac + root=/managed-var-lib/postgres-recovery/brio-identity + stage="$root/.${RECOVERY_ID}.staging" + final="$root/$RECOVERY_ID" + for path in /managed-var-lib /managed-var-lib/postgres-recovery "$root"; do + [ ! -L "$path" ] || { echo "Recovery path contains a symlink component." >&2; exit 1; } + done + install -d -o 0 -g 0 -m 0700 /managed-var-lib/postgres-recovery "$root" + [ ! -e "$stage" ] && [ ! -L "$stage" ] && [ ! -e "$final" ] && [ ! -L "$final" ] + install -d -o 0 -g 0 -m 0700 "$stage" "$stage/rollback" "$stage/database" + cat > "$stage/rollback/paths.list" <<"PATHS" +live/compose.host.yml +live/envs/production/.env.db +live/config/runtrace-pg_hba.conf +live/bootstrap/keycloak-brio-staging.sql +live/scripts/run-runtrace-backup.sh +live/scripts/run-runtrace-backup-loop.sh +live/scripts/run-brio-encrypted-backup.sh +live/scripts/run-brio-encrypted-backup-loop.sh +etc/secrets/postgres-brio-identity-backup-password +etc/tls/backups/brio-recipient.crt +PATHS + : > "$stage/rollback/present.list" + : > "$stage/rollback/absent.list" + while IFS= read -r path; do + current=/managed + old_ifs=$IFS + IFS=/ + set -- $path + IFS=$old_ifs + for component do + current="$current/$component" + [ ! -L "$current" ] || { echo "Managed snapshot path contains a symlink component: $path" >&2; exit 1; } + done + if [ -e "/managed/$path" ]; then printf "%s\n" "$path" >> "$stage/rollback/present.list"; else printf "%s\n" "$path" >> "$stage/rollback/absent.list"; fi + done < "$stage/rollback/paths.list" + tar --numeric-owner -cpf "$stage/rollback/managed.tar" -C /managed -T "$stage/rollback/present.list" + backup=/managed-var-lib/postgres-backups/keycloak-brio-staging + if [ -e "$backup" ] || [ -L "$backup" ]; then + [ -d "$backup" ] && [ ! -L "$backup" ] || { echo "Identity backup snapshot path is unsafe." >&2; exit 1; } + tar --numeric-owner -cpf "$stage/rollback/identity-backups.tar" -C /managed-var-lib postgres-backups/keycloak-brio-staging + else + printf "%s\n" absent > "$stage/rollback/identity-backup-absent" + fi + install -o 0 -g 0 -m 0700 /input/brio-db-transaction.sh "$stage/brio-db-transaction.sh" + printf "%s\n" "$PRIOR_POSTGRES_IMAGE" > "$stage/prior-postgres-image" + printf "%s\n" "$RECOVERY_ID" > "$stage/deployment-id" + find "$stage" -type d -exec chmod 0700 {} + + find "$stage" -type f -exec chmod 0600 {} + + sync -f "$stage" 2>/dev/null || sync + ' +journal_stage="${recovery_root}/.${recovery_id}.staging" +run_db_transaction prepare "${journal_stage}" +docker run --rm \ + --mount type=bind,src=/var/lib/makepad,dst=/managed-var-lib \ + -e "RECOVERY_ID=${recovery_id}" \ + "${validation_image}" sh -euc ' + root=/managed-var-lib/postgres-recovery/brio-identity + stage="$root/.${RECOVERY_ID}.staging" + final="$root/$RECOVERY_ID" + [ -d "$stage/database" ] && [ -s "$stage/database/restore.sql" ] && [ -s "$stage/database/prestate.fingerprint" ] + printf "%s\n" IN_PROGRESS > "$stage/IN_PROGRESS" + chmod 0600 "$stage/IN_PROGRESS" + sync -f "$stage" 2>/dev/null || sync + mv -T "$stage" "$final" + sync -f "$root" 2>/dev/null || sync + ' +rollback_armed=1 + +install_host_path() { + local source=$1 destination=$2 owner=$3 mode=$4 + case "${destination}" in + "${live_dir}"/*|/etc/makepad/*) ;; + *) echo "Refusing unmanaged host path: ${destination}" >&2; return 1 ;; + esac + docker run --rm \ + --mount "type=bind,src=${source},dst=/runtime/input,readonly" \ + --mount type=bind,src=/srv,dst=/host/srv \ + --mount type=bind,src=/etc,dst=/host/etc \ + -e "DESTINATION=${destination}" -e "FILE_OWNER=${owner}" -e "FILE_MODE=${mode}" -e "STAGE_TAG=${recovery_id}" \ + "${postgres_image}" sh -euc ' + case "$DESTINATION" in /srv/makepad/postgres/*|/etc/makepad/*) ;; *) exit 1 ;; esac + case "$STAGE_TAG" in ""|*[!0-9-]*|*-*-*|-*|*-|0*|*-0*) exit 1 ;; esac + case "$STAGE_TAG" in *-*) ;; *) exit 1 ;; esac + host_destination="/host$DESTINATION" + parent=${host_destination%/*} + base=${host_destination##*/} + current=/host + old_ifs=$IFS + IFS=/ + set -- ${DESTINATION#/} + IFS=$old_ifs + last=$# + index=0 + for component do + index=$((index + 1)) + current="$current/$component" + [ ! -L "$current" ] || { echo "Managed promotion path contains a symlink component: $current" >&2; exit 1; } + if [ "$index" -lt "$last" ]; then [ -d "$current" ] || { echo "Managed promotion parent is not a directory: $current" >&2; exit 1; }; fi + done + [ -d "$parent" ] && [ ! -L "$parent" ] + [ ! -L "$host_destination" ] && [ ! -d "$host_destination" ] + stage="$parent/.${base}.${STAGE_TAG}.stage" + [ ! -e "$stage" ] && [ ! -L "$stage" ] + cleanup() { rm -f -- "$stage"; } + trap cleanup EXIT HUP INT TERM + install -o "${FILE_OWNER%:*}" -g "${FILE_OWNER#*:}" -m "${FILE_MODE}" /runtime/input "$stage" + mv -fT "$stage" "$host_destination" + trap - EXIT HUP INT TERM + ' +} + +install_host_path "${candidate_compose}" "${live_dir}/compose.host.yml" 0:0 0644 +install_host_path "${db_env}" "${live_dir}/envs/production/.env.db" 0:0 0644 +install_host_path "${rendered_hba}" "${hba_host_file}" 0:0 0444 +install_host_path "${bundle_dir}/bootstrap/keycloak-brio-staging.sql" "${live_dir}/bootstrap/keycloak-brio-staging.sql" 0:0 0644 +for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + install_host_path "${bundle_dir}/scripts/${script}" "${live_dir}/scripts/${script}" 0:0 0755 +done +install_host_path "${runtime_dir}/keycloak-brio-staging-backup-password" "${backup_host_file}" 999:999 0400 +install_host_path "${runtime_dir}/brio-backup-recipient-cert.pem" "${recipient_host_file}" 0:0 0444 +docker run --rm --mount type=bind,src=/var/lib,dst=/host/var/lib \ + "${postgres_image}" sh -euc ' + for path in /host/var /host/var/lib /host/var/lib/makepad /host/var/lib/makepad/postgres-backups; do + [ -d "$path" ] && [ ! -L "$path" ] || { echo "Identity backup promotion path is unsafe: $path" >&2; exit 1; } + done + install -d -o 999 -g 999 -m 0700 /host/var/lib/makepad/postgres-backups/keycloak-brio-staging + ' +[[ $(stat -c '%u:%a' "${hba_host_file}") == "0:444" && ! -L "${hba_host_file}" ]] || { echo "Active HBA installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_file}") == "999:400" && ! -L "${backup_host_file}" ]] || { echo "Identity backup credential installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${recipient_host_file}") == "0:444" && ! -L "${recipient_host_file}" ]] || { echo "Backup recipient installation failed its ownership/mode check." >&2; exit 1; } +[[ $(stat -c '%u:%a' "${backup_host_dir}") == "999:700" && ! -L "${backup_host_dir}" ]] || { echo "Identity backup directory installation failed its ownership/mode check." >&2; exit 1; } + +case "${failure_injection}" in + after-managed-file-promotion) echo "Injected failure after identity managed-file promotion." >&2; exit 97 ;; + term-after-managed-file-promotion) kill -TERM "$$" ;; + kill-after-managed-file-promotion) kill -KILL "$$" ;; +esac + +compose=(docker compose --project-name postgres --env-file "${live_dir}/envs/production/.env.db" -f "${live_dir}/compose.host.yml") +"${compose[@]}" up -d --wait --force-recreate postgres +validate_postgres_target "${postgres_image}" >/dev/null + +journal_marker "${journal_dir}" DATABASE_MUTATION_ARMED +db_mutated=1 +docker run --rm --network host \ + -v "${superuser_host_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${runtime_dir}/keycloak-brio-staging-app-password:/run/secrets/keycloak_app_password:ro" \ + -v "${runtime_dir}/keycloak-brio-staging-backup-password:/run/secrets/keycloak_backup_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + -v "${live_dir}/bootstrap/keycloak-brio-staging.sql:/bootstrap/keycloak-brio-staging.sql:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export KEYCLOAK_APP_PASSWORD="$(cat /run/secrets/keycloak_app_password)" + export KEYCLOAK_BACKUP_PASSWORD="$(cat /run/secrets/keycloak_backup_password)" + export PGHOST="$1" PGHOSTADDR=127.0.0.1 PGSSLMODE=verify-full PGSSLROOTCERT=/etc/postgresql/ca.crt + { + printf "%s\n" "\\getenv keycloak_brio_staging_app_password KEYCLOAK_APP_PASSWORD" "\\getenv keycloak_brio_staging_backup_password KEYCLOAK_BACKUP_PASSWORD" + cat /bootstrap/keycloak-brio-staging.sql + } > /tmp/bootstrap.sql + exec psql -X -v ON_ERROR_STOP=1 -U "$2" -d postgres -f /tmp/bootstrap.sql + ' sh "${db_hostname}" "${postgres_user}" >/dev/null + +case "${failure_injection}" in + after-bootstrap) echo "Injected failure after identity database bootstrap." >&2; exit 96 ;; + kill-after-bootstrap) kill -KILL "$$" ;; +esac + +run_role_query() { + local password_file=$1 role=$2 database=$3 sslmode=$4 query=$5 + docker run --rm --network host \ + -v "${password_file}:/run/secrets/role_password:ro" \ + -v "${ca_host_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/role_password)" + export PGHOST="$1" PGHOSTADDR=127.0.0.1 PGSSLMODE="$4" PGSSLROOTCERT=/etc/postgresql/ca.crt + exec psql -X -At -U "$2" -d "$3" -c "$5" + ' sh "${db_hostname}" "${role}" "${database}" "${sslmode}" "${query}" +} + +if [[ $(run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app keycloak_brio_staging verify-full "select current_database() || ':' || current_user") != "keycloak_brio_staging:keycloak_brio_staging_app" ]]; then + echo "Keycloak Brio role failed the local verify-full identity probe." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-app-probe ]] || { echo "Injected failure after identity app-role probe." >&2; exit 95; } +if run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app keycloak_brio_staging disable "select 1" >/dev/null 2>&1; then + echo "Plaintext Keycloak Brio database access was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-plaintext-probe ]] || { echo "Injected failure after identity plaintext probe." >&2; exit 94; } +if run_role_query "${runtime_dir}/keycloak-brio-staging-app-password" keycloak_brio_staging_app postgres verify-full "select 1" >/dev/null 2>&1; then + echo "Keycloak Brio role was unexpectedly accepted by a non-target database." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-nontarget-probe ]] || { echo "Injected failure after identity non-target probe." >&2; exit 93; } +if [[ $(run_role_query "${runtime_dir}/keycloak-brio-staging-backup-password" keycloak_brio_staging_backup keycloak_brio_staging verify-full "show default_transaction_read_only") != "on" ]]; then + echo "Keycloak Brio backup role is not read-only." >&2 + exit 1 +fi +[[ "${failure_injection}" != after-backup-role-probe ]] || { echo "Injected failure after identity backup-role probe." >&2; exit 92; } + +previous_latest=$(docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + "${validation_image}" sh -euc 'readlink /backups/latest 2>/dev/null || true') +backup_started_at=$(date +%s) +"${compose[@]}" up -d --no-deps --force-recreate keycloak_brio_staging_backup +backup_verified=0 +for _ in $(seq 1 60); do + if docker run --rm --mount "type=bind,src=${backup_host_dir},dst=/backups,readonly" \ + -e "BACKUP_STARTED_AT=${backup_started_at}" -e "PREVIOUS_LATEST=${previous_latest}" \ + "${validation_image}" sh -euc ' + status=/backups/last-success.json + [ -s "$status" ] && [ "$(stat -c %Y "$status")" -ge "$BACKUP_STARTED_AT" ] + grep -q "\"database\":\"keycloak_brio_staging\"" "$status" + grep -q "\"encrypted\":true" "$status" + latest=$(readlink /backups/latest) + case "$latest" in 20??????T??????Z) ;; *) exit 1 ;; esac + [ "$latest" != "$PREVIOUS_LATEST" ] + directory="/backups/$latest" + [ -s "$directory/keycloak_brio_staging.dump.cms" ] && [ -s "$directory/SHA256SUMS" ] + (cd "$directory" && sha256sum --check --status SHA256SUMS) + openssl cms -cmsout -inform DER -in "$directory/keycloak_brio_staging.dump.cms" -noout >/dev/null 2>&1 + '; then + backup_verified=1 + break + fi + sleep 2 +done +[[ "${backup_verified}" == "1" ]] || { echo "A fresh validated encrypted keycloak_brio_staging backup was not published." >&2; exit 1; } +[[ "${failure_injection}" != after-backup-verification ]] || { echo "Injected failure after identity backup verification." >&2; exit 91; } + +journal_marker "${journal_dir}" COMMITTED +rollback_armed=0 +remove_durable_journal "${journal_dir}" +echo "Standalone Brio identity PostgreSQL target, rollback snapshot, HBA, bootstrap, local TLS policy, and encrypted backup verification passed." +echo "Release acceptance still requires the protected Keycloak-host Verify Brio Identity Database Path workflow for this PostgreSQL run ID." diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh new file mode 100755 index 0000000..d995c0b --- /dev/null +++ b/scripts/deploy-postgres-stack.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# < 3 || $# > 4)); then + echo "Usage: deploy-postgres-stack.sh [production-vif-runtime-secret-dir]" >&2 + exit 2 +fi + +remote_dir=$1 +stack_name=$2 +deploy_env=$3 +vif_runtime_dir=${4:-} +if [[ ! "${remote_dir}" =~ ^/(srv|opt)/[A-Za-z0-9._/-]+/\.deploy/postgres-[0-9]+-[0-9]+$ ]] \ + || [[ "${remote_dir}" == *"/../"* || "${remote_dir}" == *"/.." || "${remote_dir}" == *"/./"* || "${remote_dir}" == *"/." || "${remote_dir}" == *"//"* ]]; then + echo "job-bundle-dir must be a unique /srv or /opt .deploy/postgres-- path." >&2 + exit 2 +fi +case "${stack_name}" in ''|*[!a-zA-Z0-9_-]*) echo "stack-name contains unsupported characters." >&2; exit 2 ;; esac +case "${deploy_env}" in canary|production) ;; *) echo "Deployment environment must be canary or production." >&2; exit 2 ;; esac +if [[ "${deploy_env}" == "production" ]]; then + [[ $# -eq 4 && "${vif_runtime_dir}" =~ ^/tmp/postgres-brio-vif-runtime-[0-9]+-[0-9]+$ ]] || { + echo "Production requires a job-scoped /tmp/postgres-brio-vif-runtime-- directory." >&2 + exit 2 + } +elif [[ $# -ne 3 ]]; then + echo "Canary deployment must not receive a VIF credential directory." >&2 + exit 2 +fi + +server_certificate= +cleanup_deploy_material() { + [[ -z "${server_certificate}" ]] || rm -f -- "${server_certificate}" + if [[ -n "${vif_runtime_dir}" && -d "${vif_runtime_dir}" && ! -L "${vif_runtime_dir}" ]]; then + rm -f -- "${vif_runtime_dir}/vif-db-password" + rmdir -- "${vif_runtime_dir}" 2>/dev/null || true + fi +} +trap cleanup_deploy_material EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +env_deploy="${remote_dir}/envs/${deploy_env}/.env.deploy" +db_env="${remote_dir}/envs/${deploy_env}/.env.db" +db_network=$(grep '^MAKEPAD_POSTGRES_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) +le_petit_coin_db_network=$(grep '^MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) +postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_root_user=$(grep '^POSTGRES_USER=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_root_password_file=$(grep '^MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_tls_cert_config=$(grep '^MAKEPAD_POSTGRES_TLS_CERT_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_tls_key_secret=$(grep '^MAKEPAD_POSTGRES_TLS_KEY_SECRET=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_runtrace_hba_config=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-) +runtrace_backup_path=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +runtrace_backup_password_file=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +postgres_ca_cert_file=$(grep '^MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +vif_enabled=0 +brio_staging_enabled=0 +if [[ "${deploy_env}" == "canary" ]]; then + brio_staging_enabled=1 + brio_backup_image=$(grep '^BRIO_BACKUP_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_backup_recipient_cert=$(grep '^MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_staging_db_network=$(grep '^MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + brio_backup_path=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) + brio_backup_password_file=$(grep '^MAKEPAD_POSTGRES_BRIO_APP_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-) +fi +if [[ "${deploy_env}" == "production" ]]; then + vif_enabled=1 + vif_db_network=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_name=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NAME=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_user=$(grep '^MAKEPAD_POSTGRES_VIF_DB_USER=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) + vif_db_password_file="${vif_runtime_dir}/vif-db-password" +fi +: "${db_network:?MAKEPAD_POSTGRES_DB_NETWORK is missing or empty in ${env_deploy}}" +: "${le_petit_coin_db_network:?MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK is missing or empty in ${env_deploy}}" +: "${postgres_image:?POSTGRES_IMAGE is missing or empty in ${db_env}}" +: "${postgres_root_user:?POSTGRES_USER is missing or empty in ${db_env}}" +: "${postgres_root_password_file:?MAKEPAD_POSTGRES_SUPERUSER_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" +: "${postgres_tls_cert_config:?MAKEPAD_POSTGRES_TLS_CERT_CONFIG is missing or empty in ${db_env}}" +: "${postgres_tls_key_secret:?MAKEPAD_POSTGRES_TLS_KEY_SECRET is missing or empty in ${db_env}}" +: "${postgres_runtrace_hba_config:?MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG is missing or empty in ${db_env}}" +: "${postgres_ca_cert_file:?MAKEPAD_POSTGRES_CA_CERT_HOST_PATH is missing or empty in ${db_env}}" +if [[ "${deploy_env}" == "canary" ]]; then + : "${brio_backup_image:?BRIO_BACKUP_IMAGE is missing or empty in ${db_env}}" + : "${brio_backup_recipient_cert:?MAKEPAD_POSTGRES_BRIO_BACKUP_RECIPIENT_CERT_HOST_PATH is missing or empty in ${db_env}}" + : "${brio_backup_path:?Brio backup directory is missing or empty in ${db_env}}" + : "${brio_backup_password_file:?Brio backup password-file path is missing or empty in ${db_env}}" +fi +if [[ "${deploy_env}" == "production" ]]; then + : "${runtrace_backup_path:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH is missing or empty in ${db_env}}" + : "${runtrace_backup_password_file:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}" +fi +if [[ ! -s "${postgres_root_password_file}" ]]; then + echo "PostgreSQL superuser password file is missing or empty: ${postgres_root_password_file}" >&2 + exit 1 +fi +if ! docker config inspect "${postgres_tls_cert_config}" >/dev/null 2>&1; then + echo "PostgreSQL TLS certificate config does not exist: ${postgres_tls_cert_config}" >&2 + exit 1 +fi +if ! docker secret inspect "${postgres_tls_key_secret}" >/dev/null 2>&1; then + echo "PostgreSQL TLS private-key secret does not exist: ${postgres_tls_key_secret}" >&2 + exit 1 +fi +command -v openssl >/dev/null 2>&1 || { + echo "openssl is required for PostgreSQL certificate preflight." >&2 + exit 1 +} +if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then + echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2 + exit 1 +fi +postgres_ca_mode=$(stat -c '%a' "${postgres_ca_cert_file}") +if (( (8#${postgres_ca_mode} & 8#022) != 0 )); then + echo "PostgreSQL CA certificate must not be group- or world-writable: ${postgres_ca_cert_file}" >&2 + exit 1 +fi +if [[ "${deploy_env}" == "canary" ]]; then + for backup_script in run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + if [[ ! -x "${remote_dir}/scripts/${backup_script}" || -L "${remote_dir}/scripts/${backup_script}" ]]; then + echo "Brio backup script must be an executable, non-symlink file: ${remote_dir}/scripts/${backup_script}" >&2 + exit 1 + fi + done + if [[ ! -d "${brio_backup_path}" || -L "${brio_backup_path}" ]]; then + echo "Brio backup path must be a pre-provisioned non-symlink directory: ${brio_backup_path}" >&2 + exit 1 + fi + brio_backup_directory_mode=$(stat -c '%a' "${brio_backup_path}") + brio_backup_directory_uid=$(stat -c '%u' "${brio_backup_path}") + if [[ "${brio_backup_directory_mode}" != "700" || "${brio_backup_directory_uid}" != "999" ]]; then + echo "Brio backup path must be owned by uid 999 with mode 0700: ${brio_backup_path}" >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_password_file}" || -L "${brio_backup_password_file}" ]]; then + echo "Brio backup credential must be a non-empty, non-symlink file: ${brio_backup_password_file}" >&2 + exit 1 + fi + brio_backup_password_mode=$(stat -c '%a' "${brio_backup_password_file}") + brio_backup_password_uid=$(stat -c '%u' "${brio_backup_password_file}") + if [[ "${brio_backup_password_mode}" != "400" || "${brio_backup_password_uid}" != "999" ]]; then + echo "Brio backup credential must be owned by uid 999 with mode 0400." >&2 + exit 1 + fi + if [[ ! -s "${brio_backup_recipient_cert}" || -L "${brio_backup_recipient_cert}" ]] || grep -q -- 'PRIVATE KEY' "${brio_backup_recipient_cert}"; then + echo "Brio backup recipient must be a public, non-symlink X.509 certificate: ${brio_backup_recipient_cert}" >&2 + exit 1 + fi + brio_backup_recipient_mode=$(stat -c '%a' "${brio_backup_recipient_cert}") + brio_backup_recipient_uid=$(stat -c '%u' "${brio_backup_recipient_cert}") + if [[ "${brio_backup_recipient_uid}" != "0" ]] || (( (8#${brio_backup_recipient_mode} & 8#022) != 0 )); then + echo "Brio backup recipient certificate must be root-owned and not group- or world-writable." >&2 + exit 1 + fi + if ! openssl x509 -in "${brio_backup_recipient_cert}" -noout -checkend 604800 >/dev/null \ + || ! printf 'brio-backup-preflight' | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm -recip "${brio_backup_recipient_cert}" -out /dev/null; then + echo "Brio backup recipient certificate is invalid, unsuitable for CMS encryption, or expires in less than seven days." >&2 + exit 1 + fi +fi +server_certificate=$(mktemp) +docker config inspect "${postgres_tls_cert_config}" --format '{{printf "%s" .Spec.Data}}' > "${server_certificate}" +if ! openssl x509 -in "${server_certificate}" -noout -checkend 604800 >/dev/null; then + echo "PostgreSQL TLS certificate is invalid or expires in less than seven days." >&2 + exit 1 +fi +if ! openssl verify -purpose sslserver -CAfile "${postgres_ca_cert_file}" -untrusted "${server_certificate}" "${server_certificate}" >/dev/null; then + echo "PostgreSQL TLS certificate does not chain to the configured CA." >&2 + exit 1 +fi +if [[ "${deploy_env}" == "canary" ]] && ! openssl x509 -in "${server_certificate}" -noout -checkhost makepad-postgres-brio-staging >/dev/null; then + echo "Canary PostgreSQL TLS certificate does not cover makepad-postgres-brio-staging." >&2 + exit 1 +fi +if [[ "${deploy_env}" == "production" ]]; then + if [[ ! -d "${runtrace_backup_path}" || -L "${runtrace_backup_path}" ]]; then + echo "Runtrace backup path must be a pre-provisioned non-symlink directory: ${runtrace_backup_path}" >&2 + exit 1 + fi + backup_directory_mode=$(stat -c '%a' "${runtrace_backup_path}") + backup_directory_uid=$(stat -c '%u' "${runtrace_backup_path}") + if [[ "${backup_directory_mode}" != "700" || "${backup_directory_uid}" != "70" ]]; then + echo "Runtrace backup path must be owned by uid 70 with mode 0700: ${runtrace_backup_path}" >&2 + exit 1 + fi + if [[ ! -s "${runtrace_backup_password_file}" || -L "${runtrace_backup_password_file}" ]]; then + echo "Runtrace backup credential must be a non-empty, non-symlink file: ${runtrace_backup_password_file}" >&2 + exit 1 + fi + backup_password_mode=$(stat -c '%a' "${runtrace_backup_password_file}") + backup_password_uid=$(stat -c '%u' "${runtrace_backup_password_file}") + if [[ "${backup_password_mode}" != "400" || "${backup_password_uid}" != "70" ]]; then + echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2 + exit 1 + fi +fi +hba_path="${remote_dir}/config/runtrace-pg_hba.conf" +hba_sha256=$(sha256sum "${hba_path}" | awk '{print $1}') +if docker config inspect "${postgres_runtrace_hba_config}" >/dev/null 2>&1; then + deployed_hba_sha256=$(docker config inspect "${postgres_runtrace_hba_config}" --format '{{index .Spec.Labels "content-sha256"}}') + if [[ "${deployed_hba_sha256}" != "${hba_sha256}" ]]; then + echo "PostgreSQL HBA config ${postgres_runtrace_hba_config} does not match the repository policy. Create a new versioned config name and update MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG." >&2 + exit 1 + fi +else + docker config create --label "content-sha256=${hba_sha256}" "${postgres_runtrace_hba_config}" "${hba_path}" >/dev/null +fi +if [[ "${vif_enabled}" == "1" ]]; then + : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" + : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" + : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" + if [[ ! -s "${vif_db_password_file}" || -L "${vif_db_password_file}" \ + || $(stat -c '%a' "${vif_db_password_file}") != "600" \ + || $(stat -c '%a' "${vif_runtime_dir}") != "700" || -L "${vif_runtime_dir}" ]]; then + echo "VIF credential material must be a mode-0600 file in the mode-0700 job runtime directory." >&2 + exit 1 + fi + if [[ $(awk 'END { print NR }' "${vif_db_password_file}") -ne 1 ]] || grep -q $'\r' "${vif_db_password_file}"; then + echo "VIF credential must contain one line and no carriage return." >&2 + exit 1 + fi +fi +if [[ "${brio_staging_enabled}" == "1" ]]; then + : "${brio_staging_db_network:?MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK is missing or empty in ${env_deploy}}" + if [[ "${brio_staging_db_network}" != "makepad_brio_staging_db" ]]; then + echo "Brio deployment bundle must use makepad_brio_staging_db." >&2 + exit 1 + fi +fi + +ensure_encrypted_overlay_network() { + local network_name=$1 + if docker network inspect "${network_name}" >/dev/null 2>&1; then + local driver scope encrypted + driver=$(docker network inspect "${network_name}" --format '{{.Driver}}') + scope=$(docker network inspect "${network_name}" --format '{{.Scope}}') + encrypted=$(docker network inspect "${network_name}" --format '{{index .Options "encrypted"}}') + if [[ "${driver}" != "overlay" || "${scope}" != "swarm" || "${encrypted}" != "true" ]]; then + echo "Database network ${network_name} must be a Swarm overlay with encrypted=true. Drain dependent services, recreate it with --opt encrypted=true, then rerun this deployment." >&2 + exit 1 + fi + return + fi + docker network create --driver overlay --attachable --opt encrypted=true "${network_name}" >/dev/null +} + +ensure_internal_encrypted_overlay_network() { + local network_name=$1 + if ! docker network inspect "${network_name}" >/dev/null 2>&1; then + # A concurrent consumer can observe the same absence. Validate the final + # object below instead of trusting the create command's exit status. + docker network create --driver overlay --attachable --internal --opt encrypted=true \ + --label com.makepad.owner=Makepad-fr/postgres \ + --label com.makepad.environment=staging \ + --label com.makepad.instance=brio \ + --label com.makepad.purpose=database "${network_name}" >/dev/null 2>&1 || true + fi + local details expected + details=$(docker network inspect "${network_name}" --format '{{.Driver}}|{{.Scope}}|{{.Internal}}|{{.Attachable}}|{{index .Options "encrypted"}}|{{index .Labels "com.makepad.owner"}}|{{index .Labels "com.makepad.environment"}}|{{index .Labels "com.makepad.instance"}}|{{index .Labels "com.makepad.purpose"}}') || { + echo "Brio database network ${network_name} does not exist after provisioning." >&2 + exit 1 + } + expected='overlay|swarm|true|true|true|Makepad-fr/postgres|staging|brio|database' + if [[ "${details}" != "${expected}" ]]; then + echo "Brio database network ${network_name} must have the required isolation and PostgreSQL ownership metadata." >&2 + exit 1 + fi +} + +ensure_encrypted_overlay_network "${db_network}" +ensure_encrypted_overlay_network "${le_petit_coin_db_network}" +if [[ "${vif_enabled}" == "1" ]]; then + ensure_encrypted_overlay_network "${vif_db_network}" + export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" +fi +if [[ "${brio_staging_enabled}" == "1" ]]; then + ensure_internal_encrypted_overlay_network "${brio_staging_db_network}" + export MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK="${brio_staging_db_network}" +fi + +export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" +export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" +generated_dir="${remote_dir}/generated" +install -d -m 0700 "${generated_dir}" +stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml" +docker compose \ + --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ + --env-file "${env_deploy}" \ + -f "${remote_dir}/compose.yml" \ + -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ + config > "${stack_file}" + +docker stack deploy --compose-file "${stack_file}" "${stack_name}" + +wait_for_service_convergence() { + local service_name=$1 + local expected_image=$2 + local update_state desired running_snapshot running_count wrong_image + for _ in $(seq 1 60); do + if ! docker service inspect "${service_name}" >/dev/null 2>&1; then + sleep 2 + continue + fi + update_state=$(docker service inspect "${service_name}" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{else}}none{{end}}') + case "${update_state}" in + paused|rollback_started|rollback_paused|rollback_completed) + echo "Service ${service_name} update did not complete successfully: ${update_state}." >&2 + docker service ps --no-trunc "${service_name}" >&2 + return 1 + ;; + updating) + sleep 2 + continue + ;; + esac + desired=$(docker service inspect "${service_name}" --format '{{.Spec.Mode.Replicated.Replicas}}') + running_snapshot=$(docker service ps --no-trunc --filter desired-state=running --format '{{.Image}} {{.CurrentState}}' "${service_name}") + running_count=$(printf '%s\n' "${running_snapshot}" | awk '$2 == "Running" {count++} END {print count + 0}') + wrong_image=$(printf '%s\n' "${running_snapshot}" | awk -v expected="${expected_image}" '$2 == "Running" && $1 != expected {print $1; exit}') + if [[ "${running_count}" == "${desired}" && -z "${wrong_image}" && ( "${update_state}" == "completed" || "${update_state}" == "none" ) ]]; then + return 0 + fi + sleep 2 + done + echo "Service ${service_name} did not converge to ${expected_image}." >&2 + docker service ps --no-trunc "${service_name}" >&2 || true + return 1 +} + +wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}" +if [[ "${deploy_env}" == "canary" ]]; then + wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}" +fi + +if [[ "${brio_staging_enabled}" == "1" ]]; then + brio_tls_ready=0 + for _ in $(seq 1 30); do + if docker run --rm --network "${brio_staging_db_network}" \ + -e PGSSLMODE=verify-full \ + -e PGSSLROOTCERT=/etc/postgresql/ca.crt \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${postgres_ca_cert_file}:/etc/postgresql/ca.crt:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-brio-staging -U "${postgres_root_user}" -d postgres -Atc "select 1" >/dev/null 2>&1; then + brio_tls_ready=1 + break + fi + sleep 2 + done + if [[ "${brio_tls_ready}" != "1" ]]; then + echo "PostgreSQL did not pass sslmode=verify-full using makepad-postgres-brio-staging within 60 seconds." >&2 + exit 1 + fi +fi + +if [[ "${vif_enabled}" != "1" ]]; then + exit 0 +fi + +postgres_ready=0 +for _ in $(seq 1 30); do + if docker run --rm --network "${vif_db_network}" \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + "${postgres_image}" sh -ec 'export PGPASSWORD=$(cat /run/secrets/postgres_superuser_password); exec psql "$@"' sh \ + -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres -c "select 1" >/dev/null 2>&1; then + postgres_ready=1 + break + fi + sleep 2 +done +if [[ "${postgres_ready}" != "1" ]]; then + echo "Postgres did not become reachable via makepad-postgres-vif on ${vif_db_network} after 60 seconds." >&2 + exit 1 +fi + +docker run --rm --interactive --network "${vif_db_network}" \ + -v "${postgres_root_password_file}:/run/secrets/postgres_superuser_password:ro" \ + -v "${vif_db_password_file}:/run/secrets/vif_db_password:ro" \ + "${postgres_image}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres_superuser_password)" + export VIF_PASSWORD="$(cat /run/secrets/vif_db_password)" + exec psql -X -v ON_ERROR_STOP=1 -h makepad-postgres-vif -U "$1" -d postgres \ + -v vif_db="$2" -v vif_user="$3" + ' sh "${postgres_root_user}" "${vif_db_name}" "${vif_db_user}" >/dev/null <<'SQL' +\set ON_ERROR_STOP on + +\if :{?vif_db} +\else + \echo 'missing required psql variable: vif_db' + SELECT 1 / 0; +\endif + +\if :{?vif_user} +\else + \echo 'missing required psql variable: vif_user' + SELECT 1 / 0; +\endif + +\getenv vif_password VIF_PASSWORD +SELECT CASE WHEN NULLIF(btrim(:'vif_password'), '') IS NULL THEN 'false' ELSE 'true' END AS vif_password_is_nonempty \gset +\if :vif_password_is_nonempty +\else + \echo 'empty required environment variable: VIF_PASSWORD' + SELECT 1 / 0; +\endif + +SELECT format('CREATE ROLE %I LOGIN', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec +SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec +SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = :'vif_db' + AND r.rolname <> :'vif_user' +) \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec +SQL diff --git a/scripts/ensure-brio-tmp-cleaner.sh b/scripts/ensure-brio-tmp-cleaner.sh new file mode 100755 index 0000000..75dc080 --- /dev/null +++ b/scripts/ensure-brio-tmp-cleaner.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +clean_test_root() { + local test_root=$1 + [[ "${test_root}" =~ ^/tmp/postgres-brio-cleaner-test-[A-Za-z0-9._-]+$ ]] || { + echo "Test cleanup root must use /tmp/postgres-brio-cleaner-test-." >&2 + exit 2 + } + find "${test_root}" -mindepth 1 -maxdepth 1 -type d \ + \( -name 'postgres-brio-*' -o -name 'postgres-keycloak-cohort-*' \) -mmin +180 \ + -exec sh -euc 'for directory do + [ ! -L "${directory}" ] || continue + [ ! -f "${directory}/RECOVERY_REQUIRED" ] || continue + find "${directory}" -depth -delete + done' sh {} + +} + +if (($# == 2)) && [[ "$1" == "test-clean-once" ]]; then + clean_test_root "$2" + exit 0 +fi + +if (($# == 3)) && [[ "$1" == "test-clean-production-ownership" ]]; then + test_root=$2 + test_image=$3 + [[ "${test_root}" =~ ^/tmp/postgres-brio-cleaner-test-[A-Za-z0-9._-]+$ \ + && "${test_image}" == *@sha256:* ]] || { echo "Unsafe production-ownership cleaner test inputs." >&2; exit 2; } + docker run --rm --read-only --cap-drop ALL --cap-add DAC_OVERRIDE --cap-add FOWNER \ + --security-opt no-new-privileges --mount "type=bind,src=${test_root},dst=/host-tmp" \ + "${test_image}" sh -euc ' + find /host-tmp -mindepth 1 -maxdepth 1 -type d \ + \( -name "postgres-brio-*" -o -name "postgres-keycloak-cohort-*" \) -mmin +180 \ + -exec sh -euc '\''for directory do + [ ! -L "$directory" ] || continue + [ ! -f "$directory/RECOVERY_REQUIRED" ] || continue + find "$directory" -depth -delete + done'\'' sh {} + + ' + exit 0 +fi + +if (($# != 1)); then + echo "Usage: ensure-brio-tmp-cleaner.sh " >&2 + exit 2 +fi + +db_env=$1 +[[ -f "${db_env}" && ! -L "${db_env}" ]] || { echo "Database environment file is missing or a symlink." >&2; exit 2; } +postgres_image=$(grep '^POSTGRES_IMAGE=' "${db_env}" | tail -n 1 | cut -d= -f2-) +: "${postgres_image:?POSTGRES_IMAGE is missing from ${db_env}}" +[[ "${postgres_image}" == *@sha256:* ]] || { echo "The temporary-material cleaner requires a digest-pinned image." >&2; exit 1; } + +cleaner_name=makepad-postgres-brio-tmp-cleaner +cleaner_contract=brio-tmp-cleaner-v5-exact-command +# shellcheck disable=SC2016 # The nested shell expands directory inside the container. +cleaner_command='while :; do + find /host-tmp -mindepth 1 -maxdepth 1 -type d \ + \( -name "postgres-brio-*" -o -name "postgres-keycloak-cohort-*" \) -mmin +180 \ + -exec sh -euc '\''for directory do + [ ! -L "${directory}" ] || continue + [ ! -f "${directory}/RECOVERY_REQUIRED" ] || continue + find "${directory}" -depth -delete + done'\'' sh {} + + sleep 900 +done' + +verify_running() { + local state attempts=10 delay=1 + if [[ "${BRIO_DEPLOY_TEST_MODE:-}" == isolated-container ]]; then attempts=2; delay=0; fi + for _ in $(seq 1 "${attempts}"); do + state=$(docker container inspect "${cleaner_name}" --format '{{.State.Running}}|{{.State.Restarting}}' 2>/dev/null || true) + [[ "${state}" != "true|false" ]] || return 0 + sleep "${delay}" + done + echo "${cleaner_name} did not remain running after startup." >&2 + return 1 +} + +if docker container inspect "${cleaner_name}" >/dev/null 2>&1; then + image=$(docker container inspect "${cleaner_name}" --format '{{.Config.Image}}') + contract=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Labels "makepad.cleanup.contract"}}') + restart_policy=$(docker container inspect "${cleaner_name}" --format '{{.HostConfig.RestartPolicy.Name}}') + readonly_root=$(docker container inspect "${cleaner_name}" --format '{{.HostConfig.ReadonlyRootfs}}') + mount_contract=$(docker container inspect "${cleaner_name}" --format '{{range .Mounts}}{{if eq .Destination "/host-tmp"}}{{printf "%s|%s|%t" .Type .Source .RW}}{{end}}{{end}}') + cap_drop=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.CapDrop}}') + cap_add=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.CapAdd}}') + security_opt=$(docker container inspect "${cleaner_name}" --format '{{json .HostConfig.SecurityOpt}}') + observed_command_length=$(docker container inspect "${cleaner_name}" --format '{{len .Config.Cmd}}') + observed_shell=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 0}}') + observed_shell_flags=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 1}}') + observed_command=$(docker container inspect "${cleaner_name}" --format '{{index .Config.Cmd 2}}') + if [[ "${image}" != "${postgres_image}" || "${contract}" != "${cleaner_contract}" \ + || "${restart_policy}" != "unless-stopped" || "${readonly_root}" != "true" \ + || "${mount_contract}" != "bind|/tmp|true" || "${cap_drop}" != '["ALL"]' \ + || "${cap_add}" != '["DAC_OVERRIDE","FOWNER"]' \ + || "${security_opt}" != *'no-new-privileges'* || "${observed_command_length}" != 3 \ + || "${observed_shell}" != sh || "${observed_shell_flags}" != -euc \ + || "${observed_command}" != "${cleaner_command}" ]]; then + echo "Existing ${cleaner_name} does not match the fail-closed cleanup contract." >&2 + exit 1 + fi + if [[ $(docker container inspect "${cleaner_name}" --format '{{.State.Running}}') != "true" ]]; then + docker start "${cleaner_name}" >/dev/null + fi + verify_running + exit $? +fi + +docker run -d \ + --name "${cleaner_name}" \ + --restart unless-stopped \ + --read-only \ + --cap-drop ALL \ + --cap-add DAC_OVERRIDE \ + --cap-add FOWNER \ + --security-opt no-new-privileges \ + --label "makepad.cleanup.contract=${cleaner_contract}" \ + --mount type=bind,src=/tmp,dst=/host-tmp \ + "${postgres_image}" \ + sh -euc "${cleaner_command}" >/dev/null +verify_running diff --git a/scripts/fixtures/brio-deployment-failure-fixture.sh b/scripts/fixtures/brio-deployment-failure-fixture.sh new file mode 100755 index 0000000..d00bcf0 --- /dev/null +++ b/scripts/fixtures/brio-deployment-failure-fixture.sh @@ -0,0 +1,886 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/brio-mock-bin +install -d -m 0700 "${mock_bin}" + +cat > "${mock_bin}/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail + +remove_exact() { + local target=$1 + if [[ -L "${target}" || -f "${target}" ]]; then + rm -f -- "${target}" + elif [[ -d "${target}" ]]; then + find "${target}" -mindepth 1 -delete + rmdir -- "${target}" + elif [[ -e "${target}" ]]; then + echo "Refusing unexpected mock path type: ${target}" >&2 + exit 1 + fi +} + +map_mount() { + local specification=$1 source= destination= + IFS=',' read -r -a fields <<< "${specification}" + for field in "${fields[@]}"; do + case "${field}" in src=*|source=*) source=${field#*=} ;; dst=*|destination=*|target=*) destination=${field#*=} ;; esac + done + [[ -n "${source}" && -n "${destination}" ]] || return 0 + mock_mount_sources+=("${source}") + mock_mount_destinations+=("${destination}") + mkdir -p "$(dirname "${destination}")" + remove_exact "${destination}" + case "${destination}" in + /managed/live|/managed/etc) + mkdir -p "${destination}" + cp -a "${source}/." "${destination}/" + ;; + *) ln -s "${source}" "${destination}" ;; + esac +} + +create_backup() { + local database=$1 root timestamp directory recipient + case "${database}" in + brio_staging) + root=/var/lib/makepad/postgres-backups/brio-staging + recipient=/etc/makepad/tls/backups/brio-recipient.crt + ;; + keycloak_brio_staging) + root=/var/lib/makepad/postgres-backups/keycloak-brio-staging + recipient=/etc/makepad/tls/backups/brio-recipient.crt + ;; + *) return 1 ;; + esac + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + directory="${root}/${timestamp}" + mkdir -p "${directory}" + printf 'mock-%s-backup\n' "${database}" | openssl cms -encrypt -binary -stream -outform DER -aes-256-gcm \ + -recip "${recipient}" -out "${directory}/${database}.dump.cms" 2>/dev/null + (cd "${directory}" && sha256sum "${database}.dump.cms" > SHA256SUMS) + ln -sfn "${timestamp}" "${root}/latest" + printf '{"database":"%s","encrypted":true}\n' "${database}" > "${root}/last-success.json" +} + +command_name=${1:-} +shift || true +case "${command_name}" in + info) + printf '%s\n' inactive + ;; + pull) + ;; + image) + [[ "${1:-}" == inspect ]] + ;; + container) + [[ "${1:-}" == inspect ]] || exit 1 + shift + target=${1:-} + shift || true + format= + while (($#)); do + if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi + done + if [[ "${target}" == makepad-postgres-brio-tmp-cleaner ]]; then + cleaner_state=/tmp/mock-cleaner-state + [[ -d "${cleaner_state}" ]] || exit 1 + case "${format}" in + '') ;; + *'.State.Running'*'.State.Restarting'*) printf '%s|false\n' "$(< "${cleaner_state}/running")" ;; + *'.Config.Image'*) cat "${cleaner_state}/image" ;; + *'makepad.cleanup.contract'*) printf '%s\n' brio-tmp-cleaner-v5-exact-command ;; + *'.HostConfig.RestartPolicy.Name'*) printf '%s\n' unless-stopped ;; + *'.HostConfig.ReadonlyRootfs'*) printf '%s\n' true ;; + *'.Mounts'*) printf '%s\n' 'bind|/tmp|true' ;; + *'.HostConfig.CapDrop'*) printf '%s\n' '["ALL"]' ;; + *'.HostConfig.CapAdd'*) printf '%s\n' '["DAC_OVERRIDE","FOWNER"]' ;; + *'.HostConfig.SecurityOpt'*) printf '%s\n' '["no-new-privileges"]' ;; + *'len .Config.Cmd'*) printf '%s\n' 3 ;; + *'index .Config.Cmd 0'*) printf '%s\n' sh ;; + *'index .Config.Cmd 1'*) printf '%s\n' -euc ;; + *'index .Config.Cmd 2'*) cat "${cleaner_state}/command" ;; + *'.State.Running'*) cat "${cleaner_state}/running" ;; + *) echo "Unhandled cleaner container format: ${format}" >&2; exit 1 ;; + esac + exit 0 + fi + [[ "${target}" == postgres-postgres-1 || "${target}" == mock-container-id ]] || exit 1 + case "${format}" in + *'.Id'*) printf '%s\n' mock-container-id ;; + *'.Name'*) printf '%s\n' /postgres-postgres-1 ;; + *'com.docker.compose.project'*) printf '%s\n' "${MOCK_COMPOSE_PROJECT:-postgres}" ;; + *'com.docker.compose.service'*) printf '%s\n' postgres ;; + *'com.docker.compose.oneoff'*) printf '%s\n' False ;; + *'.HostConfig.NetworkMode'*) printf '%s\n' host ;; + *'.Config.Image'*) printf '%s\n' "${MOCK_POSTGRES_IMAGE}" ;; + *'.Mounts'*) printf '%s\n' 'bind|/var/lib/makepad/postgres|true' ;; + *'.State.Running'*) printf '%s\n' true ;; + *'.State.Health'*) printf '%s\n' healthy ;; + *) echo "Unhandled container format: ${format}" >&2; exit 1 ;; + esac + ;; + start) + [[ "${1:-}" == makepad-postgres-brio-tmp-cleaner && -d /tmp/mock-cleaner-state ]] || exit 1 + if [[ ! -f /tmp/mock-cleaner-stop-after-start ]]; then printf '%s\n' true > /tmp/mock-cleaner-state/running; fi + ;; + compose) + if printf '%s\n' "$@" | grep -Fxq config; then + if [[ " ${*} " == *' --quiet '* ]]; then exit 0; fi + cat <<'YAML' +services: + postgres: + network_mode: host + volumes: + - type: bind + source: /var/lib/makepad/postgres + target: /var/lib/postgresql/data +YAML + exit 0 + fi + if printf '%s\n' "$@" | grep -Fxq up; then + counter_file=/tmp/mock-compose-up-count + count=0 + [[ ! -f "${counter_file}" ]] || read -r count < "${counter_file}" + count=$((count + 1)) + printf '%s\n' "${count}" > "${counter_file}" + if [[ "${MOCK_COMPOSE_FAIL_FIRST:-0}" == 1 && "${count}" == 1 ]]; then exit 44; fi + if [[ " ${*} " == *' keycloak_brio_staging_backup '* ]]; then + create_backup keycloak_brio_staging + fi + exit 0 + fi + echo "Unhandled docker compose invocation" >&2 + exit 1 + ;; + config|secret|network) + object_kind=${command_name} + action=${1:-} + shift || true + state_root=/tmp/mock-docker-state + case "${action}" in + inspect) + object_name=${1:-} + state_file="${state_root}/${object_kind}/${object_name}" + [[ -f "${state_file}" ]] || exit 1 + format= + while (($#)); do + if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi + done + if [[ "${format}" == *makepad.brio.deployment-id* ]]; then + sed -n 's/^deployment_id=//p' "${state_file}" + elif [[ "${format}" == *content-sha256* ]]; then + sed -n 's/^digest=//p' "${state_file}" + elif [[ "${object_kind}" == config && "${format}" == *'.Spec.Data'* ]]; then + cat "${state_root}/config-data/${object_name}" + elif [[ "${object_kind}" == network && -n "${format}" ]]; then + internal=$(sed -n 's/^internal=//p' "${state_file}") + printf 'overlay swarm %s true true\n' "${internal:-false}" + fi + ;; + create) + mkdir -p "${state_root}/${object_kind}" + declare -a values=("$@") + object_name=${values[${#values[@]}-1]} + [[ "${object_kind}" == network ]] || object_name=${values[${#values[@]}-2]} + deployment_id= + digest= + internal=false + for ((index=0; index<${#values[@]}; index++)); do + if [[ "${values[index]}" == --label && $((index + 1)) -lt ${#values[@]} ]]; then + case "${values[index+1]}" in + makepad.brio.deployment-id=*) deployment_id=${values[index+1]#*=} ;; + content-sha256=*) digest=${values[index+1]#*=} ;; + esac + elif [[ "${values[index]}" == --internal ]]; then + internal=true + fi + done + printf 'deployment_id=%s\ndigest=%s\ninternal=%s\n' "${deployment_id}" "${digest}" "${internal}" > "${state_root}/${object_kind}/${object_name}" + if [[ "${object_kind}" == config ]]; then + mkdir -p "${state_root}/config-data" + cp "${values[${#values[@]}-1]}" "${state_root}/config-data/${object_name}" + fi + ;; + rm) + rm -f -- "${state_root}/${object_kind}/${1:-}" + [[ "${object_kind}" != config ]] || rm -f -- "${state_root}/config-data/${1:-}" + ;; + ls) + requested_owner= + while (($#)); do + case "$1" in + --filter) requested_owner=${2#label=makepad.brio.deployment-id=}; shift 2 ;; + --format) shift 2 ;; + *) shift ;; + esac + done + [[ -d "${state_root}/${object_kind}" ]] || exit 0 + for state_file in "${state_root}/${object_kind}"/*; do + [[ -f "${state_file}" ]] || continue + [[ -z "${requested_owner}" || $(sed -n 's/^deployment_id=//p' "${state_file}") == "${requested_owner}" ]] || continue + basename "${state_file}" + done + ;; + *) exit 1 ;; + esac + ;; + stack) + action=${1:-} + case "${action}" in + config|rm) exit 0 ;; + services) + stack_name=${2:-} + [[ -d /tmp/mock-docker-state/service ]] || exit 1 + for state_file in /tmp/mock-docker-state/service/"${stack_name}"_*; do + [[ -f "${state_file}" ]] || continue + basename "${state_file}" + done + ;; + *) exit 1 ;; + esac + ;; + service) + action=${1:-} + shift || true + state_root=/tmp/mock-docker-state/service + case "${action}" in + inspect) + service_name=${1:-}; shift || true + state_file="${state_root}/${service_name}" + [[ -f "${state_file}" ]] || exit 1 + format= + while (($#)); do if [[ "$1" == --format ]]; then format=$2; shift 2; else shift; fi; done + current=$(sed -n 's/^current=//p' "${state_file}") + previous=$(sed -n 's/^previous=//p' "${state_file}") + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + case "${format}" in + '') printf '[{"Spec":{"Name":"%s","Marker":"%s"}}]\n' "${service_name}" "${current}" ;; + *'com.docker.stack.namespace'*) printf '%s\n' "${namespace}" ;; + *'json .Spec'*) printf '{"Name":"%s","Marker":"%s"}\n' "${service_name}" "${current}" ;; + *'.PreviousSpec'*) if [[ -n "${previous}" ]]; then printf '%s\n' present; else printf '%s\n' absent; fi ;; + *) echo "Unhandled mock service format: ${format}" >&2; exit 1 ;; + esac + ;; + rollback) + while [[ "${1:-}" == --* ]]; do + if [[ "$1" == --detach=false ]]; then shift; else shift 2; fi + done + service_name=${1:-}; state_file="${state_root}/${service_name}" + [[ -f "${state_file}" ]] || exit 1 + current=$(sed -n 's/^current=//p' "${state_file}") + previous=$(sed -n 's/^previous=//p' "${state_file}") + [[ -n "${previous}" ]] || exit 1 + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + printf 'current=%s\nprevious=%s\nnamespace=%s\n' "${previous}" "${current}" "${namespace}" > "${state_file}" + ;; + rm) rm -f -- "${state_root}/${1:-}" ;; + *) exit 1 ;; + esac + ;; + run) + if printf '%s\n' "$@" | grep -Fxq makepad-postgres-brio-tmp-cleaner; then + cleaner_state=/tmp/mock-cleaner-state + install -d -m 0700 "${cleaner_state}" + declare -a cleaner_args=("$@") + cleaner_image= + cleaner_command= + for ((cleaner_index=0; cleaner_index<${#cleaner_args[@]}; cleaner_index++)); do + if [[ "${cleaner_args[cleaner_index]}" == sh && "${cleaner_args[cleaner_index+1]:-}" == -euc ]]; then + cleaner_image=${cleaner_args[cleaner_index-1]} + cleaner_command=${cleaner_args[cleaner_index+2]:-} + break + fi + done + [[ -n "${cleaner_image}" && -n "${cleaner_command}" ]] || exit 1 + printf '%s\n' "${cleaner_image}" > "${cleaner_state}/image" + printf '%s\n' true > "${cleaner_state}/running" + printf '%s\n' "${cleaner_command}" > "${cleaner_state}/command" + exit 0 + fi + declare -a environment=() + declare -a mock_mount_sources=() + declare -a mock_mount_destinations=() + while (($#)); do + case "$1" in + --rm|-i|--read-only) shift ;; + --mount) map_mount "$2"; shift 2 ;; + -v|--volume) + mount_value=$2 + source=${mount_value%%:*} + remainder=${mount_value#*:} + destination=${remainder%%:*} + map_mount "src=${source},dst=${destination}" + shift 2 + ;; + -e|--env) environment+=("$2"); shift 2 ;; + --network|--entrypoint|--tmpfs|--pids-limit|--memory|--cap-drop|--cap-add|--security-opt|--user) shift 2 ;; + --*) shift ;; + *) image=$1; shift; break ;; + esac + done + : "${image:?missing mock image}" + for assignment in "${environment[@]}"; do export "${assignment}"; done + if [[ "${1:-}" == /usr/local/bin/brio-db-transaction.sh ]]; then + operation=${2:-} + scope=${3:-} + destination=${4:-} + case "${scope}" in brio|keycloak) ;; *) exit 1 ;; esac + case "${operation}" in + prepare) + mkdir -p "${destination}" + if [[ -f /tmp/mock-db-state ]]; then cp /tmp/mock-db-state "${destination}/mock-state"; else printf '%s\n' prior > "${destination}/mock-state"; fi + printf '%s\n' '-- mock exact restore program' > "${destination}/restore.sql" + printf '%s\n' mock-prestate-fingerprint > "${destination}/prestate.fingerprint" + ;; + restore) + [[ -s "${destination}/mock-state" && -s "${destination}/restore.sql" && -s "${destination}/prestate.fingerprint" ]] + cp "${destination}/mock-state" /tmp/mock-db-state + ;; + *) exit 1 ;; + esac + exit 0 + fi + if [[ "${1:-}" == /usr/local/bin/run-brio-encrypted-backup.sh ]]; then + create_backup "${BRIO_BACKUP_DATABASE:?}" + exit 0 + fi + if [[ "${1:-}" == getent && "${2:-}" == hosts ]]; then + printf '%s %s\n' 10.44.0.2 "${3:-makepad-postgres-brio-staging}" + exit 0 + fi + if [[ "${1:-}" == sh && "${2:-}" == -euc && "${3:-}" == *'exec psql'* ]]; then + script=${3} + if [[ "${script}" == *'-f /tmp/bootstrap.sql'* ]]; then + printf '%s\n' candidate > /tmp/mock-db-state + exit 0 + fi + role=${5:-}; database=${6:-}; sslmode=${7:-}; query=${8:-} + [[ "${sslmode}" != disable ]] || exit 1 + [[ "${database}" != postgres ]] || exit 1 + if [[ "${query}" == *current_database* ]]; then printf '%s:%s\n' "${database}" "${role}" + elif [[ "${query}" == *default_transaction_read_only* ]]; then printf '%s\n' on + else printf '%s\n' 1; fi + exit 0 + fi + if [[ "${1:-}" == sh && "${2:-}" == -euc ]]; then + script=${3} + for ((mount_index=0; mount_index<${#mock_mount_sources[@]}; mount_index++)); do + mount_source=${mock_mount_sources[mount_index]} + mount_destination=${mock_mount_destinations[mount_index]} + case "${mount_destination}" in + /journal|/rollback|/managed-var-lib|/host/etc|/host/var/lib) + script=${script//${mount_destination}/${mount_source}} + ;; + esac + done + script=${script//\/host\//\/} + script=${script//\/host/\/} + shift 3 + env "${environment[@]}" sh -euc "${script}" "$@" + command_status=$? + if [[ ${command_status} -eq 0 && "${script}" == *'tar --numeric-owner -xpf'* ]]; then + for ((mount_index=0; mount_index<${#mock_mount_sources[@]}; mount_index++)); do + mount_source=${mock_mount_sources[mount_index]} + mount_destination=${mock_mount_destinations[mount_index]} + case "${mount_destination}" in + /managed/live|/managed/etc) + find "${mount_source}" -mindepth 1 -delete + cp -a "${mount_destination}/." "${mount_source}/" + ;; + esac + done + fi + exit "${command_status}" + fi + env "${environment[@]}" "$@" + ;; + *) + echo "Unhandled mock docker command: ${command_name} $*" >&2 + exit 1 + ;; +esac +MOCK +chmod 0755 "${mock_bin}/docker" +export PATH="${mock_bin}:${PATH}" + +generate_certificates() { + local destination=$1 + install -d -m 0700 "${destination}" + openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=Brio-Test-CA \ + -keyout "${destination}/ca.key" -out "${destination}/ca.crt" >/dev/null 2>&1 + openssl req -newkey rsa:2048 -nodes -subj /CN=makepad-postgres-brio-staging \ + -keyout "${destination}/server.key" -out "${destination}/server.csr" >/dev/null 2>&1 + cat > "${destination}/server.ext" <<'EOF' +subjectAltName=DNS:makepad-postgres-brio-staging,IP:65.21.134.125 +extendedKeyUsage=serverAuth +EOF + openssl x509 -req -days 30 -in "${destination}/server.csr" \ + -CA "${destination}/ca.crt" -CAkey "${destination}/ca.key" -CAcreateserial \ + -extfile "${destination}/server.ext" -out "${destination}/server.crt" >/dev/null 2>&1 + openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=Brio-Backup-Test \ + -keyout "${destination}/recipient.key" -out "${destination}/recipient.crt" >/dev/null 2>&1 +} + +pki=/tmp/brio-fixture-pki +generate_certificates "${pki}" +export MOCK_POSTGRES_IMAGE='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +validation_image='postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825' + +remove_exact() { + local target=$1 + if [[ -L "${target}" || -f "${target}" ]]; then + rm -f -- "${target}" + elif [[ -d "${target}" ]]; then + find "${target}" -mindepth 1 -delete + rmdir -- "${target}" + elif [[ -e "${target}" ]]; then + echo "Refusing unexpected fixture path type: ${target}" >&2 + exit 1 + fi +} + +reset_canary_host() { + for target in /etc/makepad /var/lib/makepad /host /runtime /rollback /managed /tmp/mock-docker-state /tmp/mock-db-state; do remove_exact "${target}"; done + install -d -m 0755 /etc/makepad/secrets /etc/makepad/tls/postgres /etc/makepad/tls/backups /var/lib/makepad/postgres-backups + printf '%s\n' old-canary-superuser > /etc/makepad/secrets/postgres-canary-superuser-password + printf '%s\n' old-canary-backup > /etc/makepad/secrets/postgres-brio-app-backup-password + cp "${pki}/ca.crt" /etc/makepad/tls/postgres/ca.crt + cp "${pki}/recipient.crt" /etc/makepad/tls/backups/brio-recipient.crt + install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/brio-staging + printf '%s\n' preserve-me > /var/lib/makepad/postgres-backups/brio-staging/sentinel + chmod 0600 /etc/makepad/secrets/postgres-canary-superuser-password + chown 999:999 /etc/makepad/secrets/postgres-brio-app-backup-password + chmod 0400 /etc/makepad/secrets/postgres-brio-app-backup-password + chmod 0444 /etc/makepad/tls/postgres/ca.crt /etc/makepad/tls/backups/brio-recipient.crt + install -d -m 0700 /tmp/mock-docker-state/service + docker network create --driver overlay --attachable --opt encrypted=true makepad_canary_primary_db >/dev/null + docker network create --driver overlay --attachable --opt encrypted=true makepad_canary_lpc_db >/dev/null + printf 'current=prior-postgres\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_postgres + printf 'current=prior-backup\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_brio_staging_backup + printf '%s\n' prior > /tmp/mock-db-state +} + +prepare_canary() { + local id=$1 + canary_bundle="/srv/test/.deploy/postgres-${id}" + canary_runtime="/tmp/postgres-brio-canary-runtime-${id}" + remove_exact "${canary_bundle}" + remove_exact "${canary_runtime}" + install -d -m 0700 "${canary_bundle}/envs/canary" "${canary_bundle}/config" "${canary_bundle}/scripts" "${canary_bundle}/bootstrap" "${canary_runtime}" + cat > "${canary_bundle}/envs/canary/.env.db" < "${canary_bundle}/envs/canary/.env.deploy" <<'EOF' +MAKEPAD_POSTGRES_DB_NETWORK=makepad_canary_primary_db +MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=makepad_canary_lpc_db +MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK=makepad_brio_staging_db +EOF + printf '%s\n' 'services: {}' > "${canary_bundle}/compose.yml" + printf '%s\n' 'services: {}' > "${canary_bundle}/envs/canary/compose.yml" + printf '%s\n' 'host all all all scram-sha-256' > "${canary_bundle}/config/runtrace-pg_hba.conf" + printf '%s\n' '# test' > "${canary_bundle}/bootstrap/brio-staging-app.sql" + cat > "${canary_bundle}/scripts/deploy-postgres-stack.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "${MOCK_STACK_RESULT:-success}" == success ]] +for state_file in /tmp/mock-docker-state/service/brio-canary_*; do + [[ -f "${state_file}" ]] || continue + current=$(sed -n 's/^current=//p' "${state_file}") + namespace=$(sed -n 's/^namespace=//p' "${state_file}") + printf 'current=candidate-%s\nprevious=%s\nnamespace=%s\n' "${state_file##*/}" "${current}" "${namespace}" > "${state_file}" +done +EOF + chmod 0755 "${canary_bundle}/scripts/deploy-postgres-stack.sh" + cp "${repo}/scripts/brio-db-transaction.sh" "${canary_bundle}/scripts/brio-db-transaction.sh" + chmod 0755 "${canary_bundle}/scripts/brio-db-transaction.sh" + printf '%s\n' new-superuser > "${canary_runtime}/postgres-superuser-password" + printf '%s\n' new-app-password > "${canary_runtime}/brio-staging-app-password" + printf '%s\n' new-backup-password > "${canary_runtime}/brio-staging-backup-password" + cp "${pki}/ca.crt" "${canary_runtime}/postgres-ca.pem" + cp "${pki}/server.crt" "${canary_runtime}/postgres-server-cert.pem" + cp "${pki}/server.key" "${canary_runtime}/postgres-server-key.pem" + cp "${pki}/recipient.crt" "${canary_runtime}/brio-backup-recipient-cert.pem" + chmod 0600 "${canary_runtime}"/* +} + +assert_canary_restored() { + [[ $(< /etc/makepad/secrets/postgres-canary-superuser-password) == old-canary-superuser ]] + [[ $(< /etc/makepad/secrets/postgres-brio-app-backup-password) == old-canary-backup ]] + [[ $(< /var/lib/makepad/postgres-backups/brio-staging/sentinel) == preserve-me ]] + [[ $(sed -n 's/^current=//p' /tmp/mock-docker-state/service/brio-canary_postgres) == prior-postgres ]] + [[ $(sed -n 's/^current=//p' /tmp/mock-docker-state/service/brio-canary_brio_staging_backup) == prior-backup ]] + [[ $(< /tmp/mock-db-state) == prior ]] + [[ -f /tmp/mock-docker-state/network/makepad_canary_primary_db ]] + [[ -f /tmp/mock-docker-state/network/makepad_canary_lpc_db ]] + [[ ! -e /tmp/mock-docker-state/network/makepad_brio_staging_db ]] + ! find /tmp/mock-docker-state/config /tmp/mock-docker-state/secret -type f -print -quit 2>/dev/null | grep -q . +} + +run_canary_failure() { + local id=$1 injection=$2 stack_result=${3:-success} + reset_canary_host + prepare_canary "${id}" + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${injection}" MOCK_STACK_RESULT="${stack_result}" \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] +} + +run_canary_failure 101-1 after-managed-file-promotion +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-101-1 ]] + +run_canary_failure 102-1 term-after-managed-file-promotion +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-102-1 ]] + +run_canary_failure 103-1 after-stack-deploy success +assert_canary_restored +[[ ! -e /tmp/postgres-brio-canary-runtime-103-1 ]] + +canary_injections=(after-bootstrap after-app-probe after-plaintext-probe after-nontarget-probe after-backup-role-probe after-backup-verification) +canary_case=110 +for injection in "${canary_injections[@]}"; do + run_canary_failure "${canary_case}-1" "${injection}" + assert_canary_restored + [[ ! -e "/var/lib/makepad/postgres-recovery/brio-canary/${canary_case}-1" ]] + canary_case=$((canary_case + 1)) +done + +# A SIGKILL cannot execute traps. The next attempt must recover the durable +# transaction before taking its own snapshot, at both host and DB boundaries. +for kill_injection in kill-after-managed-file-promotion kill-after-bootstrap; do + reset_canary_host + prepare_canary 120-1 + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${kill_injection}" \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-kill-output 2>&1 + kill_status=$? + set -e + [[ ${kill_status} -eq 137 ]] + [[ -f /var/lib/makepad/postgres-recovery/brio-canary/120-1/IN_PROGRESS ]] + prepare_canary 120-2 + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-recovery-output 2>&1 + recovery_status=$? + set -e + [[ ${recovery_status} -ne 0 ]] + assert_canary_restored + [[ ! -e /var/lib/makepad/postgres-recovery/brio-canary/120-1 ]] + remove_exact /tmp/postgres-brio-canary-runtime-120-1 +done + +# stack deploy has no implicit --prune. A stale legacy identity backup service +# is inventoried and blocks the release without mutating its exact prior Spec. +reset_canary_host +printf 'current=legacy-exact\nprevious=\nnamespace=brio-canary\n' > /tmp/mock-docker-state/service/brio-canary_keycloak_brio_staging_backup +legacy_before=$(docker service inspect brio-canary_keycloak_brio_staging_backup --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) +prepare_canary 121-1 +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-legacy-output 2>&1 +legacy_status=$? +set -e +[[ ${legacy_status} -ne 0 ]] +grep -q 'Retire it through a separately reviewed operation' /tmp/canary-legacy-output +legacy_after=$(docker service inspect brio-canary_keycloak_brio_staging_backup --format '{{json .Spec}}' | sha256sum | cut -d' ' -f1) +[[ "${legacy_before}" == "${legacy_after}" && $(< /tmp/mock-db-state) == prior ]] + +reset_canary_host +prepare_canary 103-2 +remove_exact /var/lib/makepad/postgres-backups/brio-staging +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-absent-output 2>&1 +status=$? +set -e +[[ ${status} -ne 0 ]] +[[ ! -e /var/lib/makepad/postgres-backups/brio-staging ]] + +run_canary_failure 104-1 rollback-restore fail +[[ -f /tmp/postgres-brio-canary-runtime-104-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-canary/104-1/rollback/managed.tar ]] +[[ $(stat -c '%u:%a' /var/lib/makepad/postgres-recovery/brio-canary/104-1) == 0:700 ]] +for secret in postgres-superuser-password brio-staging-app-password brio-staging-backup-password postgres-server-key.pem; do + [[ ! -e "/tmp/postgres-brio-canary-runtime-104-1/${secret}" ]] +done + +# Check every canary-controlled parent before the journal snapshot. No bind +# mount may hide a symlink in a mutable host path or touch its outside target. +canary_outside=/tmp/postgres-brio-canary-outside-sentinel +remove_exact "${canary_outside}" +install -d -m 0700 "${canary_outside}" +printf '%s\n' untouched > "${canary_outside}/sentinel" +for parent in \ + /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres /etc/makepad/tls/backups \ + /var/lib/makepad /var/lib/makepad/postgres-backups /var/lib/makepad/postgres-recovery; do + reset_canary_host + prepare_canary 105-1 + remove_exact "${parent}" + ln -s "${canary_outside}" "${parent}" + set +e + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-canary-postgres.sh" "${canary_bundle}" brio-canary "${canary_runtime}" >/tmp/canary-symlink-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] + grep -Eq 'symlink component|missing, not a directory, or a symlink' /tmp/canary-symlink-output + [[ $(< "${canary_outside}/sentinel") == untouched ]] + remove_exact "${parent}" +done +remove_exact "${canary_outside}" + +reset_identity_host() { + for target in /srv/makepad/postgres /etc/makepad /var/lib/makepad /host /runtime /rollback /managed /managed-var-lib /tmp/mock-compose-up-count /tmp/mock-db-state; do remove_exact "${target}"; done + install -d -m 0755 /srv/makepad/postgres/{bootstrap,config,envs/production,scripts} /etc/makepad/secrets /etc/makepad/tls/postgres /etc/makepad/tls/backups /var/lib/makepad/postgres /var/lib/makepad/postgres-backups + printf '%s\n' old-live-compose > /srv/makepad/postgres/compose.host.yml + printf '%s\n' old-live-env > /srv/makepad/postgres/envs/production/.env.db + printf '%s\n' old-live-hba > /srv/makepad/postgres/config/runtrace-pg_hba.conf + for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh; do printf '%s\n' old > "/srv/makepad/postgres/scripts/${script}"; done + printf '%s\n' old-superuser > /etc/makepad/secrets/postgres-superuser-password + cp "${pki}/server.crt" /etc/makepad/tls/postgres/server.crt + cp "${pki}/server.key" /etc/makepad/secrets/postgres-server.key + cp "${pki}/ca.crt" /etc/makepad/tls/postgres/ca.crt + chmod 0600 /etc/makepad/secrets/postgres-superuser-password + chown 70:70 /etc/makepad/secrets/postgres-server.key + chmod 0400 /etc/makepad/secrets/postgres-server.key + chmod 0444 /etc/makepad/tls/postgres/server.crt /etc/makepad/tls/postgres/ca.crt + install -d -o 999 -g 999 -m 0700 \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z + printf '%s\n' preserved-encrypted-backup > \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z/sentinel + ln -s 20200101T000000Z /var/lib/makepad/postgres-backups/keycloak-brio-staging/latest + printf '%s\n' '{"database":"keycloak_brio_staging","encrypted":true,"backup":"20200101T000000Z"}' > \ + /var/lib/makepad/postgres-backups/keycloak-brio-staging/last-success.json + chown -R 999:999 /var/lib/makepad/postgres-backups/keycloak-brio-staging + printf '%s\n' prior > /tmp/mock-db-state +} + +prepare_identity() { + local id=$1 + identity_bundle="/tmp/postgres-brio-identity-bundle-${id}" + identity_runtime="/tmp/postgres-brio-identity-runtime-${id}" + remove_exact "${identity_bundle}" + remove_exact "${identity_runtime}" + install -d -m 0700 "${identity_bundle}/envs/production" "${identity_bundle}/config" "${identity_bundle}/bootstrap" "${identity_bundle}/scripts" "${identity_runtime}" + cat > "${identity_bundle}/envs/production/.env.db" < "${identity_bundle}/compose.host.yml" <<'EOF' +services: + postgres: + network_mode: host + volumes: + - "${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data" +EOF + cat > "${identity_bundle}/config/runtrace-pg_hba.conf" <<'EOF' +hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 +hostssl keycloak_brio_staging keycloak_brio_staging_backup 127.0.0.1/32 scram-sha-256 +host all keycloak_brio_staging_app all reject +host all keycloak_brio_staging_backup all reject +EOF + printf '%s\n' '# bootstrap' > "${identity_bundle}/bootstrap/keycloak-brio-staging.sql" + for script in run-runtrace-backup.sh run-runtrace-backup-loop.sh run-brio-encrypted-backup.sh run-brio-encrypted-backup-loop.sh; do + printf '%s\n' '#!/bin/sh' 'exit 0' > "${identity_bundle}/scripts/${script}" + chmod 0755 "${identity_bundle}/scripts/${script}" + done + cp "${repo}/scripts/brio-db-transaction.sh" "${identity_bundle}/scripts/brio-db-transaction.sh" + chmod 0755 "${identity_bundle}/scripts/brio-db-transaction.sh" + printf '%s\n' identity-app-new > "${identity_runtime}/keycloak-brio-staging-app-password" + printf '%s\n' identity-backup-new > "${identity_runtime}/keycloak-brio-staging-backup-password" + cp "${pki}/recipient.crt" "${identity_runtime}/brio-backup-recipient-cert.pem" + chmod 0600 "${identity_runtime}"/* +} + +assert_identity_restored() { + [[ $(< /srv/makepad/postgres/compose.host.yml) == old-live-compose ]] + [[ $(< /srv/makepad/postgres/config/runtrace-pg_hba.conf) == old-live-hba ]] + [[ ! -e /etc/makepad/secrets/postgres-brio-identity-backup-password ]] + [[ $(< /tmp/mock-db-state) == prior ]] + [[ $(readlink /var/lib/makepad/postgres-backups/keycloak-brio-staging/latest) == 20200101T000000Z ]] + [[ $(< /var/lib/makepad/postgres-backups/keycloak-brio-staging/20200101T000000Z/sentinel) == preserved-encrypted-backup ]] + [[ $(< /var/lib/makepad/postgres-backups/keycloak-brio-staging/last-success.json) == '{"database":"keycloak_brio_staging","encrypted":true,"backup":"20200101T000000Z"}' ]] + [[ $(find /var/lib/makepad/postgres-backups/keycloak-brio-staging -mindepth 1 -maxdepth 1 -printf '%f\n' | sort | tr '\n' ' ') == '20200101T000000Z last-success.json latest ' ]] +} + +run_identity_failure() { + local id=$1 injection=$2 fail_first=${3:-0} + reset_identity_host + prepare_identity "${id}" + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + BRIO_DEPLOY_TEST_MODE=isolated-container BRIO_DEPLOY_FAILURE_INJECTION="${injection}" \ + MOCK_COMPOSE_FAIL_FIRST="${fail_first}" \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-output 2>&1 + status=$? + set -e + [[ ${status} -ne 0 ]] +} + +run_identity_failure 201-1 after-managed-file-promotion +assert_identity_restored +[[ ! -e /tmp/postgres-brio-identity-runtime-201-1 ]] + +run_identity_failure 202-1 term-after-managed-file-promotion +assert_identity_restored +[[ ! -e /tmp/postgres-brio-identity-runtime-202-1 ]] + +identity_injections=(after-bootstrap after-app-probe after-plaintext-probe after-nontarget-probe after-backup-role-probe after-backup-verification) +identity_case=210 +for injection in "${identity_injections[@]}"; do + run_identity_failure "${identity_case}-1" "${injection}" + assert_identity_restored + [[ ! -e "/var/lib/makepad/postgres-recovery/brio-identity/${identity_case}-1" ]] + identity_case=$((identity_case + 1)) +done + +for kill_injection in kill-after-managed-file-promotion kill-after-bootstrap; do + reset_identity_host + prepare_identity 220-1 + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes BRIO_DEPLOY_TEST_MODE=isolated-container \ + BRIO_DEPLOY_FAILURE_INJECTION="${kill_injection}" \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-kill-output 2>&1 + kill_status=$? + set -e + [[ ${kill_status} -eq 137 ]] + [[ -f /var/lib/makepad/postgres-recovery/brio-identity/220-1/IN_PROGRESS ]] + prepare_identity 220-2 + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes BRIO_DEPLOY_TEST_MODE=isolated-container \ + BRIO_DEPLOY_FAILURE_INJECTION=after-managed-file-promotion \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-recovery-output 2>&1 + recovery_status=$? + set -e + [[ ${recovery_status} -ne 0 ]] + assert_identity_restored + [[ ! -e /var/lib/makepad/postgres-recovery/brio-identity/220-1 ]] + remove_exact /tmp/postgres-brio-identity-runtime-220-1 +done + +run_identity_failure 203-1 rollback-restore 1 +[[ -f /tmp/postgres-brio-identity-runtime-203-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/203-1/rollback/managed.tar ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/203-1/RECOVERY_REQUIRED ]] +[[ $(stat -c '%u:%a' /var/lib/makepad/postgres-recovery/brio-identity/203-1) == 0:700 ]] +for secret in keycloak-brio-staging-app-password keycloak-brio-staging-backup-password brio-backup-recipient-cert.pem; do + [[ ! -e "/tmp/postgres-brio-identity-runtime-203-1/${secret}" ]] +done + +run_identity_failure 204-1 rollback-recreate 1 +assert_identity_restored +[[ -f /tmp/postgres-brio-identity-runtime-204-1/RECOVERY_REQUIRED ]] +[[ -f /var/lib/makepad/postgres-recovery/brio-identity/204-1/RECOVERY_REQUIRED ]] + +reset_identity_host +prepare_identity 205-1 +set +e +BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ +BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes MOCK_COMPOSE_PROJECT=unexpected-project \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-target-output 2>&1 +status=$? +set -e +[[ ${status} -ne 0 ]] +grep -q 'exact Compose label' /tmp/identity-target-output +assert_identity_restored + +# Every repository-controlled mutable parent is checked component-by-component +# both before snapshot and within promotion. Symlink targets remain untouched. +outside=/tmp/postgres-brio-outside-sentinel +remove_exact "${outside}" +install -d -m 0700 "${outside}" +printf '%s\n' untouched > "${outside}/sentinel" +for parent in \ + /srv/makepad /srv/makepad/postgres \ + /etc/makepad /etc/makepad/secrets /etc/makepad/tls /etc/makepad/tls/postgres /etc/makepad/tls/backups \ + /var/lib/makepad /var/lib/makepad/postgres-backups; do + reset_identity_host + prepare_identity 230-1 + remove_exact "${parent}" + ln -s "${outside}" "${parent}" + set +e + BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${repo}/scripts/deploy-brio-identity-db-host.sh" "${identity_bundle}" "${identity_runtime}" 65.21.134.125 88.99.209.165/32 \ + >/tmp/identity-parent-symlink-output 2>&1 + symlink_status=$? + set -e + [[ ${symlink_status} -ne 0 ]] + grep -Eq 'symlink|missing|unavailable|unsafe' /tmp/identity-parent-symlink-output + [[ $(< "${outside}/sentinel") == untouched ]] + remove_exact "${parent}" +done +remove_exact "${outside}" + +cleaner_root=/tmp/postgres-brio-cleaner-test-failures +remove_exact "${cleaner_root}" +install -d "${cleaner_root}/postgres-brio-delete" "${cleaner_root}/postgres-brio-preserve" +printf '%s\n' recovery > "${cleaner_root}/postgres-brio-preserve/RECOVERY_REQUIRED" +touch -t 202001010000 "${cleaner_root}/postgres-brio-delete" "${cleaner_root}/postgres-brio-preserve" +"${repo}/scripts/ensure-brio-tmp-cleaner.sh" test-clean-once "${cleaner_root}" +[[ ! -e "${cleaner_root}/postgres-brio-delete" ]] +[[ -f "${cleaner_root}/postgres-brio-preserve/RECOVERY_REQUIRED" ]] + +# Existing host cleaners are trusted only when their entire command contract +# matches, and a stopped instance must remain running after an attempted start. +cleaner_env=/tmp/postgres-brio-cleaner-test-failures.env +printf 'POSTGRES_IMAGE=%s\n' "${MOCK_POSTGRES_IMAGE}" > "${cleaner_env}" +remove_exact /tmp/mock-cleaner-state +remove_exact /tmp/mock-cleaner-stop-after-start +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" +[[ $(< /tmp/mock-cleaner-state/running) == true ]] +printf '%s\n' 'unexpected cleaner command' > /tmp/mock-cleaner-state/command +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" >/tmp/cleaner-command-output 2>&1 +cleaner_command_status=$? +set -e +[[ ${cleaner_command_status} -ne 0 ]] +grep -q 'does not match the fail-closed cleanup contract' /tmp/cleaner-command-output +remove_exact /tmp/mock-cleaner-state +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" +printf '%s\n' false > /tmp/mock-cleaner-state/running +touch /tmp/mock-cleaner-stop-after-start +set +e +BRIO_DEPLOY_TEST_MODE=isolated-container "${repo}/scripts/ensure-brio-tmp-cleaner.sh" "${cleaner_env}" >/tmp/cleaner-running-output 2>&1 +cleaner_running_status=$? +set -e +[[ ${cleaner_running_status} -ne 0 ]] +grep -q 'did not remain running after startup' /tmp/cleaner-running-output +remove_exact /tmp/mock-cleaner-stop-after-start + +echo "Brio deployment failure-injection tests passed." diff --git a/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh b/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh new file mode 100755 index 0000000..18f06ec --- /dev/null +++ b/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/keycloak-cohort-cleaner-mock-bin +state=/tmp/keycloak-cohort-cleaner-mock-state +install -d -m 0700 "${mock_bin}" "${state}" + +cat >"${mock_bin}/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +state=/tmp/keycloak-cohort-cleaner-mock-state +kind=${1:-} +action=${2:-} +shift 2 || true +case "${kind}:${action}" in + container:ls) + [[ " $* " == *' -aq '* && " $* " == *' label=makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 '* ]] + cat "${state}/containers" + ;; + container:inspect) + identifier=${1:-} + case "${identifier}" in + c-expired) printf '/pg-kc-db-101-1-catwlk|makepad-keycloak-cohort-restore-v1|1\n' ;; + c-future) printf '/pg-kc-app-101-1-catwlk|makepad-keycloak-cohort-restore-v1|4102444800\n' ;; + c-malformed) printf '/unrelated-container|makepad-keycloak-cohort-restore-v1|1\n' ;; + *) exit 1 ;; + esac + ;; + network:ls) + [[ " $* " == *' -q '* && " $* " == *' label=makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 '* ]] + cat "${state}/networks" + ;; + network:inspect) + identifier=${1:-} + case "${identifier}" in + n-expired) printf 'pg-kc-101-1-catwlk|makepad-keycloak-cohort-restore-v1|1\n' ;; + n-future) printf 'pg-kc-102-1-catwlk|makepad-keycloak-cohort-restore-v1|4102444800\n' ;; + *) exit 1 ;; + esac + ;; + rm:-f) + printf '%s\n' "${1:-}" >>"${state}/removed-containers" + ;; + network:rm) + printf '%s\n' "${1:-}" >>"${state}/removed-networks" + ;; + *) + echo "Unhandled mocked Docker operation: ${kind} ${action} $*" >&2 + exit 1 + ;; +esac +MOCK +chmod 0755 "${mock_bin}/docker" +export PATH="${mock_bin}:${PATH}" + +printf '%s\n' c-expired c-future >"${state}/containers" +printf '%s\n' n-expired n-future >"${state}/networks" +install -d -m 0700 /tmp/postgres-keycloak-cohort-expired /tmp/postgres-keycloak-cohort-future \ + /tmp/postgres-keycloak-cohort-recovery +printf '%s\n' preserve > /tmp/postgres-keycloak-cohort-recovery/RECOVERY_REQUIRED +touch -t 202001010000 /tmp/postgres-keycloak-cohort-expired /tmp/postgres-keycloak-cohort-recovery + +"${repo}/scripts/clean-keycloak-cohort-resources.sh" +[[ $(<"${state}/removed-containers") == c-expired ]] +[[ $(<"${state}/removed-networks") == n-expired ]] +[[ ! -e /tmp/postgres-keycloak-cohort-expired ]] +[[ -d /tmp/postgres-keycloak-cohort-future ]] +[[ -f /tmp/postgres-keycloak-cohort-recovery/RECOVERY_REQUIRED ]] + +printf '%s\n' c-malformed >"${state}/containers" +: >"${state}/networks" +: >"${state}/removed-containers" +if "${repo}/scripts/clean-keycloak-cohort-resources.sh" >/tmp/cohort-cleaner-malformed-output 2>&1; then + echo "Cohort cleaner accepted a malformed resource carrying its label." >&2 + exit 1 +fi +grep -q 'Refusing malformed labeled cohort container' /tmp/cohort-cleaner-malformed-output +[[ ! -s "${state}/removed-containers" ]] + +echo "Keycloak cohort resource cleaner tests passed." diff --git a/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh b/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh new file mode 100755 index 0000000..f8d6d52 --- /dev/null +++ b/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/repo +mock_bin=/tmp/keycloak-cohort-dispatch-mock-bin +install -d -m 0755 /usr/local/libexec/makepad +install -d -m 0700 "${mock_bin}" +install -o root -g root -m 0755 "${repo}/scripts/capture-keycloak-cohort-backups.sh" \ + /usr/local/libexec/makepad/capture-keycloak-cohort-backups +install -o root -g root -m 0755 "${repo}/scripts/clean-keycloak-cohort-resources.sh" \ + /usr/local/libexec/makepad/clean-keycloak-cohort-resources +install -o root -g root -m 0755 "${repo}/scripts/keycloak-cohort-capture-dispatch.sh" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch +cat >"${mock_bin}/systemctl" <<'MOCK' +#!/usr/bin/env sh +case "$1:$2" in + is-enabled:--quiet|is-active:--quiet) exit 0 ;; + show:--property=Result) printf '%s\n' success ;; + show:--property=ExecMainStatus) printf '%s\n' 0 ;; + *) exit 1 ;; +esac +MOCK +chmod 0755 "${mock_bin}/systemctl" +export PATH="${mock_bin}:${PATH}" + +helper_digest=$(sha256sum /usr/local/libexec/makepad/capture-keycloak-cohort-backups | cut -d' ' -f1) +cleaner_digest=$(sha256sum /usr/local/libexec/makepad/clean-keycloak-cohort-resources | cut -d' ' -f1) +result=$(SSH_ORIGINAL_COMMAND="probe ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch) +[[ "${result}" == cohort-capture-contract-ok ]] + +wrong_digest=$(printf '0%.0s' {1..64}) +if SSH_ORIGINAL_COMMAND="probe ${helper_digest} ${wrong_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted a substituted cleaner digest." >&2 + exit 1 +fi +if SSH_ORIGINAL_COMMAND="shell ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted an arbitrary operation." >&2 + exit 1 +fi +if SSH_ORIGINAL_COMMAND="fetch 1 1 ../../etc/passwd ${helper_digest} ${cleaner_digest}" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch >/dev/null 2>&1; then + echo "Forced command accepted an unsupported fetch path." >&2 + exit 1 +fi + +echo "Keycloak cohort forced-command tests passed." diff --git a/scripts/install-keycloak-cohort-capture-host.sh b/scripts/install-keycloak-cohort-capture-host.sh new file mode 100755 index 0000000..b269b11 --- /dev/null +++ b/scripts/install-keycloak-cohort-capture-host.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: install-keycloak-cohort-capture-host.sh " >&2 + exit 2 +fi +[[ "$(id -u)" -eq 0 ]] || { echo "The capture host installer must run as root." >&2; exit 1; } +capture_user=$1 +public_key_file=$2 +[[ "${capture_user}" =~ ^[a-z_][a-z0-9_-]{0,30}$ && "${capture_user}" != root ]] || { echo "Capture user is invalid." >&2; exit 2; } +id "${capture_user}" >/dev/null +id -nG "${capture_user}" | tr ' ' '\n' | grep -Fxq docker || { echo "Capture user must already belong to the Docker group." >&2; exit 1; } +[[ -f "${public_key_file}" && ! -L "${public_key_file}" ]] || { echo "Public key input is unsafe." >&2; exit 2; } +read -r key_type key_body _ <"${public_key_file}" +case "${key_type}" in ssh-ed25519|sk-ssh-ed25519@openssh.com) ;; *) echo "Only Ed25519 capture keys are accepted." >&2; exit 2 ;; esac +[[ "${key_body}" =~ ^[A-Za-z0-9+/]+={0,3}$ ]] || { echo "Public key encoding is invalid." >&2; exit 2; } + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +"${script_dir}/install-keycloak-cohort-cleaner.sh" +install -d -o root -g root -m 0755 /usr/local/libexec/makepad /etc/makepad/keycloak-cohort-capture +install -o root -g root -m 0755 "${script_dir}/capture-keycloak-cohort-backups.sh" \ + /usr/local/libexec/makepad/capture-keycloak-cohort-backups +install -o root -g root -m 0755 "${script_dir}/keycloak-cohort-capture-dispatch.sh" \ + /usr/local/libexec/makepad/keycloak-cohort-capture-dispatch +capture_group=$(id -gn "${capture_user}") +printf 'restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch" %s %s\n' \ + "${key_type}" "${key_body}" > /etc/makepad/keycloak-cohort-capture/authorized_keys +chown root:"${capture_group}" /etc/makepad/keycloak-cohort-capture/authorized_keys +chmod 0640 /etc/makepad/keycloak-cohort-capture/authorized_keys +cat > /etc/ssh/sshd_config.d/70-makepad-keycloak-cohort-capture.conf </dev/null || systemctl reload sshd.service +printf 'capture_helper_sha256=%s\n' "$(sha256sum /usr/local/libexec/makepad/capture-keycloak-cohort-backups | cut -d' ' -f1)" +printf 'cohort_cleaner_sha256=%s\n' "$(sha256sum /usr/local/libexec/makepad/clean-keycloak-cohort-resources | cut -d' ' -f1)" diff --git a/scripts/install-keycloak-cohort-cleaner.sh b/scripts/install-keycloak-cohort-cleaner.sh new file mode 100755 index 0000000..53eccb1 --- /dev/null +++ b/scripts/install-keycloak-cohort-cleaner.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +[[ $# -eq 0 ]] || { echo "usage: install-keycloak-cohort-cleaner.sh" >&2; exit 2; } +[[ "$(id -u)" -eq 0 ]] || { echo "The cohort cleaner installer must run as root." >&2; exit 1; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +repo_root=$(cd "${script_dir}/.." && pwd -P) +for source in "${script_dir}/clean-keycloak-cohort-resources.sh" \ + "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.service" \ + "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.timer"; do + [[ -f "${source}" && ! -L "${source}" ]] || { echo "Missing safe installer input: ${source}" >&2; exit 1; } +done +install -d -o root -g root -m 0755 /usr/local/libexec/makepad +install -o root -g root -m 0755 "${script_dir}/clean-keycloak-cohort-resources.sh" \ + /usr/local/libexec/makepad/clean-keycloak-cohort-resources +install -o root -g root -m 0644 "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.service" \ + /etc/systemd/system/makepad-keycloak-cohort-cleaner.service +install -o root -g root -m 0644 "${repo_root}/host/systemd/makepad-keycloak-cohort-cleaner.timer" \ + /etc/systemd/system/makepad-keycloak-cohort-cleaner.timer +systemctl daemon-reload +systemctl enable --now makepad-keycloak-cohort-cleaner.timer >/dev/null +systemctl start makepad-keycloak-cohort-cleaner.service +systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer +systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer diff --git a/scripts/keycloak-cohort-capture-dispatch.sh b/scripts/keycloak-cohort-capture-dispatch.sh new file mode 100755 index 0000000..58b6f73 --- /dev/null +++ b/scripts/keycloak-cohort-capture-dispatch.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C +set -f + +helper=/usr/local/libexec/makepad/capture-keycloak-cohort-backups +cleaner=/usr/local/libexec/makepad/clean-keycloak-cohort-resources +command_text=${SSH_ORIGINAL_COMMAND:-} +read -r -a words <<<"${command_text}" +[[ ${#words[@]} -ge 3 ]] || { echo "A constrained cohort capture command is required." >&2; exit 2; } +operation=${words[0]} +helper_digest=${words[${#words[@]}-2]} +cleaner_digest=${words[${#words[@]}-1]} +[[ "${helper_digest}" =~ ^[a-f0-9]{64}$ && "${cleaner_digest}" =~ ^[a-f0-9]{64}$ ]] || { + echo "The reviewed capture-helper and cleaner digests are required." >&2 + exit 2 +} +[[ -x "${helper}" && ! -L "${helper}" && -x "${cleaner}" && ! -L "${cleaner}" ]] || { echo "The root-owned cohort capture contract is not installed." >&2; exit 1; } +[[ $(sha256sum "${helper}" | cut -d' ' -f1) == "${helper_digest}" ]] || { echo "The installed capture helper differs from the reviewed release." >&2; exit 1; } +[[ $(sha256sum "${cleaner}" | cut -d' ' -f1) == "${cleaner_digest}" ]] || { echo "The installed cohort cleaner differs from the reviewed release." >&2; exit 1; } +systemctl is-enabled --quiet makepad-keycloak-cohort-cleaner.timer +systemctl is-active --quiet makepad-keycloak-cohort-cleaner.timer +[[ $(systemctl show --property=Result --value makepad-keycloak-cohort-cleaner.service) == success ]] +[[ $(systemctl show --property=ExecMainStatus --value makepad-keycloak-cohort-cleaner.service) == 0 ]] + +validate_run() { + [[ "$1" =~ ^[1-9][0-9]*$ && "$2" =~ ^[1-9][0-9]*$ ]] || { echo "Invalid cohort run identity." >&2; exit 2; } + cohort_dir="/tmp/postgres-keycloak-cohort-$1-$2" +} + +case "${operation}" in + probe) + [[ ${#words[@]} -eq 3 ]] + printf 'cohort-capture-contract-ok\n' + ;; + capture) + [[ ${#words[@]} -eq 5 ]] + validate_run "${words[1]}" "${words[2]}" + COHORT_CAPTURE_RUN_ID=${words[1]} COHORT_CAPTURE_RUN_ATTEMPT=${words[2]} \ + "${helper}" "${cohort_dir}" + ;; + fetch) + [[ ${#words[@]} -eq 6 ]] + validate_run "${words[1]}" "${words[2]}" + file=${words[3]} + case "${file}" in + keycloak_catwlk.dump|keycloak_makepad.dump|keycloak_runtrace.dump|keycloak_vestiaire.dump|keycloak_vif.dump) ;; + *) echo "Unsupported cohort artifact." >&2; exit 2 ;; + esac + [[ -f "${cohort_dir}/${file}" && ! -L "${cohort_dir}/${file}" && -s "${cohort_dir}/${file}" ]] || exit 1 + exec cat "${cohort_dir}/${file}" + ;; + cleanup) + [[ ${#words[@]} -eq 5 ]] + validate_run "${words[1]}" "${words[2]}" + if [[ -d "${cohort_dir}" && ! -L "${cohort_dir}" ]]; then + find "${cohort_dir}" -depth -delete + elif [[ -e "${cohort_dir}" || -L "${cohort_dir}" ]]; then + echo "Refusing unsafe cohort cleanup target." >&2 + exit 1 + fi + ;; + *) echo "Unsupported constrained cohort capture operation." >&2; exit 2 ;; +esac diff --git a/scripts/reconcile-github-environment-main-policy.py b/scripts/reconcile-github-environment-main-policy.py new file mode 100755 index 0000000..7f745d4 --- /dev/null +++ b/scripts/reconcile-github-environment-main-policy.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Audit or reconcile exact-main GitHub environment deployment policies.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from typing import Any +from urllib.parse import quote + + +REPOSITORY = "Makepad-fr/postgres" +REQUIRED_ENVIRONMENTS = ( + "canary", + "production", + "staging-brio-identity-db", + "release-brio-identity-db", + "keycloak-cohort-restore", +) +MAX_POLICY_PAGES = 1000 + + +class PolicyError(RuntimeError): + """Raised when provider state cannot be proven safe and complete.""" + + +def _environment_path(environment: str) -> str: + if environment not in REQUIRED_ENVIRONMENTS: + raise PolicyError(f"Unsupported environment: {environment}") + return f"repos/{REPOSITORY}/environments/{quote(environment, safe='')}" + + +class GitHubClient: + """Minimal gh-backed client; authentication remains in gh's credential store.""" + + def request(self, method: str, path: str, payload: dict[str, Any] | None = None) -> Any: + command = [ + "gh", + "api", + "--method", + method, + "-H", + "Accept: application/vnd.github+json", + "-H", + "X-GitHub-Api-Version: 2022-11-28", + path, + ] + if payload is not None: + command.extend(("--input", "-")) + try: + result = subprocess.run( + command, + input=None if payload is None else json.dumps(payload, separators=(",", ":")), + check=True, + text=True, + stdout=subprocess.PIPE, + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PolicyError(f"GitHub API {method} failed for {path}") from error + try: + return json.loads(result.stdout) if result.stdout.strip() else None + except json.JSONDecodeError as error: + raise PolicyError(f"GitHub API returned invalid JSON for {path}") from error + + def get_environment(self, environment: str) -> dict[str, Any]: + response = self.request("GET", _environment_path(environment)) + if not isinstance(response, dict): + raise PolicyError(f"Invalid environment response for {environment}") + return response + + def put_environment(self, environment: str, payload: dict[str, Any]) -> None: + self.request("PUT", _environment_path(environment), payload) + + def list_policies(self, environment: str) -> list[dict[str, Any]]: + path = f"{_environment_path(environment)}/deployment-branch-policies" + policies: list[dict[str, Any]] = [] + seen_ids: set[int] = set() + expected_total: int | None = None + for page in range(1, MAX_POLICY_PAGES + 1): + response = self.request("GET", f"{path}?per_page=100&page={page}") + if not isinstance(response, dict): + raise PolicyError(f"Invalid branch-policy listing for {environment}") + total = response.get("total_count") + page_policies = response.get("branch_policies") + if not isinstance(total, int) or total < 0 or not isinstance(page_policies, list): + raise PolicyError(f"Incomplete branch-policy listing for {environment}") + if expected_total is None: + expected_total = total + elif total != expected_total: + raise PolicyError(f"Branch-policy listing changed during pagination for {environment}") + for policy in page_policies: + if not isinstance(policy, dict) or not isinstance(policy.get("id"), int): + raise PolicyError(f"Invalid branch policy for {environment}") + policy_id = policy["id"] + if policy_id in seen_ids: + raise PolicyError(f"Duplicate branch policy returned for {environment}") + seen_ids.add(policy_id) + policies.append(policy) + if len(policies) == expected_total: + return policies + if len(policies) > expected_total or not page_policies: + raise PolicyError(f"Truncated branch-policy listing for {environment}") + raise PolicyError(f"Branch-policy pagination exceeded its bound for {environment}") + + def create_main_policy(self, environment: str) -> None: + self.request( + "POST", + f"{_environment_path(environment)}/deployment-branch-policies", + {"name": "main", "type": "branch"}, + ) + + def delete_policy(self, environment: str, policy_id: int) -> None: + if not isinstance(policy_id, int) or policy_id <= 0: + raise PolicyError(f"Invalid branch-policy ID for {environment}") + self.request( + "DELETE", + f"{_environment_path(environment)}/deployment-branch-policies/{policy_id}", + ) + + +def has_custom_policy_mode(environment: dict[str, Any]) -> bool: + policy = environment.get("deployment_branch_policy") + return ( + isinstance(policy, dict) + and policy.get("protected_branches") is False + and policy.get("custom_branch_policies") is True + ) + + +def is_exact_main_policy(policies: list[dict[str, Any]]) -> bool: + return len(policies) == 1 and policies[0].get("name") == "main" and policies[0].get("type") == "branch" + + +def build_preserving_update(environment: dict[str, Any]) -> dict[str, Any]: + rules = environment.get("protection_rules") + if not isinstance(rules, list): + raise PolicyError("Environment protection rules are missing") + + wait_timer = 0 + reviewer_entries: list[dict[str, Any]] = [] + prevent_self_review = False + seen_rule_types: set[str] = set() + for rule in rules: + if not isinstance(rule, dict) or not isinstance(rule.get("type"), str): + raise PolicyError("Environment contains an invalid protection rule") + rule_type = rule["type"] + if rule_type in seen_rule_types: + raise PolicyError(f"Environment contains duplicate {rule_type} protection rules") + seen_rule_types.add(rule_type) + if rule_type == "branch_policy": + continue + if rule_type == "wait_timer": + candidate = rule.get("wait_timer") + if not isinstance(candidate, int) or not 0 <= candidate <= 43_200: + raise PolicyError("Environment wait timer is invalid") + wait_timer = candidate + continue + if rule_type != "required_reviewers": + raise PolicyError(f"Refusing to overwrite unsupported protection rule: {rule_type}") + + candidate_prevent = rule.get("prevent_self_review", False) + if not isinstance(candidate_prevent, bool): + raise PolicyError("Environment self-review setting is invalid") + prevent_self_review = candidate_prevent + reviewers = rule.get("reviewers") + if not isinstance(reviewers, list) or not 1 <= len(reviewers) <= 6: + raise PolicyError("Environment required reviewers are invalid") + for entry in reviewers: + reviewer = entry.get("reviewer") if isinstance(entry, dict) else None + reviewer_type = entry.get("type") if isinstance(entry, dict) else None + reviewer_id = reviewer.get("id") if isinstance(reviewer, dict) else None + if reviewer_type not in {"User", "Team"} or not isinstance(reviewer_id, int) or reviewer_id <= 0: + raise PolicyError("Environment required reviewer is invalid") + reviewer_entries.append({"type": reviewer_type, "id": reviewer_id}) + + return { + "wait_timer": wait_timer, + "prevent_self_review": prevent_self_review, + "reviewers": reviewer_entries, + "deployment_branch_policy": { + "protected_branches": False, + "custom_branch_policies": True, + }, + } + + +def audit_environment(client: GitHubClient, environment: str) -> None: + current = client.get_environment(environment) + if not has_custom_policy_mode(current): + raise PolicyError(f"{environment} does not use custom deployment branch policies") + policies = client.list_policies(environment) + if not is_exact_main_policy(policies): + raise PolicyError(f"{environment} is not restricted to the exact branch main") + + +def reconcile_environment(client: GitHubClient, environment: str) -> None: + current = client.get_environment(environment) + if not has_custom_policy_mode(current): + client.put_environment(environment, build_preserving_update(current)) + + policies = client.list_policies(environment) + exact_main = [policy for policy in policies if policy.get("name") == "main" and policy.get("type") == "branch"] + if not exact_main: + client.create_main_policy(environment) + policies = client.list_policies(environment) + exact_main = [policy for policy in policies if policy.get("name") == "main" and policy.get("type") == "branch"] + if not exact_main: + raise PolicyError(f"Unable to create exact main policy for {environment}") + + keep_id = exact_main[0]["id"] + for policy in policies: + if policy["id"] != keep_id: + client.delete_policy(environment, policy["id"]) + audit_environment(client, environment) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("audit", "apply")) + parser.add_argument("--environment", choices=REQUIRED_ENVIRONMENTS, action="append") + parser.add_argument("--confirm", default="") + args = parser.parse_args() + + environments = tuple(args.environment or REQUIRED_ENVIRONMENTS) + if args.mode == "apply": + if len(environments) != 1: + parser.error("apply requires exactly one --environment") + expected_confirmation = f"{REPOSITORY}:{environments[0]}:exact-main" + if args.confirm != expected_confirmation: + parser.error(f"apply requires --confirm {expected_confirmation}") + + client = GitHubClient() + try: + for environment in environments: + if args.mode == "apply": + reconcile_environment(client, environment) + else: + audit_environment(client, environment) + print(f"{environment}: exact custom branch policy main") + except PolicyError as error: + print(f"Environment policy verification failed: {error}", file=__import__("sys").stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restore-keycloak-cohort-backups.sh b/scripts/restore-keycloak-cohort-backups.sh new file mode 100755 index 0000000..5f4e69f --- /dev/null +++ b/scripts/restore-keycloak-cohort-backups.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +if (($# != 4)); then + echo "usage: restore-keycloak-cohort-backups.sh " >&2 + exit 2 +fi + +backup_dir=$1 +result_dir=$2 +keycloak_source=$3 +catwlk_image=$4 +keycloak_image='dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f' +postgres_image='postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' +run_id=${GITHUB_RUN_ID:-1} +run_attempt=${GITHUB_RUN_ATTEMPT:-1} +[[ "${run_id}" =~ ^[1-9][0-9]*$ && "${run_attempt}" =~ ^[1-9][0-9]*$ ]] || { echo "Invalid run identity." >&2; exit 2; } +[[ -d "${backup_dir}" && ! -L "${backup_dir}" && ! -e "${result_dir}" && ! -L "${result_dir}" \ + && -d "${keycloak_source}" && ! -L "${keycloak_source}" ]] || { + echo "Backup/source inputs must be real directories and result output must be new." >&2 + exit 1 +} +[[ "${catwlk_image}" == "makepad/keycloak-catwlk-cohort:"* ]] || { echo "Catwlk runtime tag is outside the reviewed namespace." >&2; exit 2; } +for command_name in docker git python3 sha256sum; do command -v "${command_name}" >/dev/null || { echo "${command_name} is required." >&2; exit 1; }; done + +declare -A databases=( + [catwlk]=keycloak_catwlk + [makepad]=keycloak_makepad + [runtrace]=keycloak_runtrace + [vestiaire]=keycloak_vestiaire + [vif]=keycloak_vif +) +declare -A category_tables=( + [realm]='realm realm_attribute realm_default_groups realm_enabled_event_types realm_events_listeners realm_localizations realm_required_credential realm_smtp_config realm_supported_locales' + [authentication]='authentication_flow authentication_execution authenticator_config authenticator_config_entry' + [roles]='keycloak_role role_attribute composite_role scope_mapping' + [clients]='client client_attributes client_auth_flow_bindings redirect_uris web_origins client_scope client_scope_attributes client_scope_client client_scope_role_mapping default_client_scope protocol_mapper protocol_mapper_config' + [identity_providers]='identity_provider identity_provider_config identity_provider_mapper idp_mapper_config' + [components]='component component_config' + [required_actions]='required_action_provider' +) +expected=$(for slug in "${!databases[@]}"; do printf '%s.dump\n' "${databases[$slug]}"; done | sort) +observed=$(find "${backup_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | sort) +[[ "${observed}" == "${expected}" ]] || { echo "Backup input is not the exact five-database cohort." >&2; exit 1; } +if find "${backup_dir}" -mindepth 1 -maxdepth 1 -type l -print -quit | grep -q .; then echo "Backup input contains a symlink." >&2; exit 1; fi + +release_sha=$(git -C "${keycloak_source}" rev-parse HEAD) +[[ "${release_sha}" =~ ^[a-f0-9]{40}$ ]] || { echo "Checked-out Keycloak release SHA is invalid." >&2; exit 1; } +python3 - "${keycloak_source}/instances/catwlk/manifest.json" <<'PY' +import json, pathlib, sys +value=json.loads(pathlib.Path(sys.argv[1]).read_text())["runtime"] +expected={"source_dir":"providers/catwlk-email-router","dockerfile":"providers/catwlk-email-router/Dockerfile","image":"makepad/keycloak-catwlk"} +if value != expected: raise SystemExit("Catwlk manifest does not select the reviewed custom runtime") +PY +grep -Fxq 'com.makepad.catwlk.keycloak.email.RoutingEmailSenderProviderFactory' \ + "${keycloak_source}/providers/catwlk-email-router/src/main/resources/META-INF/services/org.keycloak.email.EmailSenderProviderFactory" +grep -Fxq 'com.makepad.catwlk.keycloak.apple.CatwlkAppleIdentityProviderFactory' \ + "${keycloak_source}/providers/catwlk-email-router/src/main/resources/META-INF/services/org.keycloak.broker.provider.IdentityProviderFactory" +catwlk_image_id=$(docker image inspect "${catwlk_image}" --format '{{.Id}}') +catwlk_base=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.base-image"}}') +catwlk_source_sha=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.source-sha"}}') +catwlk_runtime=$(docker image inspect "${catwlk_image}" --format '{{index .Config.Labels "org.makepad.keycloak.runtime"}}') +[[ "${catwlk_image_id}" =~ ^sha256:[a-f0-9]{64}$ && "${catwlk_base}" == "${keycloak_image}" \ + && "${catwlk_source_sha}" == "${release_sha}" && "${catwlk_runtime}" == catwlk-custom-provider ]] || { + echo "Catwlk image is not the exact custom-provider runtime from the checked-out release." >&2 + exit 1 +} +docker run --rm --network none --read-only --entrypoint sh "${catwlk_image}" -euc \ + 'test -s /opt/keycloak/providers/catwlk-keycloak-email-router.jar' + +umask 077 +install -d -m 0700 "${result_dir}" "${result_dir}/.runtime" +records="${result_dir}/.instances.jsonl" +: >"${records}" +active_network= +active_database_container= +active_keycloak_container= +cleanup_active() { + local status=$? + trap - EXIT HUP INT TERM + set +e + [[ -z "${active_keycloak_container}" ]] || docker rm -f "${active_keycloak_container}" >/dev/null 2>&1 + [[ -z "${active_database_container}" ]] || docker rm -f "${active_database_container}" >/dev/null 2>&1 + [[ -z "${active_network}" ]] || docker network rm "${active_network}" >/dev/null 2>&1 + if ((status != 0)) && [[ -d "${result_dir}" && ! -L "${result_dir}" ]]; then find "${result_dir}" -depth -delete; fi + exit "${status}" +} +trap cleanup_active EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +db_query() { + local sql=$1 + docker exec "${active_database_container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres-password)" + exec psql -X -v ON_ERROR_STOP=1 -U keycloak -d keycloak -At -c "$1" + ' sh "${sql}" +} + +fingerprint_configuration() { + local destination_name=$1 category table exists digest + local -n destination="${destination_name}" + for category in $(printf '%s\n' "${!category_tables[@]}" | sort); do + for table in ${category_tables[$category]}; do + exists=$(db_query "SELECT CASE WHEN to_regclass('public.${table}') IS NULL THEN 0 ELSE 1 END") + [[ "${exists}" == 1 ]] || { echo "Keycloak configuration fingerprint schema is missing required table ${table}." >&2; return 1; } + done + digest=$( + { + for table in ${category_tables[$category]}; do + printf 'table\t%s\n' "${table}" + db_query "SELECT to_jsonb(value)::text FROM ${table} AS value ORDER BY to_jsonb(value)::text" + done + } | sha256sum | cut -d' ' -f1 + ) + [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || return 1 + # shellcheck disable=SC2034 # destination is an associative nameref output. + destination["${category}"]=${digest} + done +} + +for slug in $(printf '%s\n' "${!databases[@]}" | sort); do + database=${databases[$slug]} + dump="${backup_dir}/${database}.dump" + [[ -s "${dump}" && -f "${dump}" && ! -L "${dump}" ]] || { echo "Invalid dump for ${slug}." >&2; exit 1; } + docker run --rm --read-only --network none --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${dump},dst=/backup.dump,readonly" "${postgres_image}" pg_restore --list /backup.dump >/dev/null + backup_sha=$(sha256sum "${dump}" | cut -d' ' -f1) + suffix="${run_id}-${run_attempt}-${slug}" + active_network="pg-kc-${suffix}" + active_database_container="pg-kc-db-${suffix}" + active_keycloak_container="pg-kc-app-${suffix}" + secret_file="${result_dir}/.runtime/${slug}-postgres-password" + python3 - "${secret_file}" <<'PY' +import pathlib, secrets, sys +pathlib.Path(sys.argv[1]).write_text(secrets.token_urlsafe(48) + "\n") +PY + chmod 0600 "${secret_file}" + expires_epoch=$(( $(date +%s) + 10800 )) + labels=(--label makepad.cleanup.contract=makepad-keycloak-cohort-restore-v1 \ + --label "makepad.cleanup.run=${run_id}-${run_attempt}" --label "makepad.cleanup.expires-epoch=${expires_epoch}") + docker network create --internal "${labels[@]}" "${active_network}" >/dev/null + docker run -d --name "${active_database_container}" --network "${active_network}" --network-alias db \ + "${labels[@]}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=64m --tmpfs /run/postgresql:rw,noexec,nosuid,size=16m \ + --tmpfs /var/lib/postgresql/data:rw,nosuid,size=2g --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password -e POSTGRES_DB=postgres \ + "${postgres_image}" >/dev/null + ready=0 + for _ in $(seq 1 60); do + if docker exec "${active_database_container}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then ready=1; break; fi + [[ $(docker inspect "${active_database_container}" --format '{{.State.Running}}') == true ]] || break + sleep 1 + done + [[ "${ready}" == 1 ]] || { docker logs "${active_database_container}" >&2; echo "Disposable PostgreSQL did not become ready." >&2; exit 1; } + docker exec "${active_database_container}" sh -euc ' + export PGPASSWORD="$(cat /run/secrets/postgres-password)" + export KEYCLOAK_PASSWORD="$PGPASSWORD" + { printf "%s\n" "\\getenv keycloak_password KEYCLOAK_PASSWORD"; printf "%s\n" "SELECT format('\''CREATE ROLE keycloak LOGIN PASSWORD %L'\'', :'\''keycloak_password'\'') \\gexec" "CREATE DATABASE keycloak OWNER keycloak;"; } | + psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null + ' + docker run --rm --network "${active_network}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=64m \ + --cap-drop ALL --security-opt no-new-privileges:true \ + --mount "type=bind,src=${dump},dst=/backup.dump,readonly" \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + "${postgres_image}" sh -euc 'export PGPASSWORD="$(cat /run/secrets/postgres-password)"; exec pg_restore -h db -U keycloak -d keycloak --no-owner --no-privileges --exit-on-error /backup.dump' >/dev/null + realm_count=$(db_query 'SELECT count(*) FROM realm') + [[ "${realm_count}" =~ ^[1-9][0-9]*$ ]] || { echo "Restored ${slug} database has no realm." >&2; exit 1; } + declare -A before=() after=() + fingerprint_configuration before + runtime_image=${keycloak_image} + runtime_kind=keycloak-base + runtime_evidence_image=${keycloak_image} + if [[ "${slug}" == catwlk ]]; then runtime_image=${catwlk_image}; runtime_evidence_image=${catwlk_image_id}; runtime_kind=catwlk-custom-provider; fi + keycloak_uid=$(docker run --rm --network none --entrypoint id "${runtime_image}" -u) + [[ "${keycloak_uid}" =~ ^[1-9][0-9]*$ ]] || { echo "Keycloak runtime must use a non-root numeric user." >&2; exit 1; } + docker run --rm --network none --cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges:true \ + --mount "type=bind,src=${secret_file},dst=/secret" "${postgres_image}" chown "${keycloak_uid}:0" /secret + chmod 0400 "${secret_file}" + docker run -d --name "${active_keycloak_container}" --network "${active_network}" --network-alias keycloak \ + "${labels[@]}" --read-only --tmpfs /tmp:rw,noexec,nosuid,size=256m \ + --tmpfs "/opt/keycloak/data:rw,nosuid,size=256m,uid=${keycloak_uid},gid=0,mode=0770" \ + --cap-drop ALL --security-opt no-new-privileges:true --entrypoint sh \ + --mount "type=bind,src=${secret_file},dst=/run/secrets/postgres-password,readonly" \ + -e KC_DB=postgres -e KC_DB_URL=jdbc:postgresql://db:5432/keycloak -e KC_DB_USERNAME=keycloak \ + -e KC_HEALTH_ENABLED=true -e KC_HTTP_ENABLED=true -e KC_HOSTNAME_STRICT=false \ + "${runtime_image}" -euc 'export KC_DB_PASSWORD="$(cat /run/secrets/postgres-password)"; exec /opt/keycloak/bin/kc.sh start-dev' >/dev/null + ready=0 + for _ in $(seq 1 180); do + if docker run --rm --network "${active_network}" --read-only --cap-drop ALL --security-opt no-new-privileges:true \ + "${postgres_image}" sh -euc 'wget -qO- http://keycloak:9000/health/ready' 2>/dev/null | grep -q '"status"[[:space:]]*:[[:space:]]*"UP"'; then ready=1; break; fi + [[ $(docker inspect "${active_keycloak_container}" --format '{{.State.Running}}') == true ]] || break + sleep 1 + done + [[ "${ready}" == 1 ]] || { docker logs "${active_keycloak_container}" >&2; echo "Keycloak 26.7.3 did not become ready for restored ${slug}." >&2; exit 1; } + fingerprint_configuration after + fingerprints=() + for category in $(printf '%s\n' "${!category_tables[@]}" | sort); do + [[ "${after[$category]}" == "${before[$category]}" ]] || { echo "Keycloak ${category} configuration regression detected for ${slug}." >&2; exit 1; } + fingerprints+=("${category}=${after[$category]}") + done + combined=$(printf '%s\n' "${fingerprints[@]}" | sha256sum | cut -d' ' -f1) + python3 - "${records}" "${slug}" "${database}" "${backup_sha}" "${runtime_kind}" "${runtime_evidence_image}" "${combined}" "${fingerprints[@]}" <<'PY' +import json, pathlib, sys +fingerprints=dict(item.split("=", 1) for item in sys.argv[8:]) +record={"slug":sys.argv[2],"database":sys.argv[3],"backup_sha256":sys.argv[4],"runtime":sys.argv[5],"runtime_image":sys.argv[6],"configuration_fingerprint":sys.argv[7],"configuration_fingerprints":fingerprints,"restore":"passed","keycloak_startup":"passed","configuration_regression":"passed"} +with pathlib.Path(sys.argv[1]).open("a", encoding="utf-8") as target: target.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") +PY + docker rm -f "${active_keycloak_container}" "${active_database_container}" >/dev/null + active_keycloak_container= + active_database_container= + docker network rm "${active_network}" >/dev/null + active_network= + rm -f "${secret_file}" + unset before after +done + +python3 - "${records}" "${result_dir}/instances.json" "${catwlk_image_id}" <<'PY' +import json, pathlib, sys +instances=[json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +payload={"catwlk_runtime_image_id":sys.argv[3],"instances":instances} +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY +rm -f "${records}" +rmdir "${result_dir}/.runtime" +chmod 0600 "${result_dir}/instances.json" +trap - EXIT HUP INT TERM +echo "Restored all five Keycloak databases with exact runtimes and verified complete configuration fingerprints." diff --git a/scripts/run-brio-encrypted-backup-loop.sh b/scripts/run-brio-encrypted-backup-loop.sh new file mode 100755 index 0000000..ec924f1 --- /dev/null +++ b/scripts/run-brio-encrypted-backup-loop.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +interval_seconds=${BRIO_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${BRIO_BACKUP_RETRY_SECONDS:-300} +backup_root=${BRIO_BACKUP_ROOT:-/backups} + +validate_timing_value() { + case "$2" in + ''|*[!0-9]*) + echo "$1 must be a positive integer." >&2 + exit 1 + ;; + esac +} +validate_timing_value BRIO_BACKUP_INTERVAL_SECONDS "${interval_seconds}" +validate_timing_value BRIO_BACKUP_RETRY_SECONDS "${retry_seconds}" +if [ "${interval_seconds}" -lt 300 ] || [ "${retry_seconds}" -lt 30 ]; then + echo "Brio backup interval must be at least 300 seconds and retry delay at least 30 seconds." >&2 + exit 1 +fi + +if [ "${1:-}" = "healthcheck" ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then + : + else + modified=$(stat -f %m "${status_file}") + fi + max_age=$((interval_seconds * 2 + retry_seconds)) + [ $((now - modified)) -le "${max_age}" ] + exit +fi + +while :; do + if /usr/local/bin/run-brio-encrypted-backup.sh; then + sleep "${interval_seconds}" + else + echo "Encrypted Brio PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-brio-encrypted-backup.sh b/scripts/run-brio-encrypted-backup.sh new file mode 100755 index 0000000..c443708 --- /dev/null +++ b/scripts/run-brio-encrypted-backup.sh @@ -0,0 +1,233 @@ +#!/bin/sh +set -eu + +umask 077 + +database=${BRIO_BACKUP_DATABASE:?BRIO_BACKUP_DATABASE must be brio_staging or keycloak_brio_staging} +backup_root=${BRIO_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_BACKUP_PASSWORD_FILE:-/run/secrets/postgres_backup_password} +recipient_cert=${BRIO_BACKUP_RECIPIENT_CERT:-/etc/postgresql/brio-backup-recipient.crt} +retention_days=${BRIO_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:?PGHOST is required} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:?PGUSER must identify the database-specific Brio backup role} + +case "${database}" in + brio_staging) expected_pg_user=brio_staging_backup ;; + keycloak_brio_staging) expected_pg_user=keycloak_brio_staging_backup ;; + *) + echo "BRIO_BACKUP_DATABASE must be brio_staging or keycloak_brio_staging." >&2 + exit 1 + ;; +esac +if [ "${pg_user}" != "${expected_pg_user}" ]; then + echo "PGUSER must be ${expected_pg_user} when backing up ${database}." >&2 + exit 1 +fi +case "${retention_days}" in + ''|*[!0-9]*) + echo "BRIO_BACKUP_RETENTION_DAYS must be a positive integer." >&2 + exit 1 + ;; +esac +if [ "${retention_days}" -ne 35 ]; then + echo "Brio backup retention must remain exactly 35 days." >&2 + exit 1 +fi +case "${pg_port}" in + ''|*[!0-9]*) + echo "PGPORT must be a positive integer." >&2 + exit 1 + ;; +esac +if [ "${pg_port}" -lt 1 ] || [ "${pg_port}" -gt 65535 ]; then + echo "PGPORT must be between 1 and 65535." >&2 + exit 1 +fi +case "${pg_host}" in + ''|*[!A-Za-z0-9._-]*) + echo "PGHOST contains unsupported characters." >&2 + exit 1 + ;; +esac +case "${pg_user}" in + ''|*[!A-Za-z0-9._-]*) + echo "PGUSER contains unsupported characters." >&2 + exit 1 + ;; +esac +case "${backup_root}" in + ''|/) + echo "BRIO_BACKUP_ROOT must identify a dedicated backup directory." >&2 + exit 1 + ;; +esac + +for command_name in pg_dump openssl sha256sum mkfifo; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "Missing required backup command: ${command_name}" >&2 + exit 1 + fi +done +if [ ! -d "${backup_root}" ] || [ -L "${backup_root}" ]; then + echo "BRIO_BACKUP_ROOT must be a pre-provisioned non-symlink directory." >&2 + exit 1 +fi +chmod 0700 "${backup_root}" +if [ ! -s "${password_file}" ] || [ -L "${password_file}" ]; then + echo "PostgreSQL backup credential must be a non-empty, non-symlink file." >&2 + exit 1 +fi +if [ ! -s "${recipient_cert}" ] || [ -L "${recipient_cert}" ]; then + echo "Brio backup recipient certificate must be a non-empty, non-symlink file." >&2 + exit 1 +fi +if ! openssl x509 -in "${recipient_cert}" -noout -checkend 604800 >/dev/null 2>&1; then + echo "Brio backup recipient certificate is invalid or expires in less than seven days." >&2 + exit 1 +fi + +lock_dir=/tmp/.brio-backup-${database}.lock +if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another Brio PostgreSQL backup is already running for ${database}." >&2 + exit 1 +fi + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +encrypted_name=${database}.dump.cms +encrypted_path=${partial_dir}/${encrypted_name} +pipe_path=/tmp/brio-backup-${database}-${timestamp}-$$.pipe +pgpass=/tmp/brio-backup-${database}-${timestamp}-$$.pgpass +encrypt_pid= + +cleanup() { + if [ -n "${encrypt_pid}" ]; then + kill "${encrypt_pid}" 2>/dev/null || true + wait "${encrypt_pid}" 2>/dev/null || true + fi + if [ -p "${pipe_path}" ] || [ -f "${pipe_path}" ]; then + rm -f "${pipe_path}" + fi + if [ -f "${pgpass}" ]; then + rm -f "${pgpass}" + fi + if [ -d "${partial_dir}" ]; then + find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${partial_dir}" 2>/dev/null || true + fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 1' HUP INT TERM + +if [ -e "${final_dir}" ] || [ -L "${final_dir}" ] || [ -e "${partial_dir}" ] || [ -L "${partial_dir}" ]; then + echo "Backup destination already exists for ${timestamp}." >&2 + exit 1 +fi + +password=$(tr -d '\r\n' < "${password_file}") +if [ -z "${password}" ]; then + echo "PostgreSQL backup credential file contains no usable value." >&2 + exit 1 +fi +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 0600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" +if [ "${PGSSLMODE}" != "verify-full" ]; then + echo "Brio backups require PGSSLMODE=verify-full." >&2 + exit 1 +fi + +mkdir "${partial_dir}" +mkfifo -m 0600 "${pipe_path}" +openssl cms \ + -encrypt \ + -binary \ + -stream \ + -outform DER \ + -aes-256-gcm \ + -recip "${recipient_cert}" \ + -in "${pipe_path}" \ + -out "${encrypted_path}" & +encrypt_pid=$! + +if pg_dump \ + --host="${pg_host}" \ + --port="${pg_port}" \ + --username="${pg_user}" \ + --dbname="${database}" \ + --format=custom \ + --compress=9 \ + --no-owner \ + --no-acl \ + > "${pipe_path}"; then + dump_status=0 +else + dump_status=$? +fi + +if wait "${encrypt_pid}"; then + encrypt_status=0 +else + encrypt_status=$? +fi +encrypt_pid= +rm -f "${pipe_path}" + +if [ "${dump_status}" -ne 0 ] || [ "${encrypt_status}" -ne 0 ]; then + echo "Encrypted Brio backup failed before publication for ${database}." >&2 + exit 1 +fi +if [ ! -s "${encrypted_path}" ]; then + echo "Encrypted Brio backup output is empty." >&2 + exit 1 +fi +if ! openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null 2>&1; then + echo "Encrypted Brio backup is not a valid CMS envelope." >&2 + exit 1 +fi + +recipient_fingerprint=$(openssl x509 -in "${recipient_cert}" -noout -fingerprint -sha256 | cut -d= -f2- | tr -d ':') +case "${recipient_fingerprint}" in + ''|*[!A-Fa-f0-9]*) + echo "Unable to derive the backup recipient certificate fingerprint." >&2 + exit 1 + ;; +esac +cat > "${partial_dir}/metadata.json" < SHA256SUMS +) +chmod 0600 "${partial_dir}"/* +mv "${partial_dir}" "${final_dir}" + +if [ -e "${backup_root}/latest" ] && [ ! -L "${backup_root}/latest" ]; then + echo "Backup latest marker exists but is not a symlink." >&2 + exit 1 +fi +ln -sfn "${timestamp}" "${backup_root}/latest" + +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","database":"%s","encrypted":true}\n' \ + "${timestamp}" "${timestamp}" "${database}" > "${status_tmp}" +chmod 0600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +retention_find_days=$((retention_days - 1)) +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_find_days}" -exec sh -c ' + for directory do + find "$directory" -mindepth 1 -delete + rmdir "$directory" + done +' sh {} + + +echo "Encrypted Brio PostgreSQL backup completed for ${database}: ${final_dir}" diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh new file mode 100755 index 0000000..85e70af --- /dev/null +++ b/scripts/run-ci.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "${repo_root}" +readonly shellcheck_image='docker.io/koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d' + +./scripts/validate-postgres-config.sh +docker version >/dev/null +shellcheck_paths=( \ + scripts/run-brio-encrypted-backup.sh \ + scripts/run-brio-encrypted-backup-loop.sh \ + scripts/deploy-brio-canary-postgres.sh \ + scripts/deploy-brio-identity-db-host.sh \ + scripts/deploy-postgres-stack.sh \ + scripts/brio-db-transaction.sh \ + scripts/ensure-brio-tmp-cleaner.sh \ + scripts/clean-keycloak-cohort-resources.sh \ + scripts/install-keycloak-cohort-cleaner.sh \ + scripts/keycloak-cohort-capture-dispatch.sh \ + scripts/install-keycloak-cohort-capture-host.sh \ + scripts/verify-brio-encrypted-restore.sh \ + scripts/test-brio-bootstrap.sh \ + scripts/test-brio-db-transaction.sh \ + scripts/test-brio-encrypted-backup.sh \ + scripts/test-brio-encrypted-restore.sh \ + scripts/test-brio-deploy-guards.sh \ + scripts/test-brio-deployment-contracts.sh \ + scripts/test-brio-deployment-failures.sh \ + scripts/test-brio-release-evidence.sh \ + scripts/test-keycloak-cohort-evidence.sh \ + scripts/test-keycloak-cohort-hardening.sh \ + scripts/capture-keycloak-cohort-backups.sh \ + scripts/restore-keycloak-cohort-backups.sh \ + scripts/fixtures/brio-deployment-failure-fixture.sh \ + scripts/fixtures/keycloak-cohort-cleaner-fixture.sh \ + scripts/fixtures/keycloak-cohort-dispatch-fixture.sh \ +) +docker run --rm --network none \ + --volume "${repo_root}:/workspace:ro" \ + --workdir /workspace \ + "${shellcheck_image}" "${shellcheck_paths[@]}" +python3 - <<'PY' +import ast +from pathlib import Path + +for source in ( + "scripts/verify-brio-release-evidence.py", + "scripts/verify-keycloak-cohort-evidence.py", + "scripts/reconcile-github-environment-main-policy.py", + "scripts/test-github-environment-main-policy.py", +): + ast.parse(Path(source).read_text(), filename=source) +PY +PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-github-environment-main-policy.py +actionlint +git diff --check +./scripts/test-brio-deploy-guards.sh +./scripts/test-brio-deployment-contracts.sh +./scripts/test-brio-deployment-failures.sh +./scripts/test-brio-release-evidence.sh +./scripts/test-keycloak-cohort-evidence.sh +./scripts/test-keycloak-cohort-hardening.sh +./scripts/test-brio-bootstrap.sh +./scripts/test-brio-db-transaction.sh +./scripts/test-brio-encrypted-backup.sh +./scripts/test-brio-encrypted-restore.sh +backup_image=$(grep '^BRIO_BACKUP_IMAGE=' envs/canary/.env.db | cut -d= -f2-) +docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-backup.sh +docker run --rm --volume "${PWD}:/repo:ro" --workdir /repo "${backup_image}" bash scripts/test-brio-encrypted-restore.sh diff --git a/scripts/test-brio-bootstrap.sh b/scripts/test-brio-bootstrap.sh new file mode 100755 index 0000000..83b79c9 --- /dev/null +++ b/scripts/test-brio-bootstrap.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +postgres_image=$(awk -F= '$1 == "POSTGRES_IMAGE" {print $2}' "${repo_root}/envs/canary/.env.db") +container_name="brio-postgres-bootstrap-${RANDOM}-$$" +postgres_password='brio-bootstrap-integration-only' +brio_app_password='brio-app-integration-only' +brio_backup_password='brio-app-backup-integration-only' +keycloak_app_password='brio-keycloak-integration-only' +keycloak_backup_password='brio-keycloak-backup-integration-only' + +cleanup() { + local status=$? + trap - EXIT + if ((status != 0)); then + echo "Brio bootstrap test failed; PostgreSQL container state and logs follow." >&2 + docker inspect "${container_name}" \ + --format 'status={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' >&2 2>/dev/null || true + docker logs --tail 200 "${container_name}" >&2 2>/dev/null || true + fi + docker rm -f "${container_name}" >/dev/null 2>&1 || true + exit "${status}" +} +trap cleanup EXIT + +docker run -d --name "${container_name}" \ + -e POSTGRES_PASSWORD="${postgres_password}" \ + -v "${repo_root}/bootstrap:/bootstrap:ro" \ + "${postgres_image}" >/dev/null + +postgres_ready=false +for _ in $(seq 1 300); do + if docker exec "${container_name}" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null 2>&1; then + postgres_ready=true + break + fi + if [[ "$(docker inspect "${container_name}" --format '{{.State.Running}}' 2>/dev/null || true)" != "true" ]]; then + echo "PostgreSQL container exited before TCP readiness." >&2 + exit 1 + fi + sleep 0.1 +done +if [[ "${postgres_ready}" != "true" ]]; then + echo "PostgreSQL did not become ready on TCP loopback within 30 seconds." >&2 + exit 1 +fi +docker exec "${container_name}" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null + +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null 2>&1; then + echo "Brio application bootstrap unexpectedly accepted a missing backup-role password." >&2 + exit 1 +fi +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password='' \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null 2>&1; then + echo "Brio Keycloak bootstrap unexpectedly accepted an empty backup-role password." >&2 + exit 1 +fi + +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -v brio_staging_backup_password="${brio_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null 2>&1; then + echo "Brio application bootstrap unexpectedly accepted a remote plaintext administrator session." >&2 + exit 1 +fi +if docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password="${keycloak_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null 2>&1; then + echo "Brio Keycloak bootstrap unexpectedly accepted a remote plaintext administrator session." >&2 + exit 1 +fi + +run_keycloak_brio_bootstrap() { + docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v keycloak_brio_staging_app_password="${keycloak_app_password}" \ + -v keycloak_brio_staging_backup_password="${keycloak_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null +} + +run_brio_bootstrap() { + docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v brio_staging_app_password="${brio_app_password}" \ + -v brio_staging_backup_password="${brio_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null +} + +# Both bootstraps are explicitly idempotent. +run_keycloak_brio_bootstrap +run_brio_bootstrap +run_keycloak_brio_bootstrap +run_brio_bootstrap + +role_contract=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_roles WHERE rolname IN ('brio_staging_app','brio_staging_backup','keycloak_brio_staging_app','keycloak_brio_staging_backup') AND rolcanlogin AND NOT rolsuper AND NOT rolcreatedb AND NOT rolcreaterole AND NOT rolreplication AND NOT rolbypassrls") +[[ "${role_contract}" == "4" ]] + +owner_contract=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_database d JOIN pg_roles r ON r.oid=d.datdba WHERE (d.datname='brio_staging' AND r.rolname='brio_staging_app') OR (d.datname='keycloak_brio_staging' AND r.rolname='keycloak_brio_staging_app')") +[[ "${owner_contract}" == "2" ]] + +public_connect=$(docker exec -e PGPASSWORD="${postgres_password}" "${container_name}" \ + psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_database d, LATERAL aclexplode(coalesce(d.datacl, acldefault('d', d.datdba))) a WHERE d.datname IN ('brio_staging','keycloak_brio_staging') AND a.grantee=0 AND a.privilege_type='CONNECT'") +[[ "${public_connect}" == "0" ]] + +assert_backup_role() { + local database=$1 + local app_user=$2 + local app_password=$3 + local backup_user=$4 + local backup_password=$5 + local other_database=$6 + + docker exec -e PGPASSWORD="${app_password}" "${container_name}" \ + psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U "${app_user}" -d "${database}" \ + -c "CREATE TABLE IF NOT EXISTS public.backup_role_probe (marker text NOT NULL)" \ + -c "TRUNCATE public.backup_role_probe" \ + -c "INSERT INTO public.backup_role_probe(marker) VALUES ('readable')" >/dev/null + + read_only=$(docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -At -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "SHOW default_transaction_read_only") + [[ "${read_only}" == "on" ]] + + readable=$(docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -At -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "SELECT marker FROM public.backup_role_probe") + [[ "${readable}" == "readable" ]] + + if docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -h 127.0.0.1 -U "${backup_user}" -d "${database}" \ + -c "INSERT INTO public.backup_role_probe(marker) VALUES ('forbidden')" >/dev/null 2>&1; then + echo "Backup role ${backup_user} unexpectedly wrote to ${database}." >&2 + exit 1 + fi + + docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + pg_dump -h 127.0.0.1 -U "${backup_user}" -d "${database}" --format=custom >/dev/null + + if docker exec -e PGPASSWORD="${backup_password}" "${container_name}" \ + psql -h 127.0.0.1 -U "${backup_user}" -d "${other_database}" -c "SELECT 1" >/dev/null 2>&1; then + echo "Backup role ${backup_user} unexpectedly connected to ${other_database}." >&2 + exit 1 + fi +} + +assert_backup_role \ + brio_staging brio_staging_app "${brio_app_password}" \ + brio_staging_backup "${brio_backup_password}" keycloak_brio_staging +assert_backup_role \ + keycloak_brio_staging keycloak_brio_staging_app "${keycloak_app_password}" \ + keycloak_brio_staging_backup "${keycloak_backup_password}" brio_staging + +echo "Brio PostgreSQL app and read-only backup role, database, privilege, and idempotency tests passed" diff --git a/scripts/test-brio-db-transaction.sh b/scripts/test-brio-db-transaction.sh new file mode 100755 index 0000000..e20d06a --- /dev/null +++ b/scripts/test-brio-db-transaction.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +postgres_image=$(awk -F= '$1 == "POSTGRES_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +container="brio-db-transaction-${RANDOM}-$$" +superuser_password='transaction-superuser-only' +old_app_password='transaction-old-app-only' +old_backup_password='transaction-old-backup-only' +new_app_password='transaction-new-app-only' +new_backup_password='transaction-new-backup-only' + +cleanup() { + local status=$? + trap - EXIT + docker rm -f "${container}" >/dev/null 2>&1 || true + exit "${status}" +} +trap cleanup EXIT + +docker run -d --name "${container}" \ + -e "POSTGRES_PASSWORD=${superuser_password}" \ + -e POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 \ + --mount "type=bind,src=${repo_root}/bootstrap,dst=/bootstrap,readonly" \ + --mount "type=bind,src=${repo_root}/scripts/brio-db-transaction.sh,dst=/usr/local/bin/brio-db-transaction.sh,readonly" \ + "${postgres_image}" >/dev/null +for _ in $(seq 1 300); do + docker exec -e "PGPASSWORD=${superuser_password}" "${container}" \ + psql -X -h 127.0.0.1 -U postgres -d postgres -c 'select 1' >/dev/null 2>&1 && break + sleep 0.1 +done +docker exec -e "PGPASSWORD=${superuser_password}" "${container}" \ + psql -X -h 127.0.0.1 -U postgres -d postgres -c 'select 1' >/dev/null +docker exec "${container}" sh -euc 'install -d -m 0700 /journal/keycloak /journal/brio; printf "%s" "$1" > /run/superuser-password; chmod 0600 /run/superuser-password' sh "${superuser_password}" + +psql_admin() { + docker exec -e "PGPASSWORD=${superuser_password}" "${container}" psql -X -v ON_ERROR_STOP=1 -U postgres "$@" +} +transaction() { + docker exec \ + -e PGUSER=postgres -e PGHOST=127.0.0.1 -e PGSSLMODE=disable -e PGPASSWORD_FILE=/run/superuser-password \ + "${container}" /usr/local/bin/brio-db-transaction.sh "$@" +} + +psql_admin -d postgres \ + -c "CREATE ROLE brio_legacy_owner NOLOGIN" \ + -c "CREATE ROLE keycloak_brio_staging_app LOGIN NOINHERIT CONNECTION LIMIT 7 PASSWORD '${old_app_password}' VALID UNTIL '2035-01-02 03:04:05+00'" \ + -c "CREATE ROLE keycloak_brio_staging_backup LOGIN INHERIT CONNECTION LIMIT 3 PASSWORD '${old_backup_password}'" \ + -c "ALTER ROLE keycloak_brio_staging_app SET statement_timeout TO '17s'" \ + -c "CREATE DATABASE keycloak_brio_staging OWNER brio_legacy_owner" \ + -c "REVOKE CONNECT ON DATABASE keycloak_brio_staging FROM PUBLIC" \ + -c "GRANT CONNECT ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_app" \ + -c "GRANT CREATE ON DATABASE keycloak_brio_staging TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "ALTER ROLE keycloak_brio_staging_backup IN DATABASE keycloak_brio_staging SET lock_timeout TO '19s'" >/dev/null +psql_admin -d keycloak_brio_staging \ + -c "REVOKE ALL ON SCHEMA public FROM PUBLIC" \ + -c "GRANT CREATE ON SCHEMA public TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "CREATE TABLE public.preexisting_acl_probe(id integer)" \ + -c "GRANT UPDATE ON public.preexisting_acl_probe TO keycloak_brio_staging_backup WITH GRANT OPTION" \ + -c "ALTER DEFAULT PRIVILEGES FOR ROLE keycloak_brio_staging_app IN SCHEMA public GRANT INSERT ON TABLES TO keycloak_brio_staging_backup WITH GRANT OPTION" >/dev/null + +transaction prepare keycloak /journal/keycloak +pre_fingerprint=$(transaction fingerprint keycloak /journal/keycloak | sha256sum | cut -d' ' -f1) +psql_admin -d postgres \ + -v "keycloak_brio_staging_app_password=${new_app_password}" \ + -v "keycloak_brio_staging_backup_password=${new_backup_password}" \ + -f /bootstrap/keycloak-brio-staging.sql >/dev/null +docker exec -e "PGPASSWORD=${new_app_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null +docker exec -e "PGPASSWORD=${new_backup_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null +if docker exec -e "PGPASSWORD=${old_app_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null 2>&1; then + echo "Old application credential unexpectedly survived the simulated mutation." >&2 + exit 1 +fi +if docker exec -e "PGPASSWORD=${old_backup_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null 2>&1; then + echo "Old backup credential unexpectedly survived the simulated mutation." >&2 + exit 1 +fi +transaction restore keycloak /journal/keycloak +post_fingerprint=$(transaction fingerprint keycloak /journal/keycloak | sha256sum | cut -d' ' -f1) +[[ "${post_fingerprint}" == "${pre_fingerprint}" ]] || { echo "Exact Keycloak Brio state fingerprint was not restored." >&2; exit 1; } +docker exec -e "PGPASSWORD=${old_app_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null +docker exec -e "PGPASSWORD=${old_backup_password}" "${container}" \ + psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null +if docker exec -e "PGPASSWORD=${new_app_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_app -d keycloak_brio_staging -c 'select 1' >/dev/null 2>&1; then + echo "New application credential remained valid after compensation." >&2 + exit 1 +fi +if docker exec -e "PGPASSWORD=${new_backup_password}" "${container}" \ + psql -X -h 127.0.0.1 -U keycloak_brio_staging_backup -d postgres -c 'select 1' >/dev/null 2>&1; then + echo "New backup credential remained valid after compensation." >&2 + exit 1 +fi + +# The absent-state path must remove every object introduced by a failed first +# deployment, not merely rotate credentials back. +transaction prepare brio /journal/brio +psql_admin -d postgres \ + -v "brio_staging_app_password=${new_app_password}" \ + -v "brio_staging_backup_password=${new_backup_password}" \ + -f /bootstrap/brio-staging-app.sql >/dev/null +transaction restore brio /journal/brio +[[ $(psql_admin -At -d postgres -c "SELECT count(*) FROM pg_database WHERE datname='brio_staging'") == 0 ]] +[[ $(psql_admin -At -d postgres -c "SELECT count(*) FROM pg_roles WHERE rolname IN ('brio_staging_app','brio_staging_backup')") == 0 ]] + +# A recovery mismatch must fail closed without leaking the stored fingerprint. +# That fingerprint includes SCRAM verifiers and therefore must never appear in +# CI or remote deployment diagnostics. +docker exec "${container}" sh -euc 'printf "%s\n" sentinel-secret-verifier >> /journal/brio/prestate.fingerprint' +if mismatch_output=$(transaction restore brio /journal/brio 2>&1); then + echo "A mismatched recovery fingerprint was unexpectedly accepted." >&2 + exit 1 +fi +[[ "${mismatch_output}" == *"Database compensation did not restore the exact prior Brio state."* ]] || { + echo "Recovery mismatch did not return the expected redacted diagnostic." >&2 + exit 1 +} +[[ "${mismatch_output}" != *sentinel-secret-verifier* ]] || { + echo "Recovery mismatch leaked protected fingerprint material." >&2 + exit 1 +} +unset mismatch_output + +echo "Brio database transaction journal restores roles, SCRAM verifiers, attributes, database ownership, ACLs, defaults, and absent state." diff --git a/scripts/test-brio-deploy-guards.sh b/scripts/test-brio-deploy-guards.sh new file mode 100755 index 0000000..4562dd6 --- /dev/null +++ b/scripts/test-brio-deploy-guards.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +expect_refusal() { + local expected=$1 + shift + local output status + set +e + output=$("$@" 2>&1) + status=$? + set -e + if [[ ${status} -eq 0 || "${output}" != *"${expected}"* ]]; then + echo "Expected deployment guard refusal containing: ${expected}" >&2 + exit 1 + fi +} + +expect_refusal "unique /srv or /opt" \ + "${script_dir}/deploy-brio-canary-postgres.sh" relative-dir brio /tmp/postgres-brio-canary-runtime-1-1 + +expect_refusal "exact standalone DB restart acknowledgement" \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 65.21.134.125 88.99.209.165/32 + +expect_refusal "reviewed standalone DB IP" \ + env BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 127.0.0.1 88.99.209.165/32 + +expect_refusal "reviewed exact egress" \ + env BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging \ + BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes \ + "${script_dir}/deploy-brio-identity-db-host.sh" /tmp/postgres-brio-identity-bundle-1-1 /tmp/postgres-brio-identity-runtime-1-1 65.21.134.125 10.80.0.1/32 + +expect_refusal "unique /srv or /opt" \ + "${script_dir}/deploy-postgres-stack.sh" /srv/makepad/postgres postgres canary + +expect_refusal "Production requires a job-scoped" \ + "${script_dir}/deploy-postgres-stack.sh" /srv/makepad/postgres/.deploy/postgres-1-1 postgres production + +echo "Brio deployment guard tests passed." diff --git a/scripts/test-brio-deployment-contracts.sh b/scripts/test-brio-deployment-contracts.sh new file mode 100755 index 0000000..9b00bdc --- /dev/null +++ b/scripts/test-brio-deployment-contracts.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) + +REPO_ROOT="${repo_root}" python3 - <<'PY' +import os +from pathlib import Path + +root = Path(os.environ["REPO_ROOT"]) +manual = (root / ".github/workflows/manual-deploy.yml").read_text() +identity_workflow = (root / ".github/workflows/deploy-brio-identity-db.yml").read_text() +release_workflow = (root / ".github/workflows/release-brio-identity-db.yml").read_text() +ci_workflow = (root / ".github/workflows/ci.yml").read_text() +cohort_workflow = (root / ".github/workflows/verify-keycloak-cohort-restores.yml").read_text() +cohort_validator = (root / "scripts/verify-keycloak-cohort-evidence.py").read_text() +identity = (root / "scripts/deploy-brio-identity-db-host.sh").read_text() +canary = (root / "scripts/deploy-brio-canary-postgres.sh").read_text() +stack = (root / "scripts/deploy-postgres-stack.sh").read_text() +hba = (root / "config/runtrace-pg_hba.conf").read_text().splitlines() +canary_env = (root / "envs/canary/.env.db").read_text() +production_env = (root / "envs/production/.env.db").read_text() + +def require(condition, message): + if not condition: + raise SystemExit(message) + +require("makepad_postgres_canary_runtrace_hba_v3" in canary_env, "canary HBA must use immutable v3") +require("makepad_postgres_runtrace_hba_v3" in production_env, "production HBA must use immutable v3") +require("runtrace_hba_v2" not in canary_env + production_env, "active HBA v2 names must not drift") + +records = [tuple(line.split()) for line in hba if line.strip() and not line.lstrip().startswith("#")] +for allow, reject in ( + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_app", "all", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_app", "all", "reject")), + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_backup", "127.0.0.1/32", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_backup", "all", "reject")), +): + require(records.index(allow) < records.index(reject), "identity HBA allow must precede its role rejection") + +require("group: postgres-shared-swarm-target" in manual, "shared Swarm target needs one concurrency group") +require('remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' in manual, "Swarm bundle must be unique per attempt") +require("${REMOTE_DIR}/stack.yml" not in manual + stack, "shared stack.yml is forbidden") +require('stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml"' in stack, "stack config must stay in the run bundle") +require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual + stack, "VIF secret must not persist in .env.deploy") +require("-v vif_password=" not in stack, "VIF secret must not enter psql argv") +require("\\getenv vif_password VIF_PASSWORD" in stack, "VIF bootstrap must use getenv") + +for marker in ( + "compose_project=postgres", + "expected_container_name=postgres-postgres-1", + "com.docker.compose.project", + "com.docker.compose.service", + "com.docker.compose.oneoff", + "bind|/var/lib/makepad/postgres|true", + '"${network_mode}" == "host"', + 'tar --numeric-owner -cpf "$stage/rollback/managed.tar"', + "restore_snapshot", + "rollback_deployment", + "trap handle_exit EXIT", + "trap 'exit 129' HUP", + "trap 'exit 130' INT", + "trap 'exit 143' TERM", + "up -d --remove-orphans --wait --force-recreate", +): + require(marker in identity, f"standalone contract missing: {marker}") + +snapshot = identity.index('tar --numeric-owner -cpf "$stage/rollback/managed.tar"') +armed = identity.index("rollback_armed=1", snapshot) +mutation = identity.index('install_host_path "${candidate_compose}"', armed) +backup = identity.index('[[ "${backup_verified}" == "1" ]]', mutation) +disarmed = identity.index("rollback_armed=0", backup) +require(snapshot < armed < mutation < backup < disarmed, "rollback boundary/order is unsafe") + +require("/tmp/postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" in identity_workflow, "identity bundle must be per attempt") +require("POSTGRES_HOST_COMPOSE_PROJECT" not in identity_workflow, "standalone project must not be user-selected") +require("brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }}" in identity_workflow, "phase one must publish its exact immutable deployment artifact") +require(identity_workflow.count("actions/upload-artifact@") == 1, "phase one must publish exactly one artifact") +require("brio-db-deployment-evidence.json" in identity_workflow and "makepad.brio-db-deployment-evidence.v1" in identity_workflow, "phase one must use the canonical single-file schema") +for marker in ( + "environment: release-brio-identity-db", + "KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN", + "verify-brio-release-evidence.py postgres-run", + "verify-brio-release-evidence.py postgres-evidence", + "verify-brio-database.yml/dispatches", + "verify-brio-release-evidence.py verifier-run", + "verify-brio-release-evidence.py attestation", + 'release_token=${RELEASE_ORCHESTRATOR_TOKEN}', + 'unset RELEASE_ORCHESTRATOR_TOKEN', + '--config -', +): + require(marker in release_workflow, f"protected release orchestrator missing: {marker}") +require("actions/upload-artifact@" not in release_workflow, "release orchestrator must not synthesize or republish attestation") +require("pull_request:" in ci_workflow and "pull_request_target:" not in ci_workflow, "PR CI must use the native pull-request event") +require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks") +require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact head") +require(ci_workflow.count("runs-on: [self-hosted, linux, x64, makepad]") == 2, "PR and protected-main CI must use the existing Makepad Linux runner") +for marker in ( + "name: Verify Keycloak Cohort Restore Compatibility", + "keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "makepad.keycloak-cohort-restore-evidence.v2", + "restored-databases-compatible", + "keycloak_release_sha", +): + require(marker in cohort_workflow + cohort_validator, f"five-database cohort evidence contract missing: {marker}") +require("vars." not in cohort_workflow, "cohort evidence cannot rely on a mutable repository variable") +require("Ensure interrupted cohort material expires on the release host" in cohort_workflow, "cohort workflow must verify the release-host TTL guard before credentials or dumps") +require(cohort_workflow.index("Ensure interrupted cohort material expires on the release host") < cohort_workflow.index("Configure isolated SSH and registry state"), "release-host TTL guard must precede credential material") +require('"probe ${helper_digest} ${cleaner_digest}"' in cohort_workflow and cohort_workflow.index('"probe ${helper_digest} ${cleaner_digest}"') < cohort_workflow.index('"capture ${GITHUB_RUN_ID}'), "remote helper/cleaner digest and TTL probe must precede every dump") +require("scp " not in cohort_workflow and "remote_script=" not in cohort_workflow, "cohort workflow must use only the forced-command capture protocol") +require('/tmp/postgres-keycloak-cohort-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}' in cohort_workflow, "cohort sensitive material must live under the exact TTL-cleaned namespace") +shared_network_validation = canary.index('prevalidate_network "${db_network}" false') +incomplete_recovery = canary.index("recover_incomplete_journals", shared_network_validation) +database_journal = canary.index('run_db_transaction prepare "${journal_stage}"', incomplete_recovery) +require(shared_network_validation < incomplete_recovery < database_journal, "shared DB transport validation must precede recovery and first-deploy journaling") +for marker in ( + "assert_no_symlink_components", + "prevalidate_swarm_config", + "prevalidate_network", + "docker stack config", + 'tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar"', + "rollback_canary", + "prior-service-spec-hashes.list", + 'mv -fT "$super_stage"', + "postgres-recovery/brio-canary", + "DATABASE_MUTATION_ARMED", + "STACK_MUTATION_ARMED", + "keycloak_brio_staging_backup", + "/usr/local/bin/run-brio-encrypted-backup.sh", +): + require(marker in canary, f"canary transactional contract missing: {marker}") +for marker in ("preserve_recovery_evidence", "postgres-recovery/brio-identity", "DATABASE_MUTATION_ARMED", "RECOVERY_REQUIRED"): + require(marker in identity, f"identity durable recovery contract missing: {marker}") +for marker in ("identity-backups.tar", "identity-backup-absent", "recovery_id=${identifier}"): + require(marker in identity, f"identity exact backup/recovery contract missing: {marker}") + +for workflow in (manual, identity_workflow): + cleaner = workflow.index("ensure-brio-tmp-cleaner.sh") + secret_copy = workflow.index('scp "${scp_opts[@]}" "${runtime_dir}') + require(cleaner < secret_copy, "host TTL cleaner must precede secret transfer") +PY + +cleaner_root=$(mktemp -d /tmp/postgres-brio-cleaner-test-contract-XXXXXX) +ownership_root="" +cleaner_image="" +cleanup_test_root() { + [[ "${cleaner_root}" =~ ^/tmp/postgres-brio-cleaner-test-contract-[A-Za-z0-9]+$ ]] || return 1 + if [[ -n "${ownership_root}" && ( -e "${ownership_root}" || -L "${ownership_root}" ) ]]; then + [[ "${ownership_root}" =~ ^/tmp/postgres-brio-cleaner-test-production-ownership-[A-Za-z0-9]+$ \ + && -d "${ownership_root}" && ! -L "${ownership_root}" \ + && "${cleaner_image}" == *@sha256:* ]] || return 1 + docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" \ + "${cleaner_image}" sh -euc 'find /fixture -mindepth 1 -depth -delete' + rmdir -- "${ownership_root}" + fi + find "${cleaner_root}" -depth -delete +} +trap cleanup_test_root EXIT +mkdir "${cleaner_root}/postgres-brio-old" "${cleaner_root}/postgres-brio-recovery" \ + "${cleaner_root}/postgres-brio-fresh" "${cleaner_root}/postgres-keycloak-cohort-old" \ + "${cleaner_root}/unrelated-old" +printf '%s\n' recovery-required > "${cleaner_root}/postgres-brio-recovery/RECOVERY_REQUIRED" +touch -t 202001010000 "${cleaner_root}/postgres-brio-old" "${cleaner_root}/postgres-brio-recovery" \ + "${cleaner_root}/postgres-keycloak-cohort-old" "${cleaner_root}/unrelated-old" +"${script_dir}/ensure-brio-tmp-cleaner.sh" test-clean-once "${cleaner_root}" +[[ ! -e "${cleaner_root}/postgres-brio-old" ]] || { echo "TTL cleaner retained an expired Brio directory." >&2; exit 1; } +[[ ! -e "${cleaner_root}/postgres-keycloak-cohort-old" ]] || { echo "TTL cleaner retained expired Keycloak cohort material." >&2; exit 1; } +[[ -d "${cleaner_root}/postgres-brio-fresh" ]] || { echo "TTL cleaner removed a fresh Brio directory." >&2; exit 1; } +[[ -f "${cleaner_root}/postgres-brio-recovery/RECOVERY_REQUIRED" ]] || { echo "TTL cleaner removed required recovery evidence." >&2; exit 1; } +[[ -d "${cleaner_root}/unrelated-old" ]] || { echo "TTL cleaner removed unrelated content." >&2; exit 1; } + +# Reproduce production ownership: SSH-created runtime directories are mode 0700 +# and owned by the deploy UID, not by the cleaner container. Minimal DAC/FOWNER +# capabilities must delete expired material while retaining recovery markers. +cleaner_image=$(awk -F= '$1 == "POSTGRES_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +ownership_root=$(mktemp -d /tmp/postgres-brio-cleaner-test-production-ownership-XXXXXX) +chmod 0700 "${ownership_root}" +docker run --rm --mount "type=bind,src=${ownership_root},dst=/fixture" "${cleaner_image}" sh -euc ' + mkdir /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + printf "%s\n" secret > /fixture/postgres-brio-deploy-owned/credential + printf "%s\n" recovery > /fixture/postgres-brio-recovery-owned/RECOVERY_REQUIRED + chown -R 12345:12345 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + chmod 0700 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned + touch -t 202001010000 /fixture/postgres-brio-deploy-owned /fixture/postgres-brio-recovery-owned +' +"${script_dir}/ensure-brio-tmp-cleaner.sh" test-clean-production-ownership "${ownership_root}" "${cleaner_image}" +[[ ! -e "${ownership_root}/postgres-brio-deploy-owned" ]] || { echo "Production cleaner retained a deploy-UID-owned expired secret directory." >&2; exit 1; } +docker run --rm --read-only --cap-drop ALL --cap-add DAC_OVERRIDE \ + --security-opt no-new-privileges --mount "type=bind,src=${ownership_root},dst=/fixture,readonly" \ + "${cleaner_image}" sh -euc 'test -f /fixture/postgres-brio-recovery-owned/RECOVERY_REQUIRED' || { + echo "Production cleaner removed recovery evidence." >&2 + exit 1 + } + +echo "Brio deployment ordering, rollback, interruption, secret, and TTL contracts passed." diff --git a/scripts/test-brio-deployment-failures.sh b/scripts/test-brio-deployment-failures.sh new file mode 100755 index 0000000..2b0d746 --- /dev/null +++ b/scripts/test-brio-deployment-failures.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) +test_image=$(grep '^BRIO_BACKUP_IMAGE=' "${repo_root}/envs/canary/.env.db" | cut -d= -f2-) +: "${test_image:?BRIO_BACKUP_IMAGE is required for isolated failure testing}" +[[ "${test_image}" == *@sha256:* ]] || { echo "Failure tests require a digest-pinned container image." >&2; exit 1; } + +docker run --rm \ + --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" \ + "${test_image}" bash /repo/scripts/fixtures/brio-deployment-failure-fixture.sh diff --git a/scripts/test-brio-encrypted-backup.sh b/scripts/test-brio-encrypted-backup.sh new file mode 100755 index 0000000..c805826 --- /dev/null +++ b/scripts/test-brio-encrypted-backup.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups/app" "${work_dir}/backups/failure" +chmod 0700 "${work_dir}/backups/app" "${work_dir}/backups/failure" +mkdir "${work_dir}/backups/app/20200101T000000Z" "${work_dir}/backups/app/20990101T000000Z" +touch -t 202001010000 "${work_dir}/backups/app/20200101T000000Z" +printf 'validation-password\n' > "${work_dir}/password" +chmod 0600 "${work_dir}/password" + +openssl req \ + -x509 \ + -newkey rsa:2048 \ + -nodes \ + -days 30 \ + -subj /CN=brio-backup-contract \ + -keyout "${work_dir}/recipient.key" \ + -out "${work_dir}/recipient.crt" >/dev/null 2>&1 +chmod 0600 "${work_dir}/recipient.key" + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +database= +for argument in "$@"; do + case "${argument}" in + --dbname=*) database=${argument#--dbname=} ;; + --file=*) + echo "Brio backup must stream pg_dump and never write a plaintext file." >&2 + exit 91 + ;; + esac +done +[ "${database}" = "brio_staging" ] || [ "${database}" = "keycloak_brio_staging" ] +printf 'contract plaintext for %s\n' "${database}" +if [ "${FAKE_PG_DUMP_FAIL:-0}" = "1" ]; then + exit 42 +fi +EOF +chmod 0700 "${work_dir}/bin/pg_dump" + +PATH="${work_dir}/bin:${PATH}" \ +BRIO_BACKUP_DATABASE=brio_staging \ +BRIO_BACKUP_ROOT="${work_dir}/backups/app" \ +BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ +BRIO_BACKUP_RETENTION_DAYS=35 \ +POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ +PGHOST=makepad-postgres-brio-staging \ +PGPORT=5432 \ +PGUSER=brio_staging_backup \ +PGSSLMODE=verify-full \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ +sh "${repo_root}/scripts/run-brio-encrypted-backup.sh" + +latest=$(readlink "${work_dir}/backups/app/latest") +backup_dir=${work_dir}/backups/app/${latest} +for expected in brio_staging.dump.cms SHA256SUMS metadata.json; do + test -s "${backup_dir}/${expected}" +done +test ! -e "${backup_dir}/brio_staging.dump" +if grep -aFq 'contract plaintext' "${backup_dir}/brio_staging.dump.cms"; then + echo "Encrypted artifact exposed plaintext backup content." >&2 + exit 1 +fi +( + cd "${backup_dir}" + sha256sum --check --strict SHA256SUMS >/dev/null +) +openssl cms \ + -decrypt \ + -binary \ + -inform DER \ + -recip "${work_dir}/recipient.crt" \ + -inkey "${work_dir}/recipient.key" \ + -in "${backup_dir}/brio_staging.dump.cms" \ + -out "${work_dir}/decrypted.dump" +grep -Fq 'contract plaintext for brio_staging' "${work_dir}/decrypted.dump" +test -s "${work_dir}/backups/app/last-success.json" +test ! -e "${work_dir}/backups/app/20200101T000000Z" +test -d "${work_dir}/backups/app/20990101T000000Z" + +BRIO_BACKUP_ROOT="${work_dir}/backups/app" \ +BRIO_BACKUP_INTERVAL_SECONDS=300 \ +BRIO_BACKUP_RETRY_SECONDS=30 \ +sh "${repo_root}/scripts/run-brio-encrypted-backup-loop.sh" healthcheck + +if PATH="${work_dir}/bin:${PATH}" \ + FAKE_PG_DUMP_FAIL=1 \ + BRIO_BACKUP_DATABASE=keycloak_brio_staging \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_BACKUP_RETENTION_DAYS=35 \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + PGPORT=5432 \ + PGUSER=keycloak_brio_staging_backup \ + PGSSLMODE=verify-full \ + PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "A failing pg_dump unexpectedly published a Brio backup." >&2 + exit 1 +fi +if find "${work_dir}/backups/failure" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' | grep -q .; then + echo "A failing pg_dump left a published timestamp directory." >&2 + exit 1 +fi +if find "${work_dir}/backups/failure" -type f -name '*.dump' -o -name '*.dump.cms' | grep -q .; then + echo "A failing pg_dump left a backup artifact." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE=brio_staging \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres-brio-staging \ + PGUSER=postgres \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "The PostgreSQL superuser unexpectedly passed Brio backup validation." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE=unexpected_database \ + BRIO_BACKUP_ROOT="${work_dir}/backups/failure" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh"; then + echo "An unallowlisted database unexpectedly passed backup validation." >&2 + exit 1 +fi + +echo "Brio encrypted PostgreSQL backup contract passed." diff --git a/scripts/test-brio-encrypted-restore.sh b/scripts/test-brio-encrypted-restore.sh new file mode 100755 index 0000000..e220cee --- /dev/null +++ b/scripts/test-brio-encrypted-restore.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p \ + "${work_dir}/bin" \ + "${work_dir}/backups/app" \ + "${work_dir}/backups/keycloak" \ + "${work_dir}/restore-tmp" +chmod 0700 "${work_dir}/backups/app" "${work_dir}/backups/keycloak" "${work_dir}/restore-tmp" +printf 'validation-password\n' > "${work_dir}/password" +printf '[brio_app_restore_test]\nhost=nonproduction.invalid\n\n[brio_keycloak_restore_test]\nhost=nonproduction.invalid\n' > "${work_dir}/pg_service.conf" +chmod 0600 "${work_dir}/password" "${work_dir}/pg_service.conf" + +openssl req \ + -x509 \ + -newkey rsa:2048 \ + -nodes \ + -days 30 \ + -subj /CN=brio-restore-contract \ + -keyout "${work_dir}/recipient.key" \ + -out "${work_dir}/recipient.crt" >/dev/null 2>&1 +chmod 0600 "${work_dir}/recipient.key" + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +database= +for argument in "$@"; do + case "${argument}" in + --dbname=*) database=${argument#--dbname=} ;; + esac +done +printf 'fake custom dump for %s\n' "${database}" +EOF + +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +if [ "${1:-}" = "--list" ]; then + grep -Fq 'fake custom dump for ' "$2" + exit 0 +fi +joined=" $* " +last_argument= +for argument in "$@"; do + last_argument=${argument} +done +for expected in '--clean' '--if-exists' '--no-owner' '--no-acl' '--exit-on-error' '--single-transaction'; do + case "${joined}" in + *" ${expected} "*) ;; + *) echo "missing restore flag: ${expected}" >&2; exit 1 ;; + esac +done +case "${joined}" in + *' --dbname=service=brio_app_restore_test '*) grep -Fq 'fake custom dump for brio_staging' "${last_argument}" ;; + *' --dbname=service=brio_keycloak_restore_test '*) grep -Fq 'fake custom dump for keycloak_brio_staging' "${last_argument}" ;; + *) echo "unexpected restore service" >&2; exit 1 ;; +esac +printf '%s\n' "$*" >> "${RESTORE_LOG}" +EOF + +cat > "${work_dir}/bin/psql" <<'EOF' +#!/bin/sh +set -eu +case " $* " in + *' service=brio_app_restore_test '*) database=brio_app_restore_test ;; + *' service=brio_keycloak_restore_test '*) database=brio_keycloak_restore_test ;; + *) echo "unexpected psql service" >&2; exit 1 ;; +esac +case " $* " in + *'SELECT current_database();'*) printf '%s\n' "${FAKE_RESTORE_DATABASE:-${database}}" ;; + *) printf 't\n' ;; +esac +EOF +chmod 0700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" "${work_dir}/bin/psql" + +create_bundle() { + local database=$1 + local root=$2 + local backup_user + case "${database}" in + brio_staging) backup_user=brio_staging_backup ;; + keycloak_brio_staging) backup_user=keycloak_brio_staging_backup ;; + *) return 1 ;; + esac + PATH="${work_dir}/bin:${PATH}" \ + BRIO_BACKUP_DATABASE="${database}" \ + BRIO_BACKUP_ROOT="${root}" \ + BRIO_BACKUP_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_BACKUP_RETENTION_DAYS=35 \ + POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ + PGHOST=makepad-postgres \ + PGPORT=5432 \ + PGUSER="${backup_user}" \ + PGSSLMODE=verify-full \ + PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ + sh "${repo_root}/scripts/run-brio-encrypted-backup.sh" >/dev/null +} + +create_bundle brio_staging "${work_dir}/backups/app" +create_bundle keycloak_brio_staging "${work_dir}/backups/keycloak" +app_backup=${work_dir}/backups/app/$(readlink "${work_dir}/backups/app/latest") +keycloak_backup=${work_dir}/backups/keycloak/$(readlink "${work_dir}/backups/keycloak/latest") + +if PATH="${work_dir}/bin:${PATH}" \ + PGSERVICEFILE="${work_dir}/pg_service.conf" \ + BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ + BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ + BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ + BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ + BRIO_RESTORE_CONFIRM=wrong-confirmation \ + bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}"; then + echo "Restore unexpectedly accepted an invalid destructive confirmation." >&2 + exit 1 +fi + +if PATH="${work_dir}/bin:${PATH}" \ + FAKE_RESTORE_DATABASE=brio_staging \ + PGSERVICEFILE="${work_dir}/pg_service.conf" \ + BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ + BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ + BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ + BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ + BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ + BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets \ + bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}"; then + echo "Restore unexpectedly accepted a production-shaped database target." >&2 + exit 1 +fi + +export RESTORE_LOG=${work_dir}/restore.log +PATH="${work_dir}/bin:${PATH}" \ +PGSERVICEFILE="${work_dir}/pg_service.conf" \ +BRIO_APP_RESTORE_SERVICE=brio_app_restore_test \ +BRIO_KEYCLOAK_RESTORE_SERVICE=brio_keycloak_restore_test \ +BRIO_RESTORE_RECIPIENT_CERT="${work_dir}/recipient.crt" \ +BRIO_RESTORE_RECIPIENT_KEY="${work_dir}/recipient.key" \ +BRIO_RESTORE_TEMP_ROOT="${work_dir}/restore-tmp" \ +BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets \ +bash "${repo_root}/scripts/verify-brio-encrypted-restore.sh" "${app_backup}" "${keycloak_backup}" + +test "$(wc -l < "${RESTORE_LOG}" | tr -d ' ')" = "2" +test -z "$(find "${work_dir}/restore-tmp" -mindepth 1 -print -quit)" + +echo "Brio encrypted PostgreSQL restore contract passed." diff --git a/scripts/test-brio-release-evidence.sh b/scripts/test-brio-release-evidence.sh new file mode 100755 index 0000000..126740a --- /dev/null +++ b/scripts/test-brio-release-evidence.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +validator="${script_dir}/verify-brio-release-evidence.py" +fixture=$(mktemp -d /tmp/postgres-brio-release-evidence-XXXXXX) +cleanup() { + [[ "${fixture}" =~ ^/tmp/postgres-brio-release-evidence-[A-Za-z0-9]+$ ]] || return 1 + find "${fixture}" -depth -delete +} +trap cleanup EXIT + +pg_run=123 +pg_attempt=2 +pg_sha=1111111111111111111111111111111111111111 +kc_run=456 +kc_attempt=1 +kc_sha=2222222222222222222222222222222222222222 + +python3 - "${fixture}" <<'PY' +import json +import pathlib +import stat +import sys +import zipfile + +root = pathlib.Path(sys.argv[1]) +pg_sha = "1" * 40 +kc_sha = "2" * 40 + +def dump(name, value): + (root / name).write_text(json.dumps(value, separators=(",", ":")), encoding="utf-8") + +def archive(name, entry, value, *, mode=None, extra=None): + with zipfile.ZipFile(root / name, "w", compression=zipfile.ZIP_DEFLATED) as target: + info = zipfile.ZipInfo(entry) + if mode is not None: + info.create_system = 3 + info.external_attr = mode << 16 + target.writestr(info, json.dumps(value, separators=(",", ":"))) + if extra: + target.writestr(extra, "unexpected") + +pg_evidence = { + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": 123, + "postgres_run_attempt": 2, + "postgres_head_sha": pg_sha, + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", +} +attestation = { + "schema": "makepad.brio-db-path-attestation.v1", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/deploy-brio-identity-db.yml", + "postgres_run_id": 123, + "postgres_run_attempt": 2, + "postgres_head_sha": pg_sha, + "keycloak_repository": "Makepad-fr/keycloak", + "keycloak_workflow": ".github/workflows/verify-brio-database.yml", + "keycloak_verifier_run_id": 456, + "keycloak_verifier_run_attempt": 1, + "keycloak_release_sha": kc_sha, + "probe": "brio-db-path-ok", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", +} +dump("pg-run.json", { + "id": 123, "run_attempt": 2, "name": "Deploy Brio Identity Database", + "path": ".github/workflows/deploy-brio-identity-db.yml", "event": "workflow_dispatch", + "head_branch": "main", "head_sha": pg_sha, "status": "completed", "conclusion": "success", + "repository": {"full_name": "Makepad-fr/postgres"}, +}) +dump("pg-workflow.json", { + "name": "Deploy Brio Identity Database", + "path": ".github/workflows/deploy-brio-identity-db.yml", "state": "active", +}) +dump("pg-artifacts.json", {"total_count": 1, "artifacts": [{ + "id": 789, "name": "brio-db-deployment-evidence-123-2", "expired": False, "size_in_bytes": 1024, +}]}) +archive("pg.zip", "brio-db-deployment-evidence.json", pg_evidence) +dump("kc-main.json", {"object": {"sha": kc_sha}}) +kc_run_value = { + "id": 456, "run_attempt": 1, "name": "Verify Brio Identity Database Path", + "display_title": f"Verify Brio DB path for PostgreSQL run 123/2 at Keycloak {kc_sha}", + "path": ".github/workflows/verify-brio-database.yml", "event": "workflow_dispatch", + "head_branch": "main", "head_sha": kc_sha, "status": "completed", "conclusion": "success", + "created_at": "2026-09-05T10:00:05Z", "repository": {"full_name": "Makepad-fr/keycloak"}, +} +dump("kc-run.json", kc_run_value) +dump("kc-runs.json", {"total_count": 1, "workflow_runs": [kc_run_value]}) +dump("kc-artifacts.json", {"total_count": 1, "artifacts": [{ + "id": 987, "name": "brio-db-path-attestation-456-1", "expired": False, "size_in_bytes": 1024, +}]}) +archive("kc.zip", "brio-db-path-attestation.json", attestation) + +bad = dict(pg_evidence); bad["schema"] = "wrong"; archive("wrong-schema.zip", "brio-db-deployment-evidence.json", bad) +bad = dict(pg_evidence); bad["extra"] = True; archive("extra-field.zip", "brio-db-deployment-evidence.json", bad) +archive("extra-entry.zip", "brio-db-deployment-evidence.json", pg_evidence, extra="extra") +archive("symlink-entry.zip", "brio-db-deployment-evidence.json", pg_evidence, mode=stat.S_IFLNK | 0o777) +with zipfile.ZipFile(root / "oversized-entry.zip", "w", compression=zipfile.ZIP_STORED) as target: + target.writestr("brio-db-deployment-evidence.json", b"x" * 65537) +bad = dict(attestation); bad["postgres_run_id"] = 999; archive("wrong-binding.zip", "brio-db-path-attestation.json", bad) +PY + +expect_failure() { + local label=$1 + shift + if "$@" >"${fixture}/${label}.out" 2>&1; then + echo "Expected ${label} to fail closed." >&2 + exit 1 + fi +} + +[[ $(python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/pg-artifacts.json" "${pg_run}" "${pg_attempt}") == "789 ${pg_sha}" ]] +python3 "${validator}" postgres-evidence "${fixture}/pg.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +[[ $(python3 "${validator}" keycloak-main "${fixture}/kc-main.json") == "${kc_sha}" ]] +[[ $(python3 "${validator}" verifier-run-select "${fixture}/kc-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z) == "${kc_run}" ]] +python3 "${validator}" verifier-run "${fixture}/kc-run.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" "${kc_run}" +[[ $(python3 "${validator}" attestation-artifact "${fixture}/kc-artifacts.json" "${kc_run}" "${kc_attempt}") == 987 ]] +python3 "${validator}" attestation "${fixture}/kc.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" "${kc_run}" "${kc_attempt}" "${kc_sha}" + +python3 - "${fixture}/pg-run.json" "${fixture}/in-progress.json" <<'PY' +import json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")); value["status"]="in_progress"; value["conclusion"]=None +json.dump(value, open(sys.argv[2], "w", encoding="utf-8")) +PY +expect_failure in-progress python3 "${validator}" postgres-run "${fixture}/in-progress.json" "${fixture}/pg-workflow.json" "${fixture}/pg-artifacts.json" "${pg_run}" "${pg_attempt}" + +python3 - "${fixture}/pg-artifacts.json" "${fixture}/duplicate-artifacts.json" "${fixture}/oversized-artifact.json" "${fixture}/truncated-artifacts.json" <<'PY' +import copy, json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")) +duplicate=copy.deepcopy(value); duplicate["artifacts"].append(copy.deepcopy(duplicate["artifacts"][0])); duplicate["total_count"]=2; json.dump(duplicate, open(sys.argv[2], "w", encoding="utf-8")) +oversized=copy.deepcopy(value); oversized["artifacts"][0]["size_in_bytes"]=131073; json.dump(oversized, open(sys.argv[3], "w", encoding="utf-8")) +truncated=copy.deepcopy(value); truncated["total_count"]=2; json.dump(truncated, open(sys.argv[4], "w", encoding="utf-8")) +PY +expect_failure duplicate-artifact python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/duplicate-artifacts.json" "${pg_run}" "${pg_attempt}" +expect_failure oversized-artifact python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/oversized-artifact.json" "${pg_run}" "${pg_attempt}" +expect_failure truncated-artifact-page python3 "${validator}" postgres-run "${fixture}/pg-run.json" "${fixture}/pg-workflow.json" "${fixture}/truncated-artifacts.json" "${pg_run}" "${pg_attempt}" +expect_failure wrong-schema python3 "${validator}" postgres-evidence "${fixture}/wrong-schema.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure extra-field python3 "${validator}" postgres-evidence "${fixture}/extra-field.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure extra-entry python3 "${validator}" postgres-evidence "${fixture}/extra-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure symlink-entry python3 "${validator}" postgres-evidence "${fixture}/symlink-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure oversized-entry python3 "${validator}" postgres-evidence "${fixture}/oversized-entry.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +ln -s "${fixture}/pg.zip" "${fixture}/symlink-archive.zip" +expect_failure symlink-archive python3 "${validator}" postgres-evidence "${fixture}/symlink-archive.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" +expect_failure wrong-attestation-binding python3 "${validator}" attestation "${fixture}/wrong-binding.zip" "${pg_run}" "${pg_attempt}" "${pg_sha}" "${kc_run}" "${kc_attempt}" "${kc_sha}" + +python3 - "${fixture}/kc-runs.json" "${fixture}/duplicate-runs.json" <<'PY' +import copy, json, sys +value=json.load(open(sys.argv[1], encoding="utf-8")); value["workflow_runs"].append(copy.deepcopy(value["workflow_runs"][0])); value["total_count"] = 2 +json.dump(value, open(sys.argv[2], "w", encoding="utf-8")) +PY +expect_failure duplicate-verifier-run python3 "${validator}" verifier-run-select "${fixture}/duplicate-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z + +python3 - "${fixture}/kc-runs.json" "${fixture}/truncated-runs.json" "${fixture}/kc-artifacts.json" "${fixture}/truncated-kc-artifacts.json" <<'PY' +import json, sys +runs=json.load(open(sys.argv[1], encoding="utf-8")); runs["total_count"]=2; json.dump(runs, open(sys.argv[2], "w", encoding="utf-8")) +artifacts=json.load(open(sys.argv[3], encoding="utf-8")); artifacts["total_count"]=2; json.dump(artifacts, open(sys.argv[4], "w", encoding="utf-8")) +PY +expect_failure truncated-verifier-page python3 "${validator}" verifier-run-select "${fixture}/truncated-runs.json" "${pg_run}" "${pg_attempt}" "${kc_sha}" 2026-09-05T10:00:00Z +expect_failure truncated-keycloak-artifact-page python3 "${validator}" attestation-artifact "${fixture}/truncated-kc-artifacts.json" "${kc_run}" "${kc_attempt}" + +echo "Brio two-phase release evidence validation tests passed." diff --git a/scripts/test-github-environment-main-policy.py b/scripts/test-github-environment-main-policy.py new file mode 100755 index 0000000..c42eaac --- /dev/null +++ b/scripts/test-github-environment-main-policy.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Behavioral tests for exact-main GitHub environment reconciliation.""" + +from __future__ import annotations + +import copy +import runpy +from pathlib import Path + + +module = runpy.run_path( + str(Path(__file__).with_name("reconcile-github-environment-main-policy.py")), + run_name="postgres_environment_policy", +) +PolicyError = module["PolicyError"] +REQUIRED_ENVIRONMENTS = module["REQUIRED_ENVIRONMENTS"] +audit_environment = module["audit_environment"] +build_preserving_update = module["build_preserving_update"] +reconcile_environment = module["reconcile_environment"] + + +class FakeClient: + def __init__(self, environment, policies): + self.environment = copy.deepcopy(environment) + self.policies = copy.deepcopy(policies) + self.calls = [] + self.next_id = 100 + + def get_environment(self, environment): + self.calls.append(("get", environment)) + return copy.deepcopy(self.environment) + + def put_environment(self, environment, payload): + self.calls.append(("put", environment, copy.deepcopy(payload))) + self.environment["deployment_branch_policy"] = copy.deepcopy(payload["deployment_branch_policy"]) + + def list_policies(self, environment): + self.calls.append(("list", environment)) + return copy.deepcopy(self.policies) + + def create_main_policy(self, environment): + self.calls.append(("create", environment, "main", "branch")) + self.policies.append({"id": self.next_id, "name": "main", "type": "branch"}) + self.next_id += 1 + + def delete_policy(self, environment, policy_id): + self.calls.append(("delete", environment, policy_id)) + self.policies = [policy for policy in self.policies if policy["id"] != policy_id] + + +protected_environment = { + "deployment_branch_policy": {"protected_branches": True, "custom_branch_policies": False}, + "protection_rules": [ + {"type": "branch_policy"}, + {"type": "wait_timer", "wait_timer": 15}, + { + "type": "required_reviewers", + "prevent_self_review": True, + "reviewers": [{"type": "Team", "reviewer": {"id": 42}}], + }, + ], +} + +assert "production" in REQUIRED_ENVIRONMENTS +preserved = build_preserving_update(protected_environment) +assert preserved == { + "wait_timer": 15, + "prevent_self_review": True, + "reviewers": [{"type": "Team", "id": 42}], + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}, +} + +client = FakeClient(protected_environment, [{"id": 7, "name": "release/*", "type": "branch"}]) +try: + audit_environment(client, "production") +except PolicyError: + pass +else: + raise AssertionError("generic protected-branch policy was accepted") + +client.calls.clear() +reconcile_environment(client, "production") +audit_environment(client, "production") +assert client.policies == [{"id": 100, "name": "main", "type": "branch"}] +assert next(index for index, call in enumerate(client.calls) if call[0] == "create") < next( + index for index, call in enumerate(client.calls) if call[0] == "delete" +) +put_call = next(call for call in client.calls if call[0] == "put") +assert put_call[2]["reviewers"] == [{"type": "Team", "id": 42}] +assert put_call[2]["wait_timer"] == 15 + +client = FakeClient( + { + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}, + "protection_rules": [{"type": "branch_policy"}], + }, + [ + {"id": 9, "name": "main", "type": "branch"}, + {"id": 10, "name": "main", "type": "tag"}, + ], +) +try: + audit_environment(client, "production") +except PolicyError: + pass +else: + raise AssertionError("additional tag policy was accepted") + +print("GitHub environment exact-main policy tests passed.") diff --git a/scripts/test-keycloak-cohort-evidence.sh b/scripts/test-keycloak-cohort-evidence.sh new file mode 100755 index 0000000..850c241 --- /dev/null +++ b/scripts/test-keycloak-cohort-evidence.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +validator="${repo_root}/scripts/verify-keycloak-cohort-evidence.py" +work_dir=$(mktemp -d) +cleanup() { + find "${work_dir}" -mindepth 1 -delete + rmdir "${work_dir}" +} +trap cleanup EXIT + +head_sha=$(printf 'a%.0s' {1..40}) +release_sha=$(printf 'b%.0s' {1..40}) +canonical="${work_dir}/keycloak-cohort-restore-evidence.json" + +python3 - "${canonical}" "${head_sha}" "${release_sha}" <<'PY' +import hashlib +import json +import pathlib +import sys + +databases = { + "catwlk": "keycloak_catwlk", + "makepad": "keycloak_makepad", + "runtrace": "keycloak_runtrace", + "vestiaire": "keycloak_vestiaire", + "vif": "keycloak_vif", +} +payload = { + "schema": "makepad.keycloak-cohort-restore-evidence.v2", + "postgres_repository": "Makepad-fr/postgres", + "postgres_workflow": ".github/workflows/verify-keycloak-cohort-restores.yml", + "postgres_run_id": 101, + "postgres_run_attempt": 2, + "postgres_head_sha": sys.argv[2], + "postgres_ref": "refs/heads/main", + "keycloak_release_sha": sys.argv[3], + "keycloak_base_image": "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "catwlk_runtime_image_id": "sha256:" + "c" * 64, + "fingerprint_schema": "makepad.keycloak-config-fingerprint.v2", + "keycloak_upstream_version": "26.7.3", + "result": "restored-databases-compatible", + "instances": [], +} +categories = ["authentication", "clients", "components", "identity_providers", "realm", "required_actions", "roles"] +for index, (slug, database) in enumerate(sorted(databases.items())): + fingerprints = {category: format((index + 1) * 10 + offset, "064x") for offset, category in enumerate(categories)} + combined = hashlib.sha256("".join(f"{category}={fingerprints[category]}\n" for category in categories).encode()).hexdigest() + payload["instances"].append({ + "slug": slug, + "database": database, + "backup_sha256": format(index + 1, "064x"), + "runtime": "catwlk-custom-provider" if slug == "catwlk" else "keycloak-base", + "runtime_image": "sha256:" + "c" * 64 if slug == "catwlk" else "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "configuration_fingerprint": combined, + "configuration_fingerprints": fingerprints, + "restore": "passed", + "keycloak_startup": "passed", + "configuration_regression": "passed", + }) +pathlib.Path(sys.argv[1]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY + +python3 "${validator}" "${canonical}" --run-id 101 --run-attempt 2 \ + --head-sha "${head_sha}" --keycloak-release-sha "${release_sha}" + +expect_failure() { + local name=$1 expression=$2 candidate + candidate="${work_dir}/${name}.json" + python3 - "${canonical}" "${candidate}" "${expression}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +exec(sys.argv[3], {"payload": payload}) +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) +PY + if python3 "${validator}" "${candidate}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted ${name}." >&2 + exit 1 + fi +} + +expect_failure wrong-schema 'payload["schema"] = "makepad.keycloak-cohort-restore-evidence.v1"' +expect_failure extra-field 'payload["unexpected"] = True' +expect_failure extra-instance-field 'payload["instances"][0]["unexpected"] = True' +expect_failure missing-instance 'payload["instances"].pop()' +expect_failure duplicate-instance 'payload["instances"][-1] = payload["instances"][0]' +expect_failure unsorted-instance 'payload["instances"][0], payload["instances"][1] = payload["instances"][1], payload["instances"][0]' +expect_failure wrong-database 'payload["instances"][0]["database"] = "keycloak_vif"' +expect_failure bad-digest 'payload["instances"][0]["backup_sha256"] = "A" * 64' +expect_failure failed-restore 'payload["instances"][0]["restore"] = "failed"' +expect_failure wrong-image 'payload["keycloak_base_image"] = "dhi.io/keycloak:latest"' +expect_failure wrong-catwlk-runtime 'next(value for value in payload["instances"] if value["slug"] == "catwlk")["runtime"] = "keycloak-base"' +expect_failure missing-fingerprint-category 'payload["instances"][0]["configuration_fingerprints"].pop("roles")' +expect_failure tampered-fingerprint 'payload["instances"][0]["configuration_fingerprints"]["roles"] = "f" * 64' +expect_failure mutable-release 'payload["keycloak_release_sha"] = "main"' + +noncanonical="${work_dir}/noncanonical.json" +python3 - "${canonical}" "${noncanonical}" <<'PY' +import json +import pathlib +import sys +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +pathlib.Path(sys.argv[2]).write_text(json.dumps(payload, indent=2)) +PY +if python3 "${validator}" "${noncanonical}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted non-canonical JSON." >&2 + exit 1 +fi + +symlink="${work_dir}/symlink.json" +ln -s "${canonical}" "${symlink}" +if python3 "${validator}" "${symlink}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted a symlink." >&2 + exit 1 +fi + +oversized="${work_dir}/oversized.json" +dd if=/dev/zero of="${oversized}" bs=65537 count=1 status=none +if python3 "${validator}" "${oversized}" >/dev/null 2>&1; then + echo "Cohort evidence validator accepted an oversized file." >&2 + exit 1 +fi + +echo "Keycloak cohort evidence contract tests passed." diff --git a/scripts/test-keycloak-cohort-hardening.sh b/scripts/test-keycloak-cohort-hardening.sh new file mode 100755 index 0000000..bc2e999 --- /dev/null +++ b/scripts/test-keycloak-cohort-hardening.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +REPO_ROOT="${repo_root}" python3 - <<'PY' +import os +from pathlib import Path + +root=Path(os.environ["REPO_ROOT"]) +workflow=(root/".github/workflows/verify-keycloak-cohort-restores.yml").read_text() +dispatch=(root/"scripts/keycloak-cohort-capture-dispatch.sh").read_text() +installer=(root/"scripts/install-keycloak-cohort-capture-host.sh").read_text() +restore=(root/"scripts/restore-keycloak-cohort-backups.sh").read_text() +cleaner=(root/"scripts/clean-keycloak-cohort-resources.sh").read_text() +timer=(root/"host/systemd/makepad-keycloak-cohort-cleaner.timer").read_text() + +def require(condition, message): + if not condition: raise SystemExit(message) + +require("scp " not in workflow and "remote_script=" not in workflow, "cohort workflow must not copy or execute checked-out code on the database host") +probe=workflow.index('"probe ${helper_digest} ${cleaner_digest}"') +capture=workflow.index('"capture ${GITHUB_RUN_ID}') +require(probe < capture, "remote forced-command digest and TTL probe must precede capture") +for marker in ("SSH_ORIGINAL_COMMAND", "sha256sum", "systemctl is-enabled", "systemctl is-active", "--property=Result", "--property=ExecMainStatus", "probe)", "capture)", "fetch)", "cleanup)"): + require(marker in dispatch, f"forced-command dispatcher is missing {marker}") +require('sha256sum "${cleaner}"' in dispatch and "cleaner_digest" in workflow, "forced commands must bind the exact installed cleaner digest") +require('restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch"' in installer, "installer must bind the key to the forced command") +require("sshd -t" in installer and "DisableForwarding yes" in installer, "installer must validate and restrict sshd") +for marker in ("makepad.cleanup.contract", "makepad.cleanup.expires-epoch", "docker container ls -aq", "docker network ls -q"): + require(marker in cleaner, f"cohort resource cleaner is missing {marker}") +require("Persistent=true" in timer and "OnUnitActiveSec=15min" in timer, "cohort cleanup timer must survive downtime and recur") +for category in ("realm", "authentication", "roles", "clients", "identity_providers", "components", "required_actions"): + require(f"[{category}]" in restore, f"v2 fingerprint is missing {category}") +for table in ("realm_smtp_config", "authentication_execution", "role_attribute", "composite_role", "client_scope_role_mapping", "protocol_mapper_config", "identity_provider_config", "component_config", "required_action_provider"): + require(table in restore, f"v2 fingerprint is missing table {table}") +require("to_regclass" in restore and "to_jsonb(value)::text" in restore, "fingerprint must fail closed on schema drift and serialize deterministically") +require("POSTGRES_PASSWORD=${" not in restore and "KC_DB_PASSWORD=${" not in restore, "test secrets must not enter Docker Config.Env") +require("POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password" in restore and 'export KC_DB_PASSWORD="$(cat /run/secrets/postgres-password)"' in restore, "runtime passwords must originate in mounted files") +require("catwlk-custom-provider" in restore and "catwlk-keycloak-email-router.jar" in restore, "Catwlk restore must validate the checked-out custom runtime") +PY + +test_image=$(awk -F= '$1 == "BRIO_BACKUP_IMAGE" { print $2 }' "${repo_root}/envs/canary/.env.db") +[[ "${test_image}" == *@sha256:* ]] +docker run --rm --network none --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" "${test_image}" \ + bash /repo/scripts/fixtures/keycloak-cohort-cleaner-fixture.sh +docker run --rm --network none --security-opt no-new-privileges:true \ + --volume "${repo_root}:/repo:ro" "${test_image}" \ + bash /repo/scripts/fixtures/keycloak-cohort-dispatch-fixture.sh + +echo "Keycloak cohort capture, runtime, fingerprint, and cleanup hardening contracts passed." diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 5f7d504..bd5513d 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -60,21 +60,95 @@ sql = read_required_text(repo_root / "bootstrap/keycloak-new-instances.sql", "SQ runtrace_sql = read_required_text(repo_root / "bootstrap/runtrace-app.sql", "Runtrace app SQL bootstrap") keycloak_runtrace_sql = read_required_text(repo_root / "bootstrap/keycloak-runtrace-app.sql", "targeted Runtrace Keycloak SQL bootstrap") openpanel_sql = read_required_text(repo_root / "bootstrap/openpanel-app.sql", "OpenPanel app SQL bootstrap") +brio_sql = read_required_text(repo_root / "bootstrap/brio-staging-app.sql", "Brio staging app SQL bootstrap") +keycloak_brio_sql = read_required_text(repo_root / "bootstrap/keycloak-brio-staging.sql", "targeted Brio Keycloak SQL bootstrap") readme = read_required_text(repo_root / "README.md", "README") base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") host_compose = read_required_text(repo_root / "compose.host.yml", "host Compose file") runtrace_hba = read_required_text(repo_root / "config/runtrace-pg_hba.conf", "Runtrace HBA policy") +hba_records = [ + tuple(line.split()) + for line in runtrace_hba.splitlines() + if line.strip() and not line.lstrip().startswith("#") +] runtrace_backup = read_required_text(repo_root / "scripts/run-runtrace-backup.sh", "Runtrace backup script") runtrace_backup_loop = read_required_text(repo_root / "scripts/run-runtrace-backup-loop.sh", "Runtrace backup loop") runtrace_restore = read_required_text(repo_root / "scripts/verify-runtrace-restore.sh", "Runtrace restore verifier") runtrace_backup_test = read_required_text(repo_root / "scripts/test-runtrace-backup.sh", "Runtrace backup contract test") +brio_backup_path = repo_root / "scripts/run-brio-encrypted-backup.sh" +brio_backup_loop_path = repo_root / "scripts/run-brio-encrypted-backup-loop.sh" +brio_restore_path = repo_root / "scripts/verify-brio-encrypted-restore.sh" +brio_backup_test_path = repo_root / "scripts/test-brio-encrypted-backup.sh" +brio_restore_test_path = repo_root / "scripts/test-brio-encrypted-restore.sh" +brio_backup = read_required_text(brio_backup_path, "Brio encrypted backup script") +brio_backup_loop = read_required_text(brio_backup_loop_path, "Brio encrypted backup loop") +brio_restore = read_required_text(brio_restore_path, "Brio encrypted restore verifier") +brio_backup_test = read_required_text(brio_backup_test_path, "Brio encrypted backup contract test") +brio_restore_test = read_required_text(brio_restore_test_path, "Brio encrypted restore contract test") canary_compose = read_required_text(repo_root / "envs/canary/compose.yml", "canary Compose override") production_compose = read_required_text(repo_root / "envs/production/compose.yml", "production Compose override") canary_env = read_required_text(repo_root / "envs/canary/.env.db", "canary database environment") production_env = read_required_text(repo_root / "envs/production/.env.db", "production database environment") -manual_deploy = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") +manual_deploy_workflow = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") +remote_deploy_path = repo_root / "scripts/deploy-postgres-stack.sh" +remote_deploy = read_required_text(remote_deploy_path, "remote deploy script") +canary_deploy_path = repo_root / "scripts/deploy-brio-canary-postgres.sh" +canary_deploy = read_required_text(canary_deploy_path, "Brio canary deploy script") +identity_deploy_path = repo_root / "scripts/deploy-brio-identity-db-host.sh" +identity_deploy = read_required_text(identity_deploy_path, "Brio identity DB-VM deploy script") +identity_workflow = read_required_text(repo_root / ".github/workflows/deploy-brio-identity-db.yml", "Brio identity DB-VM workflow") +release_workflow = read_required_text(repo_root / ".github/workflows/release-brio-identity-db.yml", "Brio identity database release orchestrator") +cohort_workflow = read_required_text(repo_root / ".github/workflows/verify-keycloak-cohort-restores.yml", "Keycloak cohort restore workflow") +environment_policy_reconciler = read_required_text(repo_root / "scripts/reconcile-github-environment-main-policy.py", "GitHub environment policy reconciler") +environment_policy_test = read_required_text(repo_root / "scripts/test-github-environment-main-policy.py", "GitHub environment policy test") +release_evidence_validator = read_required_text(repo_root / "scripts/verify-brio-release-evidence.py", "Brio release evidence validator") +cohort_evidence_validator = read_required_text(repo_root / "scripts/verify-keycloak-cohort-evidence.py", "Keycloak cohort evidence validator") +cohort_capture = read_required_text(repo_root / "scripts/capture-keycloak-cohort-backups.sh", "Keycloak cohort backup capture") +cohort_restore = read_required_text(repo_root / "scripts/restore-keycloak-cohort-backups.sh", "Keycloak cohort restore verifier") +cohort_dispatch_path = repo_root / "scripts/keycloak-cohort-capture-dispatch.sh" +cohort_dispatch = read_required_text(cohort_dispatch_path, "Keycloak cohort forced-command dispatcher") +cohort_host_installer_path = repo_root / "scripts/install-keycloak-cohort-capture-host.sh" +cohort_host_installer = read_required_text(cohort_host_installer_path, "Keycloak cohort capture-host installer") +cohort_cleaner_path = repo_root / "scripts/clean-keycloak-cohort-resources.sh" +cohort_cleaner = read_required_text(cohort_cleaner_path, "Keycloak cohort Docker-resource cleaner") +cohort_cleaner_installer_path = repo_root / "scripts/install-keycloak-cohort-cleaner.sh" +cohort_cleaner_installer = read_required_text(cohort_cleaner_installer_path, "Keycloak cohort cleaner installer") +tmp_cleaner_path = repo_root / "scripts/ensure-brio-tmp-cleaner.sh" +tmp_cleaner = read_required_text(tmp_cleaner_path, "Brio abandoned-material cleaner") +deploy_guard_test = read_required_text(repo_root / "scripts/test-brio-deploy-guards.sh", "Brio deployment guard test") +deployment_failure_test_path = repo_root / "scripts/test-brio-deployment-failures.sh" +deployment_failure_fixture_path = repo_root / "scripts/fixtures/brio-deployment-failure-fixture.sh" +deployment_failure_test = read_required_text(deployment_failure_test_path, "Brio deployment failure-injection test") +deployment_failure_fixture = read_required_text(deployment_failure_fixture_path, "Brio deployment failure-injection fixture") +manual_deploy = manual_deploy_workflow + "\n" + remote_deploy + "\n" + canary_deploy +ci_workflow = read_required_text(repo_root / ".github/workflows/ci.yml", "CI workflow") +ci_runner = read_required_text(repo_root / "scripts/run-ci.sh", "CI suite runner") normalized_readme = re.sub(r"\s+", " ", readme) +for environment in ( + "canary", + "production", + "staging-brio-identity-db", + "release-brio-identity-db", + "keycloak-cohort-restore", +): + require(f'"{environment}"' in environment_policy_reconciler, f"Environment policy reconciler must include {environment}.") +for required in ( + '"protected_branches": False', + '"custom_branch_policies": True', + '{"name": "main", "type": "branch"}', + "MAX_POLICY_PAGES = 1000", + "build_preserving_update", + "audit_environment(client, environment)", +): + require(required in environment_policy_reconciler, f"Environment policy reconciler is missing: {required}") +require('assert "production" in REQUIRED_ENVIRONMENTS' in environment_policy_test, "Environment policy test must cover production explicitly.") +require("python3 scripts/test-github-environment-main-policy.py" in ci_runner, "CI must run the environment policy behavioral test.") +require( + "exactly one custom branch deployment policy whose type is `branch` and whose name is exactly `main`" in normalized_readme, + "README must require exact-main custom deployment policies.", +) + require("docker network create" not in sql, "SQL bootstrap must not manage Docker networks.") require("${POSTGRES_ADMIN_URL:?" in readme, "README bootstrap command must fail fast for POSTGRES_ADMIN_URL.") require("PostgreSQL superuser connection URI" in normalized_readme, "README must define POSTGRES_ADMIN_URL as a PostgreSQL superuser connection URI.") @@ -130,6 +204,16 @@ require("name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK}" in production_compose, "Produ for required in ("target: 5432", "published: 5432", "protocol: tcp", "mode: host"): require(required in production_compose, f"Production Compose must publish PostgreSQL for DB VM clients: {required}") require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy workflow must reject root SSH users.") +require(remote_deploy_path.stat().st_mode & 0o111, "Remote deploy script must be executable.") +for required in ( + 'cp scripts/deploy-postgres-stack.sh "${bundle_root}/scripts/deploy-postgres-stack.sh"', + 'remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + 'remote_script="${remote_bundle}/scripts/deploy-postgres-stack.sh"', +): + require(required in manual_deploy_workflow, f"Manual deploy workflow must bundle and invoke the remote deploy script: {required}") +require("<<'EOF'" not in manual_deploy_workflow, "Manual deploy workflow must not embed the oversized remote deployment heredoc.") +require("DEPLOY_BRIO_STAGING_DB_NETWORK must be makepad_brio_staging_db" in manual_deploy, "Manual deploy must reject a non-canonical Brio database network secret.") +require("Brio deployment bundle must use makepad_brio_staging_db" in manual_deploy, "Remote deploy must revalidate the canonical Brio database network.") require("postgres:16-alpine@sha256:" in base_compose, "Base Compose must pin PostgreSQL to an immutable digest.") require("pg_isready" in base_compose, "Base Compose must define a PostgreSQL healthcheck.") for required in ( @@ -157,6 +241,61 @@ for required in ( for database in ("runtrace", "keycloak_runtrace"): require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") require(re.search(rf"^hostssl\s+{database}\s+all\s+all\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must require TLS and SCRAM for {database}.") +for database, roles in ( + ("brio_staging", ("brio_staging_app", "brio_staging_backup")), + ("keycloak_brio_staging", ("keycloak_brio_staging_app", "keycloak_brio_staging_backup")), +): + require(re.search(rf"^hostnossl\s+{database}\s+all\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject plaintext access to {database}.") + for role in roles: + address = "127.0.0.1/32" if role == "keycloak_brio_staging_backup" else "all" + require(re.search(rf"^hostssl\s+{database}\s+{role}\s+{re.escape(address)}\s+scram-sha-256$", runtrace_hba, re.MULTILINE), f"HBA must allow TLS access to {database} for {role} from {address}.") + require(re.search(rf"^host\s+all\s+{role}\s+all\s+reject$", runtrace_hba, re.MULTILINE), f"HBA must reject {role} from every non-target database.") +for allow_record, reject_record in ( + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_app", "all", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_app", "all", "reject")), + (("hostssl", "keycloak_brio_staging", "keycloak_brio_staging_backup", "127.0.0.1/32", "scram-sha-256"), ("host", "all", "keycloak_brio_staging_backup", "all", "reject")), +): + require(hba_records.index(allow_record) < hba_records.index(reject_record), "Brio identity HBA allows must precede their target-wide role rejection.") +require("MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_canary_runtrace_hba_v3" in canary_env, "Canary must use the fresh immutable Brio HBA v3 object.") +require("MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=makepad_postgres_runtrace_hba_v3" in production_env, "Production must use the fresh immutable Brio HBA v3 object.") +require("runtrace_hba_v2" not in canary_env + production_env, "Active environments must never drift an already deployed HBA v2 object.") +require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose Brio's certificate-matching database alias.") +require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach Brio's isolated database network.") +require("ensure_internal_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate Brio's internal encrypted database network.") +for required in ( + "com.makepad.owner=Makepad-fr/postgres", + "com.makepad.environment=staging", + "com.makepad.instance=brio", + "com.makepad.purpose=database", +): + require(required in manual_deploy, f"Brio database network ownership policy is missing: {required}") +for content, role, database in ( + (brio_sql, "brio_staging_app", "brio_staging"), + (keycloak_brio_sql, "keycloak_brio_staging_app", "keycloak_brio_staging"), +): + require("NOBYPASSRLS" in content, f"{role} bootstrap must strip elevated role capabilities.") + require(f"REVOKE ALL ON DATABASE {database} FROM PUBLIC" in content, f"{database} must revoke default public database access.") + +for content, app_role, backup_role, database, password_variable in ( + (brio_sql, "brio_staging_app", "brio_staging_backup", "brio_staging", "brio_staging_backup_password"), + (keycloak_brio_sql, "keycloak_brio_staging_app", "keycloak_brio_staging_backup", "keycloak_brio_staging", "keycloak_brio_staging_backup_password"), +): + for required in ( + f"CREATE ROLE {backup_role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION", + f"ALTER ROLE {backup_role} LOGIN PASSWORD :'{password_variable}'", + f"ALTER ROLE {backup_role} NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 2", + f"GRANT CONNECT ON DATABASE {database} TO {backup_role}", + f"ALTER ROLE {backup_role} IN DATABASE {database} SET default_transaction_read_only TO on", + f"GRANT USAGE ON SCHEMA public TO {backup_role}", + f"GRANT SELECT ON ALL TABLES IN SCHEMA public TO {backup_role}", + f"GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO {backup_role}", + f"ALTER DEFAULT PRIVILEGES FOR ROLE {app_role} IN SCHEMA public GRANT SELECT ON TABLES TO {backup_role}", + f"ALTER DEFAULT PRIVILEGES FOR ROLE {app_role} IN SCHEMA public GRANT SELECT ON SEQUENCES TO {backup_role}", + ): + require(required in content, f"{database} backup bootstrap is missing: {required}") + require(f"NULLIF(btrim(:'{password_variable}'), '')" in content, f"{database} backup bootstrap must reject empty passwords.") + require("inet_client_addr() IS NULL" in content, f"{database} bootstrap must allow a local Unix-domain socket.") + require("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()" in content, f"{database} bootstrap must reject remote plaintext administrator sessions.") + require(r"\quit 1" not in content and "SELECT 1 / 0;" in content, f"{database} bootstrap failure branches must terminate psql with a non-zero status.") for label, content in (("canary", canary_env), ("production", production_env)): require("POSTGRES_PASSWORD=" not in content, f"{label} database environment must not contain POSTGRES_PASSWORD.") require("POSTGRES_IMAGE=postgres:16-alpine@sha256:" in content, f"{label} database environment must pin POSTGRES_IMAGE.") @@ -169,7 +308,8 @@ for label, content in (("canary", canary_compose), ("production", production_com require("/run/secrets/postgres_superuser_password:ro" in content, f"{label} Compose must mount the superuser password file read-only.") require("resources:" in content and "limits:" in content and "reservations:" in content, f"{label} Compose must set resource limits and reservations.") require("ensure_encrypted_overlay_network" in manual_deploy, "Manual deploy must validate encrypted database overlay networks.") -require("--opt encrypted" in manual_deploy, "Manual deploy must create database overlay networks with encryption.") +require(len(re.findall(r"docker network create[^\n]+--opt encrypted=true", manual_deploy)) == 2, "Manual deploy must explicitly create both database overlay network variants with encrypted=true.") +require(re.search(r"--opt\s+encrypted(?:\s|$)", manual_deploy) is None, "Manual deploy must not use Docker's valueless encrypted option.") require("postgres_root_password_file" in manual_deploy, "Manual deploy must load the PostgreSQL superuser password from the host file.") require('docker config inspect "${postgres_tls_cert_config}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS certificate config.") require('docker secret inspect "${postgres_tls_key_secret}"' in manual_deploy, "Manual deploy must validate the PostgreSQL TLS private-key secret.") @@ -178,7 +318,7 @@ require('deployed_hba_sha256' in manual_deploy, "Manual deploy must reject Runtr require("config/runtrace-pg_hba.conf" in manual_deploy, "Manual deploy must include the Runtrace HBA policy.") require("-e PGPASSWORD=" not in manual_deploy, "Manual deploy must not expose the PostgreSQL superuser password as a container environment argument.") require("POSTGRES_PASSWORD_FILE" in normalized_readme, "README must document file-based PostgreSQL bootstrap credentials.") -require("--opt encrypted" in normalized_readme, "README must document encrypted database overlays.") +require("--opt encrypted=true" in normalized_readme, "README must document the explicit encrypted=true database overlay option.") require("sslmode=verify-full" in normalized_readme, "README must document certificate-verified Runtrace database connections.") require("docker secret create makepad_postgres_tls_key_v1" in normalized_readme, "README must document private-key secret provisioning.") require("bash scripts/test-runtrace-tls-policy.sh" in normalized_readme, "README must document the container-level Runtrace TLS policy test.") @@ -237,13 +377,23 @@ require("DEPLOY_VIF_DB_NETWORK production environment secret" in manual_deploy, require('if [[ "${deploy_env}" == "production" ]]; then' in manual_deploy, "Manual deploy workflow must gate VIF setup to production.") require('if [[ "${vif_enabled}" != "1" ]]; then' in manual_deploy, "Manual deploy workflow must skip VIF provisioning outside production.") require("postgres_ready=0" in manual_deploy, "Manual deploy workflow must track Postgres readiness.") +require("wait_for_service_convergence" in manual_deploy, "Manual deploy must wait for exact-image Swarm task convergence.") +require("docker service ps --no-trunc --filter desired-state=running" in manual_deploy, "Manual deploy must inspect running task images rather than stale replica counts.") +require('wait_for_service_convergence "${stack_name}_postgres" "${postgres_image}"' in manual_deploy, "Manual deploy must converge PostgreSQL before probing it.") +require('wait_for_service_convergence "${stack_name}_brio_staging_backup" "${brio_backup_image}"' in manual_deploy, "Canary deploy must converge the Brio application backup task.") +require("keycloak_brio_staging_backup" not in production_compose, "The Brio identity backup must never be routed through the production Swarm override.") require("Postgres did not become reachable via makepad-postgres-vif" in manual_deploy, "Manual deploy workflow must fail clearly when VIF readiness times out.") require( - not re.search(r"\S\\gexec", manual_deploy), - "Manual deploy workflow must separate every VIF provisioning \\gexec command from SQL text by whitespace.", + not re.search(r"\S\\gexec", remote_deploy), + "VIF bootstrap must separate every \\gexec command from SQL text by whitespace.", ) -require("ALTER ROLE %I LOGIN PASSWORD %L" in manual_deploy, "Manual deploy workflow must always refresh the VIF role password.") -require("ALTER DATABASE %I OWNER TO %I" in manual_deploy, "Manual deploy workflow must repair VIF database ownership drift.") +require("ALTER ROLE %I LOGIN PASSWORD %L" in remote_deploy, "VIF bootstrap must always refresh the VIF role password.") +require("ALTER DATABASE %I OWNER TO %I" in remote_deploy, "VIF bootstrap must repair VIF database ownership drift.") +require("\\getenv vif_password VIF_PASSWORD" in remote_deploy, "VIF bootstrap must read its password from the mounted-file environment only.") +require("MAKEPAD_POSTGRES_VIF_DB_PASSWORD" not in manual_deploy_workflow + remote_deploy, "VIF password must never be persisted in the deployment environment file.") +require('-v vif_password=' not in remote_deploy, "VIF password must never be placed in psql command arguments.") +for required in ("postgres-brio-vif-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", "vif-db-password", "--interactive"): + require(required in manual_deploy, f"VIF deployment is missing file-only credential control: {required}") require( sql.count("DO $$") == len(expected_instances), "SQL bootstrap must use one DO block for each expected role.", @@ -316,4 +466,531 @@ require("NULLIF(btrim(:'openpanel_app_password'), '')" in openpanel_sql, "OpenPa require("openpanel_app" in normalized_readme, "README must document the OpenPanel app role.") require("postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable" in readme, "README must document the OpenPanel DB VM host connection URI.") require("${OPENPANEL_DB_PASSWORD:?" in readme, "README bootstrap command must fail fast for OPENPANEL_DB_PASSWORD.") + +for expected in ( + "brio_staging_app", + "brio_staging", + "brio_staging_app_password", + "pg_advisory_lock", + "pg_advisory_unlock", + "CREATE DATABASE brio_staging OWNER brio_staging_app", + "ALTER DATABASE brio_staging OWNER TO brio_staging_app", +): + require(expected in brio_sql, f"Brio staging bootstrap is missing {expected}.") +require("PostgreSQL superuser connection" in brio_sql, "Brio staging bootstrap must document its superuser requirement.") +require("NULLIF(btrim(:'brio_staging_app_password'), '')" in brio_sql, "Brio staging bootstrap must reject empty passwords.") +require("NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION" in brio_sql, "Brio staging role must be least privilege.") +require("brio_staging_app" in normalized_readme, "README must document the Brio staging app role.") +require("keycloak_brio_staging_app" in normalized_readme, "README must document the Brio staging Keycloak role.") +require("${BRIO_STAGING_DB_PASSWORD:?" in readme, "README must fail fast for BRIO_STAGING_DB_PASSWORD.") +require("${KEYCLOAK_BRIO_STAGING_DB_PASSWORD:?" in readme, "README must fail fast for KEYCLOAK_BRIO_STAGING_DB_PASSWORD.") +require("${BRIO_STAGING_BACKUP_DB_PASSWORD:?" in readme, "README must fail fast for BRIO_STAGING_BACKUP_DB_PASSWORD.") +require("${KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD:?" in readme, "README must fail fast for KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD.") +require("refuse a remote plaintext administrator session" in normalized_readme, "README must document the Brio bootstrap transport guard.") +require("bootstrap/keycloak-brio-staging.sql" in readme, "README must document the targeted Brio Keycloak bootstrap.") +require("keycloak_brio_staging_app_password" not in sql, "The shared Keycloak bootstrap must not rotate the Brio staging credential.") +require("MAKEPAD_POSTGRES_BRIO_STAGING_DB_NETWORK" in canary_compose, "Canary Compose must attach the isolated Brio staging DB network.") +require("makepad-postgres-brio-staging" in canary_compose, "Canary Compose must expose the certificate-matching Brio DB alias.") +backup_image = "postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825" +require(f"BRIO_BACKUP_IMAGE={backup_image}" in canary_env, "Canary must pin the exact Brio backup image.") +require(f"BRIO_BACKUP_IMAGE={backup_image}" in production_env, "Production must pin the exact Brio backup image.") +for config_name in ("brio_encrypted_backup_script", "brio_encrypted_backup_loop_script"): + require(config_name in base_compose, f"Base Compose is missing Brio backup config {config_name}.") + +require(" brio_staging_backup:" in canary_compose, "Canary Compose must run the Brio application backup service.") +canary_backup_service = canary_compose.split(" brio_staging_backup:", 1)[1].split("\nnetworks:", 1)[0] +for required in ( + "BRIO_BACKUP_DATABASE: brio_staging", + "PGHOST: makepad-postgres-brio-staging", + "PGUSER: brio_staging_backup", + "PGSSLMODE: verify-full", + "BRIO_BACKUP_RETENTION_DAYS", + "BRIO_BACKUP_RECIPIENT_CERT", + "user: \"999:999\"", + "read_only: true", + "no-new-privileges:true", + "- brio_staging", +): + require(required in canary_backup_service, f"Canary Brio backup service is missing: {required}") +require("- db" not in canary_backup_service, "Canary Brio backup must attach only to Brio's isolated database network.") + +require("BRIO_RESTORE_RECIPIENT_KEY" not in canary_compose + production_compose + host_compose, "Backup services must never mount the Brio recovery private key.") +for required in ( + "keycloak_brio_staging_backup:", + "BRIO_BACKUP_DATABASE: keycloak_brio_staging", + "MAKEPAD_POSTGRES_BRIO_IDENTITY_BACKUP_DB_HOST", + "PGUSER: keycloak_brio_staging_backup", + "PGSSLMODE: verify-full", +): + require(required in host_compose, f"Standalone DB-VM Compose is missing Brio identity backup control: {required}") + +for path in (brio_backup_path, brio_backup_loop_path, brio_restore_path, brio_backup_test_path, brio_restore_test_path): + require(os.access(path, os.X_OK), f"Brio backup/restore script must be executable: {path}") +for required in ( + "brio_staging) expected_pg_user=brio_staging_backup", + "keycloak_brio_staging) expected_pg_user=keycloak_brio_staging_backup", + "BRIO_BACKUP_RETENTION_DAYS:-35", + "Brio backup retention must remain exactly 35 days", + "retention_find_days=$((retention_days - 1))", + "mkfifo", + "pg_dump", + "openssl cms", + "-aes-256-gcm", + ".dump.cms", + "PGSSLMODE=verify-full", + "PGUSER must identify the database-specific Brio backup role", + "mv \"${partial_dir}\" \"${final_dir}\"", + "sha256sum \"${encrypted_name}\" metadata.json", +): + require(required in brio_backup, f"Brio encrypted backup script is missing: {required}") +require("--file=" not in brio_backup, "Brio backup must stream pg_dump instead of writing a plaintext dump file.") +require("BRIO_RESTORE_RECIPIENT_KEY" not in brio_backup, "Brio backup service must not receive the recovery private key.") +require("PGUSER=postgres" not in canary_backup_service + host_compose, "Brio backup services must never run as the PostgreSQL superuser.") +require("healthcheck" in brio_backup_loop and "interval_seconds * 2" in brio_backup_loop, "Brio backup health check must enforce freshness.") +for required in ( + "replace-nonproduction-brio-restore-targets", + "BRIO_APP_RESTORE_SERVICE", + "BRIO_KEYCLOAK_RESTORE_SERVICE", + "BRIO_RESTORE_RECIPIENT_CERT", + "BRIO_RESTORE_RECIPIENT_KEY", + "BRIO_RESTORE_TEMP_ROOT", + "*_restore_test", + "SELECT current_database();", + "openssl cms", + "-decrypt", + "--single-transaction", + "--exit-on-error", + "public.schema_migrations", + "public.communities", + "public.realm", +): + require(required in brio_restore, f"Brio encrypted restore verifier is missing: {required}") +require("run-brio-encrypted-backup.sh" in brio_backup_test, "Brio backup contract test must execute the real backup script.") +require("verify-brio-encrypted-restore.sh" in brio_restore_test, "Brio restore contract test must execute the real restore verifier.") +for required in ("test-brio-bootstrap.sh", "test-brio-encrypted-backup.sh", "test-brio-encrypted-restore.sh"): + require(required in ci_runner, f"CI must run the Brio PostgreSQL contract: {required}") +for required in ( + "independently administered off-host storage", + "successful recorded restore of both databases remain external release gates", + "private key must never be copied to a database host", + "scripts/test-brio-encrypted-backup.sh", + "scripts/test-brio-encrypted-restore.sh", +): + require(required in normalized_readme, f"README is missing Brio backup/restore guidance: {required}") + +for required in ( + "-checkhost makepad-postgres-brio-staging", + "-checkend 604800", + "openssl verify -purpose sslserver", + "PGSSLMODE=verify-full", + "-h makepad-postgres-brio-staging", + "cp scripts/run-brio-encrypted-backup.sh", + "cp scripts/run-brio-encrypted-backup-loop.sh", + "brio_backup_directory_mode", + "brio_backup_password_mode", + "brio_backup_recipient_mode", + "uid 999 with mode 0700", + "uid 999 with mode 0400", + "openssl cms -encrypt", +): + require(required in manual_deploy, f"Manual deploy is missing Brio certificate/connection preflight marker: {required}") +for required in ( + "postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "postgres-deploy-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + 'UserKnownHostsFile=${known_hosts_file}', + "-F /dev/null", + "GlobalKnownHostsFile=/dev/null", + "IdentitiesOnly=yes", + "Remove job-scoped deployment material", + "if: always()", +): + require(required in manual_deploy_workflow, f"Self-hosted deploy workflow is missing job-scoped cleanup control: {required}") +for forbidden in ('${HOME}/.ssh', "$HOME/.ssh", "~/.ssh", "add-ssh-host-key-action"): + require(forbidden not in manual_deploy_workflow, f"Self-hosted deploy workflow must not persist SSH state via {forbidden}.") +for workflow_name, workflow_text in ( + ("CI", ci_workflow), + ("manual deploy", manual_deploy_workflow), + ("identity DB-VM deploy", identity_workflow), + ("identity database release", release_workflow), + ("Keycloak cohort restore", cohort_workflow), +): + checkout_ref = "uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5" + checkout_count = workflow_text.count(checkout_ref) + require(checkout_count > 0, f"{workflow_name} workflow must check out the repository.") + require( + workflow_text.count("persist-credentials: false") == checkout_count, + f"Every self-hosted checkout in the {workflow_name} workflow must disable persisted Git credentials.", + ) + +# Every Actions job uses the existing Makepad Linux runner. Pull-request jobs +# additionally reject forks before a runner is assigned. +workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorted( + (repo_root / ".github/workflows").glob("*.yaml") +) +require(workflow_paths, "At least one GitHub Actions workflow must exist.") +for workflow_path in workflow_paths: + workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") + runs_on_count = len(re.findall(r"(?m)^ runs-on:", workflow_text)) + existing_runner_count = workflow_text.count(" runs-on: [self-hosted, linux, x64, makepad]") + require(runs_on_count > 0, f"Workflow {workflow_path.name} must define at least one job runner.") + require( + runs_on_count == existing_runner_count, + f"Every job in {workflow_path.name} must use the existing Makepad Linux runner labels.", + ) + require( + not re.search(r"(?i)(ubuntu|windows|macos)-(latest|[0-9]+)", workflow_text), + f"Workflow {workflow_path.name} must not use a GitHub-hosted runner image.", + ) + +# Credential material is canonical in Proton Pass and may be mirrored only to +# the protected environment that consumes it. Validate each complete table row +# so a field cannot silently drift into a different environment or item. +credential_inventory = { + "Hetzner Database Server makepad": ( + ("canary", "production", "staging-brio-identity-db", "keycloak-cohort-restore"), + ( + "DEPLOY_SSH_HOST", "DEPLOY_SSH_PORT", "DEPLOY_SSH_USER", + "DEPLOY_SSH_PRIVATE_KEY", "DEPLOY_SSH_KNOWN_HOSTS", + "BRIO_IDENTITY_DB_DEPLOY_SSH_HOST", "BRIO_IDENTITY_DB_DEPLOY_SSH_PORT", + "BRIO_IDENTITY_DB_DEPLOY_SSH_USER", "BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY", + "BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS", "KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY", + "KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS", "KEYCLOAK_COHORT_DB_SSH_HOST", + "KEYCLOAK_COHORT_DB_SSH_PORT", "KEYCLOAK_COHORT_DB_SSH_USER", + ), + ), + "Brio Staging - PostgreSQL": ( + ("canary", "staging-brio-identity-db"), + ( + "POSTGRES_CANARY_SUPERUSER_PASSWORD", "BRIO_STAGING_DB_PASSWORD", + "BRIO_STAGING_BACKUP_DB_PASSWORD", "KEYCLOAK_BRIO_STAGING_DB_PASSWORD", + "KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD", + ), + ), + "Brio Staging - PKI and Backup Keys": ( + ("canary", "staging-brio-identity-db"), + ( + "POSTGRES_CA_PEM", "POSTGRES_SERVER_CERT_PEM", "POSTGRES_SERVER_KEY_PEM", + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + ), + ), + "PostgreSQL · Brio identity release orchestrator": ( + ("release-brio-identity-db",), + ("KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN",), + ), + "PostgreSQL · Keycloak cohort source reader": ( + ("keycloak-cohort-restore",), + ("KEYCLOAK_COHORT_SOURCE_TOKEN",), + ), + "Makepad Docker Hardened Images": ( + ("keycloak-cohort-restore",), + ("DOCKERHUB_USERNAME", "DOCKERHUB_PRO_PAT", "DHI_REGISTRY_USERNAME", "DHI_REGISTRY_PASSWORD"), + ), +} +readme_lines = readme.splitlines() +for item, (environments, fields) in credential_inventory.items(): + candidate_rows = [line for line in readme_lines if line.startswith("|") and f"`{item}`" in line] + require(candidate_rows, f"README credential inventory is missing canonical Proton item {item}.") + require( + any(all(value in row for value in (*environments, *fields)) for row in candidate_rows), + f"README must map every field for {item} to its exact protected GitHub environment.", + ) +require("pass-cli item view --item-title '' --field ''" in readme, "README must document stdin-only pass-cli credential synchronization.") +require("| gh secret set '' --env '' --repo 'Makepad-fr/postgres'" in normalized_readme, "README must mirror workflow secrets only into protected GitHub environments.") +for workflow_path in workflow_paths: + workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") + for field in set(re.findall(r"(?:secrets|vars)\.([A-Z][A-Z0-9_]*)", workflow_text)): + require(field in readme, f"Workflow field {field} in {workflow_path.name} is absent from the credential inventory.") +for policy in ( + "hostnossl brio_staging", + "hostnossl keycloak_brio_staging", + "hostssl brio_staging", + "hostssl keycloak_brio_staging", +): + require(policy in runtrace_hba, f"PostgreSQL HBA policy is missing {policy}.") + +for path in ( + canary_deploy_path, + identity_deploy_path, + tmp_cleaner_path, + repo_root / "scripts/test-brio-deploy-guards.sh", + deployment_failure_test_path, + deployment_failure_fixture_path, + repo_root / "scripts/capture-keycloak-cohort-backups.sh", + repo_root / "scripts/restore-keycloak-cohort-backups.sh", + repo_root / "scripts/test-keycloak-cohort-evidence.sh", + repo_root / "scripts/test-keycloak-cohort-hardening.sh", + cohort_dispatch_path, + cohort_host_installer_path, + cohort_cleaner_path, + cohort_cleaner_installer_path, +): + require(os.access(path, os.X_OK), f"Brio deployment script must be executable: {path}") + +for required in ( + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + "BRIO_STAGING_BACKUP_DB_PASSWORD", + "BRIO_STAGING_DB_PASSWORD", + "POSTGRES_CANARY_SUPERUSER_PASSWORD", + "POSTGRES_CA_PEM", + "POSTGRES_SERVER_CERT_PEM", + "POSTGRES_SERVER_KEY_PEM", + "Materialize job-scoped Brio canary inputs", + "postgres-brio-canary-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "install -d -m 0700", + "chmod 0600", + "bootstrap/brio-staging-app.sql", + "deploy-brio-canary-postgres.sh", + "Remove remote job-scoped deployment material", +): + require(required in manual_deploy_workflow, f"Canary workflow is missing secure Brio input/deploy control: {required}") +require("runs-on: [self-hosted, linux, x64, makepad]" in manual_deploy_workflow, "Manual deployment must use the existing Makepad Linux runner.") +require('[[ "${GITHUB_REF}" == "refs/heads/main" ]]' in manual_deploy_workflow, "Manual deployment must refuse unreviewed refs.") +require("pull_request:" in ci_workflow and "pull_request_target:" not in ci_workflow, "PR CI must use the native pull-request event.") +require("github.event.pull_request.head.repo.full_name == github.repository" in ci_workflow, "PR CI must reject forks.") +require("ref: ${{ github.event.pull_request.head.sha }}" in ci_workflow, "PR CI must check out the exact candidate head.") +require(ci_workflow.count("runs-on: [self-hosted, linux, x64, makepad]") == 2, "PR and protected-main CI must use the existing Makepad Linux runner.") +require("permissions:\n contents: read" in ci_workflow, "CI must keep default permissions read-only.") +require('"${RUNNER_TEMP}"/postgres-deploy-*|"${RUNNER_TEMP}"/postgres-brio-canary-runtime-*|"${RUNNER_TEMP}"/postgres-brio-vif-runtime-*' in manual_deploy_workflow, "Cleanup must allow only the exact job-scoped deployment directory prefixes.") +require("for cleanup_target in" in manual_deploy_workflow, "Manual workflow cleanup must use a narrowly named cleanup target variable.") +require("group: postgres-shared-swarm-target" in manual_deploy_workflow, "Canary and production must share one target-wide Swarm concurrency group.") +require("postgres-swarm-${{ inputs.environment }}" not in manual_deploy_workflow, "Swarm concurrency must not split by environment on the shared target.") +require("${REMOTE_DIR}/stack.yml" not in manual_deploy_workflow + remote_deploy, "Deployment must never write the shared remote stack.yml path.") +require('stack_file="${generated_dir}/stack-${stack_name}-${deploy_env}.yml"' in remote_deploy, "Generated stack configuration must stay inside the unique run bundle.") + +for required in ( + "postgres-server-cert.pem", + "postgres-server-key.pem", + "PostgreSQL TLS certificate and private key do not match", + "-checkhost makepad-postgres-brio-staging", + "prevalidate_swarm_config", + "content-sha256", + "prevalidate_swarm_secret", + "bootstrap/brio-staging-app.sql", + "\\getenv brio_staging_app_password", + "PGSSLMODE=verify-full", + "Plaintext access to brio_staging was unexpectedly accepted", + "Brio application role was unexpectedly accepted by a non-target database", + "show default_transaction_read_only", + "makepad-postgres-brio-staging", + "last-success.json", + "sha256sum --check --status", + "openssl cms -cmsout", +): + require(required in canary_deploy, f"Canary deployment orchestrator is missing: {required}") +require('-e PGPASSWORD=' not in canary_deploy, "Canary deployment must not put database passwords in Docker command arguments.") +shared_network_validation = canary_deploy.find('prevalidate_network "${db_network}" false') +incomplete_recovery = canary_deploy.find("recover_incomplete_journals", shared_network_validation) +database_journal = canary_deploy.find('run_db_transaction prepare "${journal_stage}"', incomplete_recovery) +require(-1 not in (shared_network_validation, incomplete_recovery, database_journal) and shared_network_validation < incomplete_recovery < database_journal, "The validated shared database network must precede recovery and first-deployment journal capture.") +for required in ( + "assert_no_symlink_components", + "candidate-stack.yml", + "docker stack config", + 'tar --numeric-owner --no-recursion -cpf "$stage/rollback/managed.tar"', + "rollback_canary", + "prior-service-spec-hashes.list", + "mv -fT \"$super_stage\"", + "rollback_armed=0", +): + require(required in canary_deploy, f"Canary atomic deployment contract is missing: {required}") + +for required in ( + "environment: staging-brio-identity-db", + 'refs/heads/main', + "restart-standalone-postgres-for-brio-staging", + "backup_restore_confirmed", + "BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY", + "BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS", + "BRIO_IDENTITY_DB_DEPLOY_SSH_HOST", + "BRIO_IDENTITY_DB_DEPLOY_SSH_USER", + "KEYCLOAK_BRIO_STAGING_DB_PASSWORD", + "KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD", + "BRIO_BACKUP_RECIPIENT_CERT_PEM", + "BRIO_IDENTITY_DB_HOSTNAME", + "BRIO_KEYCLOAK_DB_SOURCE_CIDR", + "deploy-brio-identity-db-host.sh", + "Remove remote job-scoped identity secrets", + "postgres-brio-identity-bundle-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + "ensure-brio-tmp-cleaner.sh", + "brio-db-deployment-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "brio-db-deployment-evidence.json", + "makepad.brio-db-deployment-evidence.v1", + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", + "runs-on: [self-hosted, linux, x64, makepad]", +): + require(required in identity_workflow, f"Standalone identity DB workflow is missing: {required}") +for required in ( + "environment: release-brio-identity-db", + "KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN", + "verify-brio-database.yml/dispatches", + "verify-brio-release-evidence.py postgres-run", + "verify-brio-release-evidence.py postgres-evidence", + "verify-brio-release-evidence.py verifier-run", + "verify-brio-release-evidence.py attestation", + "brio-db-path-attestation", + "fetch_complete_listing", + '--config -', + 'release_token=${RELEASE_ORCHESTRATOR_TOKEN}', + 'unset RELEASE_ORCHESTRATOR_TOKEN', +): + require(required in release_workflow + release_evidence_validator, f"Protected two-phase database release orchestrator is missing: {required}") +for forbidden in ("brio-db-path-attestation.json\" <<", "actions/upload-artifact"): + require(forbidden not in release_workflow, "The release orchestrator must never synthesize or republish Keycloak attestation evidence.") +for required in ( + "name: Verify Keycloak Cohort Restore Compatibility", + "workflow_dispatch:", + "keycloak_release_sha:", + "environment: keycloak-cohort-restore", + "KEYCLOAK_COHORT_SOURCE_TOKEN", + "repos/Makepad-fr/keycloak/git/ref/heads/main", + "keycloak-cohort-restore-evidence-${{ github.run_id }}-${{ github.run_attempt }}", + "keycloak-cohort-restore-evidence.json", + "makepad.keycloak-cohort-restore-evidence.v2", + "restored-databases-compatible", + "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f", + "KEYCLOAK_UPSTREAM_VERSION=26.7.3", + "restore-keycloak-cohort-backups.sh", + "verify-keycloak-cohort-evidence.py", +): + require(required in cohort_workflow + cohort_evidence_validator, f"Five-database Keycloak cohort producer is missing: {required}") +require("vars." not in cohort_workflow, "The cohort producer must not accept a mutable repository variable as release evidence.") +for slug, database in ( + ("catwlk", "keycloak_catwlk"), + ("makepad", "keycloak_makepad"), + ("runtrace", "keycloak_runtrace"), + ("vestiaire", "keycloak_vestiaire"), + ("vif", "keycloak_vif"), +): + require(slug in cohort_evidence_validator and database in cohort_capture + cohort_restore, f"Cohort contract is missing {slug}/{database}.") +for required in ("pg_dump", "--no-owner", "--no-privileges", "pg_restore --list", "postgres-postgres-1"): + require(required in cohort_capture, f"Live cohort capture is missing: {required}") +for required in ( + "pg_restore", "start-dev", "/health/ready", "realm_smtp_config", + "authentication_execution", "role_attribute", "composite_role", "client_scope_role_mapping", "protocol_mapper_config", + "identity_provider_config", "component_config", "required_action_provider", + "configuration_regression", "catwlk-custom-provider", "POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password", +): + require(required in cohort_restore, f"Disposable cohort restore/startup verifier is missing: {required}") +require("scp " not in cohort_workflow and "remote_script=" not in cohort_workflow, "Cohort workflow must not execute checked-out code on the database host.") +for required in ( + "SSH_ORIGINAL_COMMAND", "sha256sum", 'sha256sum "${cleaner}"', "systemctl is-enabled", "systemctl is-active", + "--property=Result", "--property=ExecMainStatus", + "probe)", "capture)", "fetch)", "cleanup)", +): + require(required in cohort_dispatch, f"Cohort forced-command dispatcher is missing: {required}") +require('restrict,command="/usr/local/libexec/makepad/keycloak-cohort-capture-dispatch"' in cohort_host_installer, "Capture key must be bound to the exact forced command.") +for required in ("makepad.cleanup.contract", "makepad.cleanup.expires-epoch", "docker container ls -aq", "docker network ls -q"): + require(required in cohort_cleaner, f"Cohort resource cleaner is missing: {required}") +require("install-keycloak-cohort-cleaner.sh" in cohort_host_installer and "makepad-keycloak-cohort-cleaner.timer" in cohort_cleaner_installer, "Capture host must install the persistent cohort resource cleaner.") +require("makepad.keycloak-config-fingerprint.v2" in cohort_evidence_validator, "Cohort evidence must bind the v2 fingerprint schema.") +require("test-keycloak-cohort-hardening.sh" in ci_runner, "CI must run the cohort hardening contract test.") +require("POSTGRES_HOST_COMPOSE_PROJECT" not in identity_workflow + readme, "The standalone Compose project must be fixed in code, not selected by a workflow variable.") + +for required in ( + "Swarm.LocalNodeState", + '[[ "${swarm_state}" == "inactive" ]]', + "/srv/makepad/postgres", + "compose_project=postgres", + "expected_container_name=postgres-postgres-1", + "com.docker.compose.project", + "com.docker.compose.service", + "com.docker.compose.oneoff", + "bind|/var/lib/makepad/postgres|true", + '"${network_mode}" == "host"', + "keycloak-db-source-cidr", + "-checkip", + "127.0.0.1/32", + "65.21.134.125", + "88.99.209.165/32", + "Failed to render ordered, exact source-restricted Keycloak Brio HBA rules", + "--force-recreate", + "--project-name", + "bootstrap/keycloak-brio-staging.sql", + "\\getenv keycloak_brio_staging_app_password", + "PGHOSTADDR=127.0.0.1", + "PGSSLMODE=verify-full", + "Plaintext Keycloak Brio database access was unexpectedly accepted", + "Keycloak Brio role was unexpectedly accepted by a non-target database", + "show default_transaction_read_only", + "last-success.json", + "sha256sum --check --status", + "openssl cms -cmsout", + "restore_snapshot", + "rollback_deployment", + "rollback_armed=1", + "trap handle_exit EXIT", + "trap 'exit 129' HUP", + "trap 'exit 130' INT", + "trap 'exit 143' TERM", + 'tar --numeric-owner -cpf "$stage/rollback/managed.tar"', + 'identity-backups.tar', + 'identity-backup-absent', + "up -d --remove-orphans --wait --force-recreate", + "preserve_recovery_evidence", + "postgres-recovery/brio-identity", + "RECOVERY_REQUIRED", + "recovery_id=${identifier}", +): + require(required in identity_deploy, f"Standalone identity DB orchestrator is missing: {required}") +require("docker stack" not in identity_deploy and "docker swarm" not in identity_deploy, "Standalone identity DB deployment must not invoke Swarm deployment commands.") +require('"${key_uid}:${key_gid}:${key_mode}" == "70:70:400"' in identity_deploy, "Standalone DB-VM preflight must preserve the exact live server-key owner/group/mode contract.") +require('-e PGPASSWORD=' not in identity_deploy, "Identity DB deployment must not put database passwords in Docker command arguments.") +snapshot_index = identity_deploy.find('tar --numeric-owner -cpf "$stage/rollback/managed.tar"') +arm_index = identity_deploy.find("rollback_armed=1", snapshot_index) +first_install_index = identity_deploy.find('install_host_path "${candidate_compose}"', arm_index) +fresh_backup_index = identity_deploy.find('[[ "${backup_verified}" == "1" ]]', first_install_index) +disarm_index = identity_deploy.find("rollback_armed=0", fresh_backup_index) +require(-1 not in (snapshot_index, arm_index, first_install_index, fresh_backup_index, disarm_index), "Standalone rollback boundary markers are incomplete.") +require(snapshot_index < arm_index < first_install_index < fresh_backup_index < disarm_index, "Rollback must arm after snapshot and disarm only after probes and fresh backup verification.") +for required in ( + "MAKEPAD_POSTGRES_TLS_CERT_HOST_PATH=", + "MAKEPAD_POSTGRES_TLS_KEY_HOST_PATH=", + "MAKEPAD_POSTGRES_RUNTRACE_HBA_HOST_PATH=", + "MAKEPAD_POSTGRES_BRIO_BACKUP_SCRIPT_HOST_PATH=", +): + require(required in production_env, f"Production host environment is missing explicit standalone input: {required}") +require("test-brio-deploy-guards.sh" in ci_runner, "CI must run the deployment guard contract test.") +require("test-brio-deployment-failures.sh" in ci_runner, "CI must run executable Brio deployment failure-injection tests.") +for required in ( + "after-managed-file-promotion", + "term-after-managed-file-promotion", + "after-stack-deploy", + "rollback-restore", + "rollback-recreate", + "RECOVERY_REQUIRED", + ".State.Restarting", + "unexpected cleaner command", + "cleaner-running-output", + "symlink component", +): + require(required in deployment_failure_fixture, f"Failure-injection fixture is missing behavioral case: {required}") +require("PGHOSTADDR: 127.0.0.1" in host_compose, "Standalone identity backup must use a deterministic local transport address while verifying the configured certificate host.") +for required in ( + "makepad-postgres-brio-tmp-cleaner", + "--restart unless-stopped", + "--read-only", + "--cap-drop ALL", + "--security-opt no-new-privileges", + "type=bind,src=/tmp,dst=/host-tmp", + "-name 'postgres-brio-*'", + "-mmin +180", + "sleep 900", + "RECOVERY_REQUIRED", + "observed_command", + "verify_running", + ".State.Restarting", +): + require(required in tmp_cleaner, f"Host TTL cleaner is missing its restricted contract: {required}") +for workflow in (manual_deploy_workflow, identity_workflow): + require("ensure-brio-tmp-cleaner.sh" in workflow, "Every Brio deployment target must install the host-enforced TTL cleaner.") + require(workflow.find("ensure-brio-tmp-cleaner.sh") < workflow.find('scp "${scp_opts[@]}" "${runtime_dir}'), "TTL cleaner must be installed before job secrets are transferred.") +for required in ( + "Verify Brio Identity Database Path", + "Verify Brio DB path for PostgreSQL run ", + "brio-db-path-ok", + "65.21.134.125", + "88.99.209.165", + "no Keycloak database credential is granted to the PostgreSQL runner", +): + require(required in normalized_readme, f"README is missing the Keycloak-origin database release gate: {required}") PY diff --git a/scripts/verify-brio-encrypted-restore.sh b/scripts/verify-brio-encrypted-restore.sh new file mode 100755 index 0000000..96f8fb9 --- /dev/null +++ b/scripts/verify-brio-encrypted-restore.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +app_backup_dir=$1 +keycloak_backup_dir=$2 +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a protected libpq service file}" +: "${BRIO_APP_RESTORE_SERVICE:?BRIO_APP_RESTORE_SERVICE must name an explicit non-production database}" +: "${BRIO_KEYCLOAK_RESTORE_SERVICE:?BRIO_KEYCLOAK_RESTORE_SERVICE must name an explicit non-production database}" +: "${BRIO_RESTORE_RECIPIENT_CERT:?BRIO_RESTORE_RECIPIENT_CERT must identify the external recipient certificate}" +: "${BRIO_RESTORE_RECIPIENT_KEY:?BRIO_RESTORE_RECIPIENT_KEY must identify the external recipient private key}" +: "${BRIO_RESTORE_TEMP_ROOT:?BRIO_RESTORE_TEMP_ROOT must identify a mode-0700 temporary storage directory}" +: "${BRIO_RESTORE_CONFIRM:?set BRIO_RESTORE_CONFIRM=replace-nonproduction-brio-restore-targets}" + +if [[ "${BRIO_RESTORE_CONFIRM}" != "replace-nonproduction-brio-restore-targets" ]]; then + echo "Brio restore verification requires explicit non-production replacement confirmation." >&2 + exit 1 +fi +if [[ "${BRIO_APP_RESTORE_SERVICE}" == "${BRIO_KEYCLOAK_RESTORE_SERVICE}" ]]; then + echo "Brio application and Keycloak restore services must be different." >&2 + exit 1 +fi +for service_name in "${BRIO_APP_RESTORE_SERVICE}" "${BRIO_KEYCLOAK_RESTORE_SERVICE}"; do + if [[ ! "${service_name}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "Restore service names contain unsupported characters." >&2 + exit 1 + fi + if [[ "${service_name}" != *_restore_test ]]; then + echo "Restore service names must end in _restore_test." >&2 + exit 1 + fi +done +for command_name in openssl sha256sum pg_restore psql mktemp; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "Missing required restore command: ${command_name}" >&2 + exit 1 + fi +done + +validate_protected_file() { + local path=$1 + local label=$2 + if [[ ! -s "${path}" || -L "${path}" ]]; then + echo "${label} must be a non-empty, non-symlink file." >&2 + exit 1 + fi + local mode + mode=$(stat -c '%a' "${path}" 2>/dev/null || stat -f '%Lp' "${path}") + if (( (8#${mode} & 8#022) != 0 )); then + echo "${label} must not be group- or world-writable." >&2 + exit 1 + fi +} + +validate_protected_file "${PGSERVICEFILE}" "PGSERVICEFILE" +validate_protected_file "${BRIO_RESTORE_RECIPIENT_CERT}" "Brio restore recipient certificate" +validate_protected_file "${BRIO_RESTORE_RECIPIENT_KEY}" "Brio restore recipient private key" +service_file_mode=$(stat -c '%a' "${PGSERVICEFILE}" 2>/dev/null || stat -f '%Lp' "${PGSERVICEFILE}") +if (( (8#${service_file_mode} & 8#077) != 0 )); then + echo "PGSERVICEFILE must not be accessible to group or other users." >&2 + exit 1 +fi +key_mode=$(stat -c '%a' "${BRIO_RESTORE_RECIPIENT_KEY}" 2>/dev/null || stat -f '%Lp' "${BRIO_RESTORE_RECIPIENT_KEY}") +if (( (8#${key_mode} & 8#077) != 0 )); then + echo "Brio restore recipient private key must not be accessible to group or other users." >&2 + exit 1 +fi +if [[ ! -d "${BRIO_RESTORE_TEMP_ROOT}" || -L "${BRIO_RESTORE_TEMP_ROOT}" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must be a pre-provisioned non-symlink directory." >&2 + exit 1 +fi +temp_mode=$(stat -c '%a' "${BRIO_RESTORE_TEMP_ROOT}" 2>/dev/null || stat -f '%Lp' "${BRIO_RESTORE_TEMP_ROOT}") +if [[ "${temp_mode}" != "700" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must have mode 0700." >&2 + exit 1 +fi +temp_owner=$(stat -c '%u' "${BRIO_RESTORE_TEMP_ROOT}" 2>/dev/null || stat -f '%u' "${BRIO_RESTORE_TEMP_ROOT}") +if [[ "${temp_owner}" != "$(id -u)" ]]; then + echo "BRIO_RESTORE_TEMP_ROOT must be owned by the restore operator." >&2 + exit 1 +fi + +cert_public_key=$(openssl x509 -in "${BRIO_RESTORE_RECIPIENT_CERT}" -pubkey -noout \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | sha256sum | awk '{print $1}') +private_public_key=$(openssl pkey -in "${BRIO_RESTORE_RECIPIENT_KEY}" -pubout -outform DER 2>/dev/null \ + | sha256sum | awk '{print $1}') +if [[ -z "${cert_public_key}" || "${cert_public_key}" != "${private_public_key}" ]]; then + echo "Brio restore recipient certificate and private key do not match." >&2 + exit 1 +fi + +validate_bundle() { + local backup_dir=$1 + local database=$2 + local encrypted_name=${database}.dump.cms + if [[ ! -d "${backup_dir}" || -L "${backup_dir}" ]]; then + echo "Backup directory must be a regular directory and not a symlink: ${backup_dir}" >&2 + exit 1 + fi + for required in "${encrypted_name}" SHA256SUMS metadata.json; do + if [[ ! -s "${backup_dir}/${required}" || -L "${backup_dir}/${required}" ]]; then + echo "Encrypted backup artifact is missing, empty, or a symlink: ${required}" >&2 + exit 1 + fi + done + ( + cd "${backup_dir}" + sha256sum --check --strict SHA256SUMS + ) + if ! grep -Fq "\"database\":\"${database}\"" "${backup_dir}/metadata.json" \ + || ! grep -Fq '"envelope":"openssl-cms-der"' "${backup_dir}/metadata.json" \ + || ! grep -Fq '"cipher":"aes-256-gcm"' "${backup_dir}/metadata.json"; then + echo "Encrypted backup metadata does not match ${database}." >&2 + exit 1 + fi + if ! openssl cms -cmsout -inform DER -in "${backup_dir}/${encrypted_name}" -noout >/dev/null 2>&1; then + echo "Encrypted backup is not a valid CMS envelope for ${database}." >&2 + exit 1 + fi +} + +validate_bundle "${app_backup_dir}" brio_staging +validate_bundle "${keycloak_backup_dir}" keycloak_brio_staging + +restore_dir=$(mktemp -d "${BRIO_RESTORE_TEMP_ROOT%/}/brio-restore.XXXXXX") +chmod 0700 "${restore_dir}" +cleanup() { + find "${restore_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${restore_dir}" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 1' HUP INT TERM + +decrypt_bundle() { + local backup_dir=$1 + local database=$2 + local output=$3 + openssl cms \ + -decrypt \ + -binary \ + -inform DER \ + -recip "${BRIO_RESTORE_RECIPIENT_CERT}" \ + -inkey "${BRIO_RESTORE_RECIPIENT_KEY}" \ + -in "${backup_dir}/${database}.dump.cms" \ + -out "${output}" + chmod 0600 "${output}" + pg_restore --list "${output}" >/dev/null +} + +app_dump=${restore_dir}/brio_staging.dump +keycloak_dump=${restore_dir}/keycloak_brio_staging.dump +decrypt_bundle "${app_backup_dir}" brio_staging "${app_dump}" +decrypt_bundle "${keycloak_backup_dir}" keycloak_brio_staging "${keycloak_dump}" + +restore_dump() { + local service=$1 + local dump=$2 + local database + database=$(psql "service=${service}" -v ON_ERROR_STOP=1 -Atc "SELECT current_database();") + if [[ "${database}" != *_restore_test ]]; then + echo "Restore service ${service} resolved to a database without the _restore_test suffix." >&2 + exit 1 + fi + pg_restore \ + --dbname="service=${service}" \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl \ + --exit-on-error \ + --single-transaction \ + "${dump}" +} + +restore_dump "${BRIO_APP_RESTORE_SERVICE}" "${app_dump}" +restore_dump "${BRIO_KEYCLOAK_RESTORE_SERVICE}" "${keycloak_dump}" + +app_state=$(psql "service=${BRIO_APP_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.schema_migrations') IS NOT NULL AND to_regclass('public.communities') IS NOT NULL;") +keycloak_state=$(psql "service=${BRIO_KEYCLOAK_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.realm') IS NOT NULL;") +if [[ "${app_state}" != "t" || "${keycloak_state}" != "t" ]]; then + echo "Restored databases are missing Brio or Keycloak durable state tables." >&2 + exit 1 +fi + +echo "Brio application and Keycloak encrypted restore verification completed against non-production targets." diff --git a/scripts/verify-brio-release-evidence.py b/scripts/verify-brio-release-evidence.py new file mode 100755 index 0000000..867bc3c --- /dev/null +++ b/scripts/verify-brio-release-evidence.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Fail-closed validation for the two-phase Brio database release contract.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +import stat +import sys +import zipfile +from pathlib import Path + + +POSTGRES_REPOSITORY = "Makepad-fr/postgres" +POSTGRES_WORKFLOW = ".github/workflows/deploy-brio-identity-db.yml" +KEYCLOAK_REPOSITORY = "Makepad-fr/keycloak" +KEYCLOAK_WORKFLOW = ".github/workflows/verify-brio-database.yml" +SHA = re.compile(r"[0-9a-f]{40}") +POSITIVE = re.compile(r"[1-9][0-9]*") + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def load(path: str) -> object: + with Path(path).open(encoding="utf-8") as source: + return json.load(source) + + +def positive(value: str, label: str) -> int: + if not POSITIVE.fullmatch(value): + fail(f"{label} must be a positive integer") + return int(value) + + +def full_sha(value: str, label: str) -> str: + if not SHA.fullmatch(value): + fail(f"{label} must be a lowercase full commit SHA") + return value + + +def exact_object(value: object, expected: dict[str, object], label: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != set(expected): + fail(f"{label} must have exactly the canonical fields") + for key, expected_value in expected.items(): + if value.get(key) != expected_value: + fail(f"{label} {key} mismatch") + return value + + +def complete_listing(value: object, collection: str, label: str) -> list[object]: + if not isinstance(value, dict): + fail(f"{label} must be a JSON object") + rows = value.get(collection) + total = value.get("total_count") + if ( + not isinstance(rows, list) + or isinstance(total, bool) + or not isinstance(total, int) + or total < 0 + or total != len(rows) + ): + fail(f"{label} is truncated or has an invalid total_count") + identifiers = [row.get("id") for row in rows if isinstance(row, dict)] + if len(identifiers) != len(rows) or any(isinstance(identifier, bool) or not isinstance(identifier, int) or identifier <= 0 for identifier in identifiers) or len(set(identifiers)) != len(identifiers): + fail(f"{label} has missing or duplicate IDs") + return rows + + +def safe_single_json(archive_path: str, expected_name: str) -> object: + path = Path(archive_path) + if not path.is_file() or path.is_symlink() or not 1 <= path.stat().st_size <= 131072: + fail("Evidence archive is missing, symlinked, empty, or oversized") + try: + with zipfile.ZipFile(path) as archive: + entries = archive.infolist() + if len(entries) != 1 or entries[0].filename != expected_name: + fail("Evidence archive must contain exactly its canonical JSON file") + entry = entries[0] + if stat.S_ISLNK(entry.external_attr >> 16) or not 1 <= entry.file_size <= 65536: + fail("Evidence archive entry is symlinked, empty, or oversized") + if entry.compress_size > 131072: + fail("Evidence archive entry has an unsafe compressed size") + return json.loads(archive.read(entry)) + except (zipfile.BadZipFile, UnicodeDecodeError, json.JSONDecodeError) as error: + fail(f"Evidence archive is invalid: {error}") + + +def postgres_run(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("postgres-run expects run, workflow, artifacts, run ID, and attempt") + run = load(arguments[0]) + workflow = load(arguments[1]) + artifacts = load(arguments[2]) + run_id = positive(arguments[3], "PostgreSQL run ID") + attempt = positive(arguments[4], "PostgreSQL run attempt") + if not isinstance(run, dict) or not isinstance(workflow, dict) or not isinstance(artifacts, dict): + fail("GitHub metadata must be JSON objects") + head_sha = full_sha(str(run.get("head_sha", "")), "PostgreSQL head SHA") + expected = { + "id": run_id, + "run_attempt": attempt, + "name": "Deploy Brio Identity Database", + "path": POSTGRES_WORKFLOW, + "event": "workflow_dispatch", + "head_branch": "main", + "head_sha": head_sha, + "status": "completed", + "conclusion": "success", + } + for key, expected_value in expected.items(): + if run.get(key) != expected_value: + fail(f"PostgreSQL deployment run {key} mismatch") + if run.get("repository", {}).get("full_name") != POSTGRES_REPOSITORY: + fail("PostgreSQL deployment repository mismatch") + if (workflow.get("name"), workflow.get("path"), workflow.get("state")) != ( + "Deploy Brio Identity Database", + POSTGRES_WORKFLOW, + "active", + ): + fail("PostgreSQL deployment workflow identity mismatch") + expected_name = f"brio-db-deployment-evidence-{run_id}-{attempt}" + artifact_rows = complete_listing(artifacts, "artifacts", "PostgreSQL artifact listing") + matches = [ + artifact + for artifact in artifact_rows + if artifact.get("name") == expected_name and not artifact.get("expired") + ] + if len(matches) != 1: + fail("Expected exactly one unexpired PostgreSQL deployment evidence artifact") + artifact = matches[0] + artifact_id = artifact.get("id") + size = artifact.get("size_in_bytes") + if not isinstance(artifact_id, int) or artifact_id <= 0: + fail("Invalid PostgreSQL deployment artifact ID") + if not isinstance(size, int) or not 1 <= size <= 131072: + fail("Unsafe PostgreSQL deployment artifact size") + print(f"{artifact_id} {head_sha}") + + +def postgres_evidence(arguments: list[str]) -> None: + if len(arguments) != 4: + fail("postgres-evidence expects archive, run ID, attempt, and head SHA") + run_id = positive(arguments[1], "PostgreSQL run ID") + attempt = positive(arguments[2], "PostgreSQL run attempt") + head_sha = full_sha(arguments[3], "PostgreSQL head SHA") + evidence = safe_single_json(arguments[0], "brio-db-deployment-evidence.json") + exact_object( + evidence, + { + "schema": "makepad.brio-db-deployment-evidence.v1", + "postgres_repository": POSTGRES_REPOSITORY, + "postgres_workflow": POSTGRES_WORKFLOW, + "postgres_run_id": run_id, + "postgres_run_attempt": attempt, + "postgres_head_sha": head_sha, + "postgres_ref": "refs/heads/main", + "deployment": "brio-db-host-ready", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, + "PostgreSQL deployment evidence", + ) + + +def keycloak_main(arguments: list[str]) -> None: + if len(arguments) != 1: + fail("keycloak-main expects the ref response") + payload = load(arguments[0]) + if not isinstance(payload, dict): + fail("Keycloak ref response must be an object") + print(full_sha(str(payload.get("object", {}).get("sha", "")), "Keycloak main SHA")) + + +def verifier_run_select(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("verifier-run-select expects runs, PostgreSQL run/attempt, Keycloak SHA, and dispatch time") + payload = load(arguments[0]) + run_id = positive(arguments[1], "PostgreSQL run ID") + attempt = positive(arguments[2], "PostgreSQL run attempt") + keycloak_sha = full_sha(arguments[3], "Keycloak release SHA") + try: + started = dt.datetime.fromisoformat(arguments[4].replace("Z", "+00:00")) + except ValueError as error: + fail(f"Invalid dispatch timestamp: {error}") + expected_title = f"Verify Brio DB path for PostgreSQL run {run_id}/{attempt} at Keycloak {keycloak_sha}" + matches: list[int] = [] + if not isinstance(payload, dict): + fail("Verifier run listing must be an object") + for run in complete_listing(payload, "workflow_runs", "Keycloak verifier run listing"): + try: + created = dt.datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) + except (KeyError, ValueError): + continue + if ( + run.get("display_title") == expected_title + and run.get("name") == "Verify Brio Identity Database Path" + and run.get("path") == KEYCLOAK_WORKFLOW + and run.get("repository", {}).get("full_name") == KEYCLOAK_REPOSITORY + and run.get("head_branch") == "main" + and run.get("head_sha") == keycloak_sha + and run.get("event") == "workflow_dispatch" + and created >= started - dt.timedelta(minutes=2) + ): + matches.append(run.get("id")) + if len(matches) > 1: + fail("Ambiguous Keycloak verifier runs") + if len(matches) == 1: + if not isinstance(matches[0], int) or matches[0] <= 0: + fail("Invalid Keycloak verifier run ID") + print(matches[0]) + + +def verifier_run(arguments: list[str]) -> None: + if len(arguments) != 5: + fail("verifier-run expects run, PostgreSQL run/attempt, Keycloak SHA, and verifier run ID") + run = load(arguments[0]) + postgres_run_id = positive(arguments[1], "PostgreSQL run ID") + postgres_attempt = positive(arguments[2], "PostgreSQL run attempt") + keycloak_sha = full_sha(arguments[3], "Keycloak release SHA") + verifier_id = positive(arguments[4], "Keycloak verifier run ID") + expected = { + "id": verifier_id, + "run_attempt": 1, + "name": "Verify Brio Identity Database Path", + "display_title": f"Verify Brio DB path for PostgreSQL run {postgres_run_id}/{postgres_attempt} at Keycloak {keycloak_sha}", + "path": KEYCLOAK_WORKFLOW, + "event": "workflow_dispatch", + "head_branch": "main", + "head_sha": keycloak_sha, + "status": "completed", + "conclusion": "success", + } + if not isinstance(run, dict): + fail("Keycloak verifier run must be an object") + for key, expected_value in expected.items(): + if run.get(key) != expected_value: + fail(f"Keycloak verifier run {key} mismatch") + if run.get("repository", {}).get("full_name") != KEYCLOAK_REPOSITORY: + fail("Keycloak verifier repository mismatch") + + +def attestation_artifact(arguments: list[str]) -> None: + if len(arguments) != 3: + fail("attestation-artifact expects artifacts, verifier run ID, and attempt") + payload = load(arguments[0]) + run_id = positive(arguments[1], "Keycloak verifier run ID") + attempt = positive(arguments[2], "Keycloak verifier run attempt") + expected_name = f"brio-db-path-attestation-{run_id}-{attempt}" + artifacts = complete_listing(payload, "artifacts", "Keycloak attestation artifact listing") + matches = [ + artifact + for artifact in artifacts + if artifact.get("name") == expected_name and not artifact.get("expired") + ] + if len(matches) != 1: + fail("Expected exactly one unexpired Keycloak path attestation artifact") + artifact = matches[0] + artifact_id, size = artifact.get("id"), artifact.get("size_in_bytes") + if not isinstance(artifact_id, int) or artifact_id <= 0: + fail("Invalid Keycloak attestation artifact ID") + if not isinstance(size, int) or not 1 <= size <= 131072: + fail("Unsafe Keycloak attestation artifact size") + print(artifact_id) + + +def attestation(arguments: list[str]) -> None: + if len(arguments) != 7: + fail("attestation expects archive, PostgreSQL run/attempt/SHA, Keycloak run/attempt/SHA") + postgres_run = positive(arguments[1], "PostgreSQL run ID") + postgres_attempt = positive(arguments[2], "PostgreSQL run attempt") + postgres_sha = full_sha(arguments[3], "PostgreSQL head SHA") + keycloak_run = positive(arguments[4], "Keycloak verifier run ID") + keycloak_attempt = positive(arguments[5], "Keycloak verifier run attempt") + keycloak_sha = full_sha(arguments[6], "Keycloak release SHA") + evidence = safe_single_json(arguments[0], "brio-db-path-attestation.json") + exact_object( + evidence, + { + "schema": "makepad.brio-db-path-attestation.v1", + "postgres_repository": POSTGRES_REPOSITORY, + "postgres_workflow": POSTGRES_WORKFLOW, + "postgres_run_id": postgres_run, + "postgres_run_attempt": postgres_attempt, + "postgres_head_sha": postgres_sha, + "keycloak_repository": KEYCLOAK_REPOSITORY, + "keycloak_workflow": KEYCLOAK_WORKFLOW, + "keycloak_verifier_run_id": keycloak_run, + "keycloak_verifier_run_attempt": keycloak_attempt, + "keycloak_release_sha": keycloak_sha, + "probe": "brio-db-path-ok", + "database": "keycloak_brio_staging", + "role": "keycloak_brio_staging_app", + "tls_host": "65.21.134.125", + "keycloak_source_cidr": "88.99.209.165/32", + }, + "Keycloak database-path attestation", + ) + + +COMMANDS = { + "postgres-run": postgres_run, + "postgres-evidence": postgres_evidence, + "keycloak-main": keycloak_main, + "verifier-run-select": verifier_run_select, + "verifier-run": verifier_run, + "attestation-artifact": attestation_artifact, + "attestation": attestation, +} + +if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: + fail("Usage: verify-brio-release-evidence.py [arguments ...]") +COMMANDS[sys.argv[1]](sys.argv[2:]) diff --git a/scripts/verify-keycloak-cohort-evidence.py b/scripts/verify-keycloak-cohort-evidence.py new file mode 100755 index 0000000..f855a8e --- /dev/null +++ b/scripts/verify-keycloak-cohort-evidence.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Validate the immutable five-database Keycloak restore evidence contract.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + +SCHEMA = "makepad.keycloak-cohort-restore-evidence.v2" +WORKFLOW = ".github/workflows/verify-keycloak-cohort-restores.yml" +BASE_IMAGE = "dhi.io/keycloak:26-debian13@sha256:fab1484b1762fd1269e63a40f068ec73ea75b498eaaa5d02f62f022a5d00ff0f" +CATEGORIES = { + "realm", + "authentication", + "roles", + "clients", + "identity_providers", + "components", + "required_actions", +} +DATABASES = { + "catwlk": "keycloak_catwlk", + "makepad": "keycloak_makepad", + "runtrace": "keycloak_runtrace", + "vestiaire": "keycloak_vestiaire", + "vif": "keycloak_vif", +} +TOP_KEYS = { + "schema", + "postgres_repository", + "postgres_workflow", + "postgres_run_id", + "postgres_run_attempt", + "postgres_head_sha", + "postgres_ref", + "keycloak_release_sha", + "keycloak_base_image", + "catwlk_runtime_image_id", + "fingerprint_schema", + "keycloak_upstream_version", + "result", + "instances", +} +INSTANCE_KEYS = { + "slug", + "database", + "backup_sha256", + "runtime", + "runtime_image", + "configuration_fingerprint", + "configuration_fingerprints", + "restore", + "keycloak_startup", + "configuration_regression", +} + + +def positive_integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{label} must be a positive integer") + return value + + +def exact_keys(value: object, expected: set[str], label: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != expected: + raise ValueError(f"{label} has unexpected fields") + return value + + +def validate( + path: Path, + *, + expected_run_id: int | None = None, + expected_run_attempt: int | None = None, + expected_head_sha: str | None = None, + expected_keycloak_release_sha: str | None = None, +) -> dict[str, object]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024: + raise ValueError("evidence must be a regular file no larger than 64 KiB") + raw = path.read_bytes() + if b"\x00" in raw: + raise ValueError("evidence contains a NUL byte") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("evidence is not valid UTF-8 JSON") from error + top = exact_keys(payload, TOP_KEYS, "evidence") + run_id = positive_integer(top["postgres_run_id"], "postgres_run_id") + attempt = positive_integer(top["postgres_run_attempt"], "postgres_run_attempt") + if top["schema"] != SCHEMA: + raise ValueError("unexpected evidence schema") + if top["postgres_repository"] != "Makepad-fr/postgres" or top["postgres_workflow"] != WORKFLOW: + raise ValueError("unexpected PostgreSQL producer identity") + if top["postgres_ref"] != "refs/heads/main": + raise ValueError("evidence did not originate from protected main") + for label in ("postgres_head_sha", "keycloak_release_sha"): + if not isinstance(top[label], str) or not re.fullmatch(r"[a-f0-9]{40}", top[label]): + raise ValueError(f"{label} is not a lowercase commit SHA") + if top["keycloak_base_image"] != BASE_IMAGE or top["keycloak_upstream_version"] != "26.7.3": + raise ValueError("evidence is not bound to the reviewed Keycloak runtime") + if not isinstance(top["catwlk_runtime_image_id"], str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", top["catwlk_runtime_image_id"]): + raise ValueError("Catwlk runtime image ID is not immutable") + if top["fingerprint_schema"] != "makepad.keycloak-config-fingerprint.v2": + raise ValueError("unexpected configuration fingerprint schema") + if top["result"] != "restored-databases-compatible": + raise ValueError("cohort compatibility did not pass") + instances = top["instances"] + if not isinstance(instances, list) or len(instances) != len(DATABASES): + raise ValueError("instances must contain the exact five-database cohort") + expected_slugs = sorted(DATABASES) + actual_slugs: list[str] = [] + for index, raw_instance in enumerate(instances): + instance = exact_keys(raw_instance, INSTANCE_KEYS, f"instances[{index}]") + slug = instance["slug"] + if not isinstance(slug, str): + raise ValueError("instance slug must be a string") + actual_slugs.append(slug) + if slug not in DATABASES or instance["database"] != DATABASES[slug]: + raise ValueError("instance slug/database mapping is invalid") + if not isinstance(instance["backup_sha256"], str) or not re.fullmatch(r"[a-f0-9]{64}", instance["backup_sha256"]): + raise ValueError("backup_sha256 must be a lowercase SHA-256 digest") + expected_runtime = "catwlk-custom-provider" if slug == "catwlk" else "keycloak-base" + expected_image = top["catwlk_runtime_image_id"] if slug == "catwlk" else BASE_IMAGE + if instance["runtime"] != expected_runtime or instance["runtime_image"] != expected_image: + raise ValueError(f"{slug} runtime does not match its reviewed exact image") + fingerprints = exact_keys(instance["configuration_fingerprints"], CATEGORIES, f"{slug} configuration fingerprints") + for category, digest in fingerprints.items(): + if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): + raise ValueError(f"{slug} {category} fingerprint is invalid") + canonical_fingerprints = "".join(f"{category}={fingerprints[category]}\n" for category in sorted(CATEGORIES)).encode() + combined = hashlib.sha256(canonical_fingerprints).hexdigest() + if instance["configuration_fingerprint"] != combined: + raise ValueError(f"{slug} combined configuration fingerprint is invalid") + for status in ("restore", "keycloak_startup", "configuration_regression"): + if instance[status] != "passed": + raise ValueError(f"{slug} {status} did not pass") + if actual_slugs != expected_slugs: + raise ValueError("instances are duplicated, missing, or not sorted by slug") + expected_values = ( + (expected_run_id, run_id, "run ID"), + (expected_run_attempt, attempt, "run attempt"), + (expected_head_sha, top["postgres_head_sha"], "PostgreSQL head SHA"), + (expected_keycloak_release_sha, top["keycloak_release_sha"], "Keycloak release SHA"), + ) + for expected, actual, label in expected_values: + if expected is not None and expected != actual: + raise ValueError(f"evidence {label} mismatch") + canonical = json.dumps(top, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + if raw not in (canonical, canonical + b"\n"): + raise ValueError("evidence JSON is not canonical") + return top + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("path", type=Path) + parser.add_argument("--run-id", type=int) + parser.add_argument("--run-attempt", type=int) + parser.add_argument("--head-sha") + parser.add_argument("--keycloak-release-sha") + arguments = parser.parse_args() + validate( + arguments.path, + expected_run_id=arguments.run_id, + expected_run_attempt=arguments.run_attempt, + expected_head_sha=arguments.head_sha, + expected_keycloak_release_sha=arguments.keycloak_release_sha, + ) + + +if __name__ == "__main__": + main()