diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 926a88d..01bd472 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -8,7 +8,6 @@ on: required: true type: choice options: - - canary - production jobs: @@ -23,165 +22,124 @@ jobs: - name: Configure SSH key shell: bash + env: + DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + DEPLOY_SSH_KNOWN_HOSTS: ${{ secrets.DEPLOY_SSH_KNOWN_HOSTS }} run: | set -euo pipefail + : "${DEPLOY_SSH_PRIVATE_KEY:?set DEPLOY_SSH_PRIVATE_KEY environment secret}" mkdir -p "${HOME}/.ssh" chmod 700 "${HOME}/.ssh" - printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > "${HOME}/.ssh/id_ed25519" + printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" > "${HOME}/.ssh/id_ed25519" chmod 600 "${HOME}/.ssh/id_ed25519" - - - name: Prepare deployment bundle - shell: bash - env: - DEPLOY_CATWLK_DB_NETWORK: ${{ secrets.DEPLOY_CATWLK_DB_NETWORK }} - DEPLOY_LE_PETIT_COIN_DB_NETWORK: ${{ secrets.DEPLOY_LE_PETIT_COIN_DB_NETWORK }} - DEPLOY_VIF_DB_NETWORK: ${{ secrets.DEPLOY_VIF_DB_NETWORK }} - DEPLOY_VIF_DB_NAME: ${{ secrets.DEPLOY_VIF_DB_NAME }} - DEPLOY_VIF_DB_USER: ${{ secrets.DEPLOY_VIF_DB_USER }} - DEPLOY_VIF_DB_PASSWORD: ${{ secrets.DEPLOY_VIF_DB_PASSWORD }} - run: | - set -euo pipefail - deploy_env="${{ inputs.environment }}" - : "${DEPLOY_CATWLK_DB_NETWORK:?set DEPLOY_CATWLK_DB_NETWORK environment secret}" - : "${DEPLOY_LE_PETIT_COIN_DB_NETWORK:?set DEPLOY_LE_PETIT_COIN_DB_NETWORK environment secret}" - if [[ "${deploy_env}" == "production" ]]; then - : "${DEPLOY_VIF_DB_NETWORK:?set DEPLOY_VIF_DB_NETWORK production environment secret}" - : "${DEPLOY_VIF_DB_PASSWORD:?set DEPLOY_VIF_DB_PASSWORD production environment secret}" - DEPLOY_VIF_DB_NAME="${DEPLOY_VIF_DB_NAME:-vif}" - DEPLOY_VIF_DB_USER="${DEPLOY_VIF_DB_USER:-vif}" - fi - bundle_root="${RUNNER_TEMP}/bundle" - mkdir -p "${bundle_root}/envs/${{ inputs.environment }}" - cp compose.yml "${bundle_root}/compose.yml" - cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" - cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" - cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <> "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" < "${HOME}/.ssh/known_hosts" + chmod 600 "${HOME}/.ssh/known_hosts" fi - - name: Deploy via Docker Swarm + - name: Upload deployment bundle shell: bash env: REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} REMOTE_DIR: ${{ secrets.DEPLOY_REMOTE_DIR }} - STACK_NAME: ${{ secrets.DEPLOY_STACK_NAME }} run: | set -euo pipefail - : "${REMOTE_HOST:?}" "${REMOTE_USER:?}" "${REMOTE_DIR:?}" "${STACK_NAME:?}" + : "${REMOTE_HOST:?set DEPLOY_SSH_HOST environment secret}" + : "${REMOTE_USER:?set DEPLOY_SSH_USER environment secret}" + : "${REMOTE_DIR:?set DEPLOY_REMOTE_DIR environment secret}" if [[ "${REMOTE_USER}" == "root" ]]; then echo "DEPLOY_SSH_USER must not be root." >&2 exit 1 fi - bundle_root="${RUNNER_TEMP}/bundle" remote_port=${REMOTE_PORT:-22} ssh_opts=(-o StrictHostKeyChecking=accept-new -p "${remote_port}") scp_opts=(-o StrictHostKeyChecking=accept-new -P "${remote_port}") + if [[ -s "${HOME}/.ssh/known_hosts" ]]; then + ssh_opts=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="${HOME}/.ssh/known_hosts" -p "${remote_port}") + scp_opts=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="${HOME}/.ssh/known_hosts" -P "${remote_port}") + fi remote_target="${REMOTE_USER}@${REMOTE_HOST}" - ssh "${ssh_opts[@]}" "${remote_target}" mkdir -p "${REMOTE_DIR}/envs/${{ inputs.environment }}" - scp "${scp_opts[@]}" "${bundle_root}/compose.yml" "${remote_target}:${REMOTE_DIR}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/compose.yml" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.db" - scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.deploy" + ssh "${ssh_opts[@]}" "${remote_target}" mkdir -p "${REMOTE_DIR}/envs/production" "${REMOTE_DIR}/bootstrap" + scp "${scp_opts[@]}" compose.yml "${remote_target}:${REMOTE_DIR}/compose.yml" + scp "${scp_opts[@]}" envs/production/compose.yml "${remote_target}:${REMOTE_DIR}/envs/production/compose.yml" + scp "${scp_opts[@]}" bootstrap/scraping-app.sql "${remote_target}:${REMOTE_DIR}/bootstrap/scraping-app.sql" - ssh "${ssh_opts[@]}" "${remote_target}" bash -se -- "${REMOTE_DIR}" "${STACK_NAME}" "${{ inputs.environment }}" <<'EOF' + - name: Deploy compose service and provision scraping DB + shell: bash + env: + DEPLOY_FASHION_CATALOG_READER_PASSWORD: ${{ secrets.DEPLOY_FASHION_CATALOG_READER_PASSWORD }} + DEPLOY_SCRAPING_DB_PASSWORD: ${{ secrets.DEPLOY_SCRAPING_DB_PASSWORD }} + REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} + REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} + REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} + REMOTE_DIR: ${{ secrets.DEPLOY_REMOTE_DIR }} + run: | set -euo pipefail - remote_dir=$1 - stack_name=$2 - deploy_env=$3 - env_deploy="${remote_dir}/envs/${deploy_env}/.env.deploy" - db_network=$(grep '^MAKEPAD_POSTGRES_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - le_petit_coin_db_network=$(grep '^MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_enabled=0 - if [[ "${deploy_env}" == "production" ]]; then - vif_enabled=1 - vif_db_network=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NETWORK=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_name=$(grep '^MAKEPAD_POSTGRES_VIF_DB_NAME=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_user=$(grep '^MAKEPAD_POSTGRES_VIF_DB_USER=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - vif_db_password=$(grep '^MAKEPAD_POSTGRES_VIF_DB_PASSWORD=' "${env_deploy}" | tail -n 1 | cut -d= -f2-) - postgres_root_user=$(grep '^POSTGRES_USER=' "${remote_dir}/envs/${deploy_env}/.env.db" | tail -n 1 | cut -d= -f2-) - postgres_root_password=$(grep '^POSTGRES_PASSWORD=' "${remote_dir}/envs/${deploy_env}/.env.db" | tail -n 1 | cut -d= -f2-) - fi - : "${db_network:?MAKEPAD_POSTGRES_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${le_petit_coin_db_network:?MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK is missing or empty in ${env_deploy}}" - if [[ "${vif_enabled}" == "1" ]]; then - : "${vif_db_network:?MAKEPAD_POSTGRES_VIF_DB_NETWORK is missing or empty in ${env_deploy}}" - : "${vif_db_name:?MAKEPAD_POSTGRES_VIF_DB_NAME is missing or empty in ${env_deploy}}" - : "${vif_db_user:?MAKEPAD_POSTGRES_VIF_DB_USER is missing or empty in ${env_deploy}}" - : "${vif_db_password:?MAKEPAD_POSTGRES_VIF_DB_PASSWORD is missing or empty in ${env_deploy}}" - : "${postgres_root_user:?POSTGRES_USER is missing or empty in .env.db}" - : "${postgres_root_password:?POSTGRES_PASSWORD is missing or empty in .env.db}" - fi - - docker network inspect "${db_network}" >/dev/null 2>&1 \ - || docker network create --driver overlay --attachable "${db_network}" - docker network inspect "${le_petit_coin_db_network}" >/dev/null 2>&1 \ - || docker network create --driver overlay --attachable "${le_petit_coin_db_network}" - if [[ "${vif_enabled}" == "1" ]]; then - docker network inspect "${vif_db_network}" >/dev/null 2>&1 \ - || docker network create --driver overlay --attachable "${vif_db_network}" - export MAKEPAD_POSTGRES_VIF_DB_NETWORK="${vif_db_network}" + : "${DEPLOY_FASHION_CATALOG_READER_PASSWORD:?set DEPLOY_FASHION_CATALOG_READER_PASSWORD environment secret}" + : "${DEPLOY_SCRAPING_DB_PASSWORD:?set DEPLOY_SCRAPING_DB_PASSWORD environment secret}" + remote_port=${REMOTE_PORT:-22} + ssh_opts=(-o StrictHostKeyChecking=accept-new -p "${remote_port}") + if [[ -s "${HOME}/.ssh/known_hosts" ]]; then + ssh_opts=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="${HOME}/.ssh/known_hosts" -p "${remote_port}") fi + remote_target="${REMOTE_USER}@${REMOTE_HOST}" - export MAKEPAD_POSTGRES_DB_NETWORK="${db_network}" - export MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK="${le_petit_coin_db_network}" - docker compose \ - --env-file "${remote_dir}/envs/${deploy_env}/.env.db" \ - --env-file "${env_deploy}" \ - -f "${remote_dir}/compose.yml" \ - -f "${remote_dir}/envs/${deploy_env}/compose.yml" \ - config > "${remote_dir}/stack.yml" - - docker stack deploy --compose-file "${remote_dir}/stack.yml" "${stack_name}" + ssh "${ssh_opts[@]}" "${remote_target}" bash -se -- \ + "${REMOTE_DIR}" \ + "${DEPLOY_SCRAPING_DB_PASSWORD}" \ + "${DEPLOY_FASHION_CATALOG_READER_PASSWORD}" <<'EOF' + set -euo pipefail + remote_dir=$1 + scraping_db_password=$2 + fashion_catalog_reader_password=$3 - if [[ "${vif_enabled}" != "1" ]]; then - exit 0 + cd "${remote_dir}" + container_name="${POSTGRES_CONTAINER_NAME:-postgres-postgres-1}" + if ! docker inspect "${container_name}" >/dev/null 2>&1; then + echo "Postgres container ${container_name} is not running; start the systemd-managed compose service first." >&2 + exit 1 fi + postgres_env="$(docker inspect "${container_name}" --format '{{range .Config.Env}}{{println .}}{{end}}')" + POSTGRES_USER="$(printf '%s\n' "${postgres_env}" | sed -n 's/^POSTGRES_USER=//p' | tail -n 1)" + POSTGRES_DB="$(printf '%s\n' "${postgres_env}" | sed -n 's/^POSTGRES_DB=//p' | tail -n 1)" + POSTGRES_PASSWORD="$(printf '%s\n' "${postgres_env}" | sed -n 's/^POSTGRES_PASSWORD=//p' | tail -n 1)" + : "${POSTGRES_USER:?POSTGRES_USER missing from ${container_name}}" + : "${POSTGRES_DB:?POSTGRES_DB missing from ${container_name}}" + : "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD missing from ${container_name}}" - postgres_ready=0 for attempt in $(seq 1 30); do - if docker run --rm --network "${vif_db_network}" -e PGPASSWORD="${postgres_root_password}" postgres:16-alpine \ - psql -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres -c "select 1" >/dev/null 2>&1; then - postgres_ready=1 + if docker exec "${container_name}" pg_isready -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" >/dev/null 2>&1; then break fi + if [[ "${attempt}" == "30" ]]; then + echo "Postgres did not become ready after 60 seconds." >&2 + exit 1 + fi sleep 2 done - if [[ "${postgres_ready}" != "1" ]]; then - echo "Postgres did not become reachable via makepad-postgres-vif on ${vif_db_network} after 60 seconds." >&2 - exit 1 - fi - docker run --rm --network "${vif_db_network}" -e PGPASSWORD="${postgres_root_password}" postgres:16-alpine \ - psql -h makepad-postgres-vif -U "${postgres_root_user}" -d postgres \ - -v ON_ERROR_STOP=1 \ - -v vif_db="${vif_db_name}" \ - -v vif_user="${vif_db_user}" \ - -v vif_password="${vif_db_password}" <<'SQL' - SELECT format('CREATE ROLE %I LOGIN', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'vif_user') \gexec - SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'vif_user', :'vif_password') \gexec - SELECT format('CREATE DATABASE %I OWNER %I', :'vif_db', :'vif_user') - WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'vif_db') \gexec - SELECT format('ALTER DATABASE %I OWNER TO %I', :'vif_db', :'vif_user') - WHERE EXISTS ( - SELECT 1 - FROM pg_database d - JOIN pg_roles r ON r.oid = d.datdba - WHERE d.datname = :'vif_db' - AND r.rolname <> :'vif_user' - ) \gexec - SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'vif_db', :'vif_user') \gexec - SQL + tmp_sql="$(mktemp)" + chmod 600 "${tmp_sql}" + python3 - \ + "${scraping_db_password}" \ + "${fashion_catalog_reader_password}" \ + "${remote_dir}/bootstrap/scraping-app.sql" \ + "${tmp_sql}" <<'PY' + import sys + + crawler_password, reader_password, bootstrap_path, output_path = sys.argv[1:5] + crawler_quoted = "'" + crawler_password.replace("'", "''") + "'" + reader_quoted = "'" + reader_password.replace("'", "''") + "'" + with open(output_path, "w", encoding="utf-8") as output: + output.write("\\set scraping_crawler_password " + crawler_quoted + "\n") + output.write("\\set fashion_catalog_reader_password " + reader_quoted + "\n") + output.write(open(bootstrap_path, encoding="utf-8").read()) + PY + + docker exec -i "${container_name}" env PGPASSWORD="${POSTGRES_PASSWORD}" \ + psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" < "${tmp_sql}" + rm -f "${tmp_sql}" EOF diff --git a/README.md b/README.md index 4964fd7..4838d19 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `envs/production/.env.db`: production PostgreSQL settings - `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the Vif, Makepad, Vestiaire, and Runtrace Keycloak databases - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database +- `bootstrap/scraping-app.sql`: idempotent SQL bootstrap for the shared scraping frontier database +- `bootstrap/iceberg-catalog.sql`: idempotent SQL bootstrap for durable Iceberg catalog metadata +- `bootstrap/vestiaire-developer-platform.sql`: idempotent bootstrap for Vestiaire organizations, memberships, and API credential metadata ## Networks @@ -21,19 +24,21 @@ The database joins external overlay networks configured through Compose: - `${MAKEPAD_POSTGRES_DB_NETWORK}` - `${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK}` -Production also joins the VIF-specific external overlay network: +Production also joins production-only application-specific external overlay networks: - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` +- `${MAKEPAD_POSTGRES_SCRAPING_DB_NETWORK}` The manual deploy workflow sources these Compose variables from environment secrets with this mapping: - `${MAKEPAD_POSTGRES_DB_NETWORK}` <- `DEPLOY_CATWLK_DB_NETWORK` - `${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK}` <- `DEPLOY_LE_PETIT_COIN_DB_NETWORK` - `${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK` production only +- `${MAKEPAD_POSTGRES_SCRAPING_DB_NETWORK}` <- `DEPLOY_SCRAPING_DB_NETWORK` production only Application network topology is owned by the consuming application repositories. New Keycloak instances keep their own DB-facing Docker networks in the Keycloak repository and connect to this PostgreSQL server through the configured DB endpoint. -When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. Canary does not attach the VIF network. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead; that host-based path depends on the standalone DB VM deployment exposing PostgreSQL on the VM host. +When using this repository's overlay-network deployment model, application stacks attached to the shared database network should use the stable service alias `makepad-postgres`. Le Petit Coin stacks attach through their app-specific database network and should use `makepad-postgres-le-petit-coin`. The production VIF stack attaches through its production-only app-specific database network and should use `makepad-postgres-vif`. The production Scraping crawler stack attaches through its production-only app-specific database network and should use `makepad-postgres-scraping`. Canary does not attach the VIF or Scraping networks. The current production Keycloak deployment is separate from this stack and uses the DB VM host address instead; that host-based path depends on the standalone DB VM deployment exposing PostgreSQL on the VM host. ## Node Labels @@ -62,8 +67,12 @@ Production additionally requires: - `DEPLOY_VIF_DB_NETWORK` - `DEPLOY_VIF_DB_PASSWORD` +- `DEPLOY_SCRAPING_DB_NETWORK` +- `DEPLOY_SCRAPING_DB_PASSWORD` +- `DEPLOY_FASHION_CATALOG_READER_PASSWORD` Production can override the VIF database and role names with `DEPLOY_VIF_DB_NAME` and `DEPLOY_VIF_DB_USER`; both default to `vif`. +Production can override the Scraping database and role names with `DEPLOY_SCRAPING_DB_NAME` and `DEPLOY_SCRAPING_DB_USER`; they default to `scraping` and `scraping_crawler`. `DEPLOY_SSH_USER` must be a non-root deployment account with the Docker permissions needed to create overlay networks and deploy the stack. The workflow rejects `DEPLOY_SSH_USER=root`. @@ -88,6 +97,12 @@ Runtrace application persistence uses: | --- | --- | --- | | Runtrace app | `runtrace` | `runtrace_app` | +The Vestiaire developer platform uses an isolated database and role: + +| Application | Database | Role | +| --- | --- | --- | +| Vestiaire developer platform | `vestiaire_developer` | `vestiaire_developer_app` | + Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must be a PostgreSQL superuser connection URI for the target server, usually using the `postgres` role, because the bootstrap creates roles, sets passwords, creates databases, and assigns database ownership. For example: `postgres://postgres@:5432/postgres?sslmode=disable`. ```bash @@ -96,7 +111,10 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must : "${KEYCLOAK_MAKEPAD_DB_PASSWORD:?set KEYCLOAK_MAKEPAD_DB_PASSWORD to a generated password}" : "${KEYCLOAK_VESTIAIRE_DB_PASSWORD:?set KEYCLOAK_VESTIAIRE_DB_PASSWORD to a generated password}" : "${KEYCLOAK_RUNTRACE_DB_PASSWORD:?set KEYCLOAK_RUNTRACE_DB_PASSWORD to a generated password}" +: "${SCRAPING_DB_PASSWORD:?set SCRAPING_DB_PASSWORD to a generated password}" +: "${FASHION_CATALOG_READER_PASSWORD:?set FASHION_CATALOG_READER_PASSWORD to a generated password}" : "${RUNTRACE_DB_PASSWORD:?set RUNTRACE_DB_PASSWORD to a generated password}" +: "${VESTIAIRE_DEVELOPER_DB_PASSWORD:?set VESTIAIRE_DEVELOPER_DB_PASSWORD to a generated password}" psql "$POSTGRES_ADMIN_URL" \ -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ @@ -108,6 +126,20 @@ psql "$POSTGRES_ADMIN_URL" \ psql "$POSTGRES_ADMIN_URL" \ -v runtrace_app_password="$RUNTRACE_DB_PASSWORD" \ -f bootstrap/runtrace-app.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v vestiaire_developer_app_password="$VESTIAIRE_DEVELOPER_DB_PASSWORD" \ + -f bootstrap/vestiaire-developer-platform.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v scraping_crawler_password="$SCRAPING_DB_PASSWORD" \ + -v fashion_catalog_reader_password="$FASHION_CATALOG_READER_PASSWORD" \ + -f bootstrap/scraping-app.sql + +# Provision or rotate only the catalog reader without touching the crawler role. +psql "$POSTGRES_ADMIN_URL" \ + -v fashion_catalog_reader_password="$FASHION_CATALOG_READER_PASSWORD" \ + -f bootstrap/fashion-catalog-reader.sql ``` The current production Keycloak environments connect with the DB VM host: @@ -118,6 +150,7 @@ postgres://keycloak_makepad_app:@:5432/keycloak_makepad?sslm postgres://keycloak_vestiaire_app:@:5432/keycloak_vestiaire?sslmode=disable postgres://keycloak_runtrace_app:@:5432/keycloak_runtrace?sslmode=disable postgres://runtrace_app:@:5432/runtrace?sslmode=disable +postgres://vestiaire_developer_app:@:5432/vestiaire_developer?sslmode=disable ``` Stacks deployed through this repository's shared overlay network should use the `makepad-postgres` alias instead: @@ -145,6 +178,22 @@ postgres://vif:@makepad-postgres-vif:5432/vif?sslmode=disable If production overrides `DEPLOY_VIF_DB_NAME` or `DEPLOY_VIF_DB_USER`, use those values in the connection URI. +The production Scraping crawler uses its app-specific overlay alias and deploy-time provisioned database: + +```text +postgres://scraping_crawler:@makepad-postgres-scraping:5432/scraping +``` + +The fashion product catalog inventory indexer uses a separate read-only role on the same database: + +```text +postgres://fashion_catalog_reader:@makepad-postgres-scraping:5432/scraping +``` + +`fashion_catalog_reader` receives `CONNECT`, schema `USAGE`, and `SELECT` on current and future tables created by `scraping_crawler`. It cannot mutate crawler frontier or inventory state. The catalog deployment should store its password as a dedicated external Docker secret. + +If production overrides `DEPLOY_SCRAPING_DB_NAME` or `DEPLOY_SCRAPING_DB_USER`, use those values in the connection URI. + ## Validation Run the local static checks before opening a deployment PR: diff --git a/bootstrap/fashion-catalog-reader.sql b/bootstrap/fashion-catalog-reader.sql new file mode 100644 index 0000000..9af239d --- /dev/null +++ b/bootstrap/fashion-catalog-reader.sql @@ -0,0 +1,39 @@ +\set ON_ERROR_STOP on + +-- Provision the catalog inventory reader without changing crawler credentials. +\if :{?fashion_catalog_reader_password} +\else + \echo 'missing required psql variable: fashion_catalog_reader_password' + \quit 1 +\endif + +SELECT CASE + WHEN NULLIF(btrim(:'fashion_catalog_reader_password'), '') IS NULL THEN 'false' + ELSE 'true' +END AS fashion_catalog_reader_password_is_nonempty \gset +\if :fashion_catalog_reader_password_is_nonempty +\else + \echo 'empty required psql variable: fashion_catalog_reader_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('fashion-catalog-reader')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'fashion_catalog_reader') THEN + CREATE ROLE fashion_catalog_reader LOGIN; + END IF; +END; +$$; +ALTER ROLE fashion_catalog_reader LOGIN PASSWORD :'fashion_catalog_reader_password'; +GRANT CONNECT ON DATABASE scraping TO fashion_catalog_reader; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('fashion-catalog-reader')); + +\connect scraping + +GRANT USAGE ON SCHEMA public TO fashion_catalog_reader; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader; +ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler IN SCHEMA public + GRANT SELECT ON TABLES TO fashion_catalog_reader; diff --git a/bootstrap/iceberg-catalog.sql b/bootstrap/iceberg-catalog.sql new file mode 100644 index 0000000..b03f31a --- /dev/null +++ b/bootstrap/iceberg-catalog.sql @@ -0,0 +1,41 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. The role stores only Iceberg +-- catalog metadata; product data remains in object storage. + +\if :{?iceberg_catalog_password} +\else + \echo 'missing required psql variable: iceberg_catalog_password' + \quit 1 +\endif + +SELECT CASE WHEN NULLIF(btrim(:'iceberg_catalog_password'), '') IS NULL THEN 'false' ELSE 'true' END AS iceberg_catalog_password_is_nonempty \gset +\if :iceberg_catalog_password_is_nonempty +\else + \echo 'empty required psql variable: iceberg_catalog_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('iceberg-catalog-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'iceberg_catalog') THEN + CREATE ROLE iceberg_catalog LOGIN; + END IF; +END; +$$; +ALTER ROLE iceberg_catalog LOGIN PASSWORD :'iceberg_catalog_password'; +SELECT 'CREATE DATABASE iceberg_catalog OWNER iceberg_catalog' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'iceberg_catalog') \gexec +SELECT 'ALTER DATABASE iceberg_catalog OWNER TO iceberg_catalog' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'iceberg_catalog' + AND r.rolname <> 'iceberg_catalog' +) \gexec +GRANT CONNECT ON DATABASE iceberg_catalog TO iceberg_catalog; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('iceberg-catalog-bootstrap')); diff --git a/bootstrap/scraping-app.sql b/bootstrap/scraping-app.sql new file mode 100644 index 0000000..a3a4bcd --- /dev/null +++ b/bootstrap/scraping-app.sql @@ -0,0 +1,70 @@ +\set ON_ERROR_STOP on + +-- Run this bootstrap with a PostgreSQL superuser connection. It creates the +-- Scraping crawler application role/database pair without embedding secrets. + +\if :{?scraping_crawler_password} +\else + \echo 'missing required psql variable: scraping_crawler_password' + \quit 1 +\endif + +\if :{?fashion_catalog_reader_password} +\else + \echo 'missing required psql variable: fashion_catalog_reader_password' + \quit 1 +\endif + +SELECT CASE WHEN NULLIF(btrim(:'scraping_crawler_password'), '') IS NULL THEN 'false' ELSE 'true' END AS scraping_crawler_password_is_nonempty \gset +\if :scraping_crawler_password_is_nonempty +\else + \echo 'empty required psql variable: scraping_crawler_password' + \quit 1 +\endif + +SELECT CASE WHEN NULLIF(btrim(:'fashion_catalog_reader_password'), '') IS NULL THEN 'false' ELSE 'true' END AS fashion_catalog_reader_password_is_nonempty \gset +\if :fashion_catalog_reader_password_is_nonempty +\else + \echo 'empty required psql variable: fashion_catalog_reader_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('scraping-app-bootstrap')); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'scraping_crawler') THEN + CREATE ROLE scraping_crawler LOGIN; + END IF; +END; +$$; +ALTER ROLE scraping_crawler LOGIN PASSWORD :'scraping_crawler_password'; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'fashion_catalog_reader') THEN + CREATE ROLE fashion_catalog_reader LOGIN; + END IF; +END; +$$; +ALTER ROLE fashion_catalog_reader LOGIN PASSWORD :'fashion_catalog_reader_password'; +SELECT 'CREATE DATABASE scraping OWNER scraping_crawler' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'scraping') \gexec +SELECT 'ALTER DATABASE scraping OWNER TO scraping_crawler' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'scraping' + AND r.rolname <> 'scraping_crawler' +) \gexec +GRANT CONNECT ON DATABASE scraping TO scraping_crawler; +GRANT CONNECT ON DATABASE scraping TO fashion_catalog_reader; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('scraping-app-bootstrap')); + +\connect scraping + +GRANT USAGE ON SCHEMA public TO fashion_catalog_reader; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader; +ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler IN SCHEMA public + GRANT SELECT ON TABLES TO fashion_catalog_reader; diff --git a/bootstrap/vestiaire-developer-platform.sql b/bootstrap/vestiaire-developer-platform.sql new file mode 100644 index 0000000..6f4474a --- /dev/null +++ b/bootstrap/vestiaire-developer-platform.sql @@ -0,0 +1,57 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. The application owns this +-- database; Flyway creates and evolves its tenant schema at startup. + +\if :{?vestiaire_developer_app_password} +\else + \echo 'missing required psql variable: vestiaire_developer_app_password' + \quit 1 +\endif + +SELECT CASE + WHEN NULLIF(btrim(:'vestiaire_developer_app_password'), '') IS NULL THEN 'false' + ELSE 'true' +END AS vestiaire_developer_app_password_is_nonempty \gset +\if :vestiaire_developer_app_password_is_nonempty +\else + \echo 'empty required psql variable: vestiaire_developer_app_password' + \quit 1 +\endif + +SELECT pg_advisory_lock( + hashtext('makepad-postgres'), + hashtext('vestiaire-developer-platform-bootstrap') +); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vestiaire_developer_app') THEN + CREATE ROLE vestiaire_developer_app LOGIN; + END IF; +END; +$$; + +ALTER ROLE vestiaire_developer_app + LOGIN PASSWORD :'vestiaire_developer_app_password'; + +SELECT 'CREATE DATABASE vestiaire_developer OWNER vestiaire_developer_app' +WHERE NOT EXISTS ( + SELECT 1 FROM pg_database WHERE datname = 'vestiaire_developer' +) \gexec + +SELECT 'ALTER DATABASE vestiaire_developer OWNER TO vestiaire_developer_app' +WHERE EXISTS ( + SELECT 1 + FROM pg_database d + JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'vestiaire_developer' + AND r.rolname <> 'vestiaire_developer_app' +) \gexec + +GRANT CONNECT ON DATABASE vestiaire_developer TO vestiaire_developer_app; + +SELECT pg_advisory_unlock( + hashtext('makepad-postgres'), + hashtext('vestiaire-developer-platform-bootstrap') +); diff --git a/compose.yml b/compose.yml index d04d34a..f25f6de 100644 --- a/compose.yml +++ b/compose.yml @@ -1,14 +1,15 @@ services: postgres: image: postgres:16-alpine - networks: - db: - aliases: - - makepad-postgres - le_petit_coin: - aliases: - - makepad-postgres-le-petit-coin - -networks: - db: - le_petit_coin: + network_mode: host + env_file: + - /etc/makepad/postgres/postgres.env + volumes: + - "${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""] + interval: 10s + timeout: 5s + retries: 6 + start_period: 20s + restart: unless-stopped diff --git a/envs/production/compose.yml b/envs/production/compose.yml index 7a7d8ea..ec1ca5c 100644 --- a/envs/production/compose.yml +++ b/envs/production/compose.yml @@ -1,36 +1,2 @@ services: - postgres: - env_file: - - ./envs/production/.env.db - networks: - vif: - aliases: - - makepad-postgres-vif - volumes: - - "${MAKEPAD_POSTGRES_DATA_PATH:-/var/lib/makepad/postgres}:/var/lib/postgresql/data" - deploy: - replicas: 1 - placement: - constraints: - - node.labels.infra.makepad.postgres == true - restart_policy: - condition: on-failure - delay: 5s - update_config: - parallelism: 1 - order: stop-first - failure_action: rollback - rollback_config: - parallelism: 1 - order: stop-first - -networks: - db: - external: true - name: ${MAKEPAD_POSTGRES_DB_NETWORK} - le_petit_coin: - external: true - name: ${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK} - vif: - external: true - name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK} + postgres: {} diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 32288f6..4d091b4 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -1,182 +1,51 @@ #!/usr/bin/env bash set -euo pipefail -for binary in python3; do - if ! command -v "${binary}" >/dev/null 2>&1; then - echo "Missing required binary for postgres validation: ${binary}" >&2 - exit 1 - fi -done +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -repo_root=$(cd "${script_dir}/.." && pwd) - -REPO_ROOT="${repo_root}" python3 - <<'PY' -import os -import re +python3 - "${repo_root}" <<'PY' from pathlib import Path +import sys +root = Path(sys.argv[1]) def require(condition, message): if not condition: raise SystemExit(message) - -def read_required_text(path, label): - require(path.is_file(), f"{label} is missing or is not a file: {path}") - try: - return path.read_text(encoding="utf-8") - except OSError as error: - raise SystemExit(f"Unable to read {label} at {path}: {error}") from error - - -expected_instances = { - "vif": { - "role": "keycloak_vif_app", - "database": "keycloak_vif", - "password_variable": "keycloak_vif_app_password", - "environment_variable": "KEYCLOAK_VIF_DB_PASSWORD", - }, - "makepad": { - "role": "keycloak_makepad_app", - "database": "keycloak_makepad", - "password_variable": "keycloak_makepad_app_password", - "environment_variable": "KEYCLOAK_MAKEPAD_DB_PASSWORD", - }, - "vestiaire": { - "role": "keycloak_vestiaire_app", - "database": "keycloak_vestiaire", - "password_variable": "keycloak_vestiaire_app_password", - "environment_variable": "KEYCLOAK_VESTIAIRE_DB_PASSWORD", - }, - "runtrace": { - "role": "keycloak_runtrace_app", - "database": "keycloak_runtrace", - "password_variable": "keycloak_runtrace_app_password", - "environment_variable": "KEYCLOAK_RUNTRACE_DB_PASSWORD", - }, -} - -repo_root = Path(os.environ["REPO_ROOT"]) -sql = read_required_text(repo_root / "bootstrap/keycloak-new-instances.sql", "SQL bootstrap") -runtrace_sql = read_required_text(repo_root / "bootstrap/runtrace-app.sql", "Runtrace app SQL bootstrap") -readme = read_required_text(repo_root / "README.md", "README") -base_compose = read_required_text(repo_root / "compose.yml", "base Compose file") -canary_compose = read_required_text(repo_root / "envs/canary/compose.yml", "canary Compose override") -production_compose = read_required_text(repo_root / "envs/production/compose.yml", "production Compose override") -manual_deploy = read_required_text(repo_root / ".github/workflows/manual-deploy.yml", "manual deploy workflow") -normalized_readme = re.sub(r"\s+", " ", readme) - -require("docker network create" not in sql, "SQL bootstrap must not manage Docker networks.") -require("${POSTGRES_ADMIN_URL:?" in readme, "README bootstrap command must fail fast for POSTGRES_ADMIN_URL.") -require("PostgreSQL superuser connection URI" in normalized_readme, "README must define POSTGRES_ADMIN_URL as a PostgreSQL superuser connection URI.") -require( - re.search(r"creates\s+roles.*sets\s+passwords.*creates\s+databases.*assigns\s+database\s+ownership", normalized_readme, re.IGNORECASE), - "README must document that the bootstrap requires superuser-level role and database ownership privileges.", -) -require("PostgreSQL superuser connection" in sql, "SQL bootstrap must document its superuser connection requirement.") -require("pg_advisory_lock" in sql, "SQL bootstrap must serialize concurrent runs with an advisory lock.") -require("pg_advisory_unlock" in sql, "SQL bootstrap must release its advisory lock after provisioning.") -password_check_index = sql.find("keycloak_runtrace_app_password_is_nonempty") -lock_index = sql.find("pg_advisory_lock") -role_block_index = sql.find("DO $$") -require(password_check_index != -1, "SQL bootstrap must include the Vestiaire password non-empty check marker.") -require(lock_index != -1, "SQL bootstrap must include the advisory lock marker.") -require(role_block_index != -1, "SQL bootstrap must include the first role provisioning DO block marker.") -require( - password_check_index < lock_index < role_block_index, - "SQL bootstrap must validate required password variables before waiting on the advisory lock.", -) -require("" in normalized_readme, "README must document the standalone DB VM host connection path.") -require("`makepad-postgres`" in normalized_readme, "README must document the exact shared overlay service alias connection path.") -require( - re.search(r"standalone\s+DB\s+VM\s+deployment.*expos(?:e|ing).*PostgreSQL.*VM\s+host", normalized_readme, re.IGNORECASE), - "README must explain that host-based connections depend on the standalone DB VM deployment exposing PostgreSQL.", -) -require("MAKEPAD_POSTGRES_DB_NETWORK" in normalized_readme, "README must document the Compose network variable.") -require("MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK" in normalized_readme, "README must document the Le Petit Coin Compose network variable.") -require("MAKEPAD_POSTGRES_VIF_DB_NETWORK" in normalized_readme, "README must document the production VIF Compose network variable.") -require( - "`${MAKEPAD_POSTGRES_DB_NETWORK}` <- `DEPLOY_CATWLK_DB_NETWORK`" in normalized_readme, - "README must document that DEPLOY_CATWLK_DB_NETWORK feeds MAKEPAD_POSTGRES_DB_NETWORK during deploy.", -) -require( - "`${MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK}` <- `DEPLOY_LE_PETIT_COIN_DB_NETWORK`" in normalized_readme, - "README must document that DEPLOY_LE_PETIT_COIN_DB_NETWORK feeds MAKEPAD_POSTGRES_LE_PETIT_COIN_DB_NETWORK during deploy.", -) -require( - "`${MAKEPAD_POSTGRES_VIF_DB_NETWORK}` <- `DEPLOY_VIF_DB_NETWORK`" in normalized_readme, - "README must document that DEPLOY_VIF_DB_NETWORK feeds MAKEPAD_POSTGRES_VIF_DB_NETWORK during production deploy.", -) -require("makepad-postgres-le-petit-coin" in normalized_readme, "README must document the Le Petit Coin database network alias.") -require("makepad-postgres-vif" in normalized_readme, "README must document the VIF database network alias.") -require("Canary does not attach the VIF network" in readme, "README must document that VIF network attachment is production-only.") -require("DEPLOY_VIF_DB_PASSWORD" in normalized_readme, "README must document the production VIF database password secret.") -require("DEPLOY_VIF_DB_NAME" in normalized_readme, "README must document the optional production VIF database name override.") -require("DEPLOY_VIF_DB_USER" in normalized_readme, "README must document the optional production VIF database user override.") -require("DEPLOY_SSH_USER=root" in normalized_readme, "README must document that the deploy workflow rejects root SSH users.") -require("makepad-postgres-vif" not in base_compose, "Base Compose file must not attach VIF; VIF is production-only.") -require("MAKEPAD_POSTGRES_VIF_DB_NETWORK" not in canary_compose, "Canary Compose override must not require the VIF network variable.") -require("makepad-postgres-vif" in production_compose, "Production Compose override must expose the VIF database alias.") -require("name: ${MAKEPAD_POSTGRES_VIF_DB_NETWORK}" in production_compose, "Production Compose override must map the VIF network variable.") -require("DEPLOY_SSH_USER must not be root" in manual_deploy, "Manual deploy workflow must reject root SSH users.") -require("DEPLOY_VIF_DB_NETWORK production environment secret" in manual_deploy, "Manual deploy workflow must require VIF network secret only for production.") -require('if [[ "${deploy_env}" == "production" ]]; then' in manual_deploy, "Manual deploy workflow must gate VIF setup to production.") -require('if [[ "${vif_enabled}" != "1" ]]; then' in manual_deploy, "Manual deploy workflow must skip VIF provisioning outside production.") -require("postgres_ready=0" in manual_deploy, "Manual deploy workflow must track Postgres readiness.") -require("Postgres did not become reachable via makepad-postgres-vif" in manual_deploy, "Manual deploy workflow must fail clearly when VIF readiness times out.") -require( - not re.search(r"\S\\gexec", manual_deploy), - "Manual deploy workflow must separate every VIF provisioning \\gexec command from SQL text by whitespace.", -) -require("ALTER ROLE %I LOGIN PASSWORD %L" in manual_deploy, "Manual deploy workflow must always refresh the VIF role password.") -require("ALTER DATABASE %I OWNER TO %I" in manual_deploy, "Manual deploy workflow must repair VIF database ownership drift.") -require( - sql.count("DO $$") == len(expected_instances), - "SQL bootstrap must use one DO block for each expected role.", -) -require( - sql.count("END;\n$$;") == len(expected_instances), - "Each SQL bootstrap DO block must terminate the PL/pgSQL block with END; before $$.", -) -require( - sql.count(r"\gexec") == 2 * len(expected_instances), - "SQL bootstrap must use psql gexec commands for conditional database creation and ownership repair.", -) -require( - not re.search(r"\S\\gexec", sql), - "Each SQL bootstrap \\gexec command must be separated from SQL text by whitespace.", -) - -for slug, expected in expected_instances.items(): - for field in ("role", "database", "password_variable"): - require(expected[field] in sql, f"SQL bootstrap is missing {expected[field]} for {slug}.") - require(f"CREATE ROLE {expected['role']} LOGIN" in sql, f"SQL bootstrap must create {slug} role idempotently.") - require(f"CREATE DATABASE {expected['database']} OWNER {expected['role']}" in sql, f"SQL bootstrap must create {slug} database.") - require(f"ALTER ROLE {expected['role']} LOGIN PASSWORD :'{expected['password_variable']}'" in sql, f"SQL bootstrap must set {slug} role password from a psql variable.") - require(f"ALTER DATABASE {expected['database']} OWNER TO {expected['role']}" in sql, f"SQL bootstrap must be able to repair {slug} database ownership.") - require(f"WHERE d.datname = '{expected['database']}'" in sql, f"SQL bootstrap must check current {slug} database ownership before altering it.") - require(f"r.rolname <> '{expected['role']}'" in sql, f"SQL bootstrap must avoid altering {slug} database ownership when it is already correct.") - require(f"{expected['password_variable']}_is_nonempty" in sql, f"SQL bootstrap must reject empty {slug} passwords.") - require(f"NULLIF(btrim(:'{expected['password_variable']}'), '')" in sql, f"SQL bootstrap must trim-check {slug} password emptiness.") - require(expected["database"] in normalized_readme, f"README is missing {expected['database']}.") - require(expected["role"] in normalized_readme, f"README is missing {expected['role']}.") - require(f"${{{expected['environment_variable']}:?" in readme, f"README bootstrap command must fail fast for {expected['environment_variable']}.") - -for literal in ("change-me", "password123"): - require(literal not in sql, f"SQL bootstrap must not contain literal {literal}.") - -for expected in ("runtrace_app", "runtrace", "runtrace_app_password"): - require(expected in runtrace_sql, f"Runtrace app SQL bootstrap is missing {expected}.") -require("PostgreSQL superuser connection" in runtrace_sql, "Runtrace app SQL bootstrap must document its superuser connection requirement.") -require("pg_advisory_lock" in runtrace_sql, "Runtrace app SQL bootstrap must serialize concurrent runs with an advisory lock.") -require("pg_advisory_unlock" in runtrace_sql, "Runtrace app SQL bootstrap must release its advisory lock after provisioning.") -require("ALTER ROLE runtrace_app LOGIN PASSWORD :'runtrace_app_password'" in runtrace_sql, "Runtrace app SQL bootstrap must set the role password from a psql variable.") -require("CREATE DATABASE runtrace OWNER runtrace_app" in runtrace_sql, "Runtrace app SQL bootstrap must create the Runtrace database.") -require("ALTER DATABASE runtrace OWNER TO runtrace_app" in runtrace_sql, "Runtrace app SQL bootstrap must repair Runtrace database ownership drift.") -require("NULLIF(btrim(:'runtrace_app_password'), '')" in runtrace_sql, "Runtrace app SQL bootstrap must reject empty passwords.") -require("runtrace_app" in normalized_readme, "README must document the Runtrace app role.") -require("keycloak_runtrace_app" in normalized_readme, "README must document the Runtrace Keycloak role.") -require("${RUNTRACE_DB_PASSWORD:?" in readme, "README bootstrap command must fail fast for RUNTRACE_DB_PASSWORD.") -require("${KEYCLOAK_RUNTRACE_DB_PASSWORD:?" in readme, "README bootstrap command must fail fast for KEYCLOAK_RUNTRACE_DB_PASSWORD.") +compose = (root / "compose.yml").read_text() +workflow = (root / ".github/workflows/manual-deploy.yml").read_text() +scraping_sql = (root / "bootstrap/scraping-app.sql").read_text() +catalog_reader_sql = (root / "bootstrap/fashion-catalog-reader.sql").read_text() +iceberg_sql = (root / "bootstrap/iceberg-catalog.sql").read_text() +developer_platform_sql = (root / "bootstrap/vestiaire-developer-platform.sql").read_text() + +require("network_mode: host" in compose, "Postgres compose must match the live host-network deployment.") +require("docker inspect \"${container_name}\"" in workflow, "Workflow must provision through the running DB container.") +require("docker stack deploy" not in workflow, "Workflow must not use Swarm for the DB VM.") +require("DEPLOY_SSH_USER must not be root" in workflow, "Workflow must reject root SSH users.") +require("DEPLOY_SCRAPING_DB_PASSWORD" in workflow, "Workflow must require the scraping DB password secret.") +require("DEPLOY_FASHION_CATALOG_READER_PASSWORD" in workflow, "Workflow must require the fashion catalog reader password secret.") +require("scraping_crawler" in scraping_sql, "Scraping bootstrap must create the scraping_crawler role.") +require("CREATE DATABASE scraping OWNER scraping_crawler" in scraping_sql, "Scraping bootstrap must create the scraping database.") +require("ALTER ROLE scraping_crawler LOGIN PASSWORD :'scraping_crawler_password'" in scraping_sql, "Scraping bootstrap must set the role password from a psql variable.") +require("NULLIF(btrim(:'scraping_crawler_password'), '')" in scraping_sql, "Scraping bootstrap must reject empty passwords.") +require("fashion_catalog_reader" in scraping_sql, "Scraping bootstrap must create the fashion catalog reader role.") +require("ALTER ROLE fashion_catalog_reader LOGIN PASSWORD :'fashion_catalog_reader_password'" in scraping_sql, "Scraping bootstrap must set the reader password from a psql variable.") +require("NULLIF(btrim(:'fashion_catalog_reader_password'), '')" in scraping_sql, "Scraping bootstrap must reject an empty reader password.") +require("GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader" in scraping_sql, "Reader role must receive SELECT on current scraping tables.") +require("ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler" in scraping_sql, "Reader role must receive SELECT on future scraping tables.") +require("pg_advisory_lock" in scraping_sql and "pg_advisory_unlock" in scraping_sql, "Scraping bootstrap must serialize concurrent runs.") +require("ALTER ROLE fashion_catalog_reader LOGIN PASSWORD :'fashion_catalog_reader_password'" in catalog_reader_sql, "Dedicated catalog reader bootstrap must set its password from a psql variable.") +require("GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader" in catalog_reader_sql, "Dedicated catalog reader bootstrap must grant read access to current tables.") +require("ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler" in catalog_reader_sql, "Dedicated catalog reader bootstrap must grant read access to future crawler tables.") +require("pg_advisory_lock" in catalog_reader_sql and "pg_advisory_unlock" in catalog_reader_sql, "Dedicated catalog reader bootstrap must serialize concurrent runs.") +require("iceberg_catalog" in iceberg_sql, "Iceberg bootstrap must create the Iceberg catalog role and database.") +require("NULLIF(btrim(:'iceberg_catalog_password'), '')" in iceberg_sql, "Iceberg bootstrap must reject empty passwords.") +require("pg_advisory_lock" in iceberg_sql and "pg_advisory_unlock" in iceberg_sql, "Iceberg bootstrap must serialize concurrent runs.") +require("vestiaire_developer_app" in developer_platform_sql, "Developer platform bootstrap must create its application role.") +require("CREATE DATABASE vestiaire_developer OWNER vestiaire_developer_app" in developer_platform_sql, "Developer platform bootstrap must create its database.") +require("ALTER ROLE vestiaire_developer_app" in developer_platform_sql and "PASSWORD :'vestiaire_developer_app_password'" in developer_platform_sql, "Developer platform bootstrap must set its password from a psql variable.") +require("NULLIF(btrim(:'vestiaire_developer_app_password'), '')" in developer_platform_sql, "Developer platform bootstrap must reject an empty password.") +require("pg_advisory_lock" in developer_platform_sql and "pg_advisory_unlock" in developer_platform_sql, "Developer platform bootstrap must serialize concurrent runs.") PY