From a7bafa9f89df3f51bc9ee13ce2fe596b95c766f7 Mon Sep 17 00:00:00 2001 From: DarKylX Date: Mon, 19 Oct 2026 12:12:39 +0300 Subject: [PATCH 1/2] ops: inspect DEV disk and prune unused build cache only --- .github/workflows/dev-ci.yml | 413 ++++++----------------------------- 1 file changed, 67 insertions(+), 346 deletions(-) diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml index c2afc0bd..9501269c 100644 --- a/.github/workflows/dev-ci.yml +++ b/.github/workflows/dev-ci.yml @@ -1,10 +1,18 @@ -name: Deploy Dev +name: DEV Results disk diagnostics and build cache recovery on: - push: - branches: - - dev workflow_dispatch: + inputs: + mode: + description: Read-only inspection or unused build cache only + type: choice + options: [inspect, cache-only] + default: inspect + required: true + confirmation: + description: RESULTS_DEV_CACHE + type: string + required: true permissions: contents: read @@ -14,345 +22,58 @@ concurrency: cancel-in-progress: false jobs: - deploy: - name: Deploy - if: github.ref == 'refs/heads/dev' - environment: dev - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Prepare isolated DEV migration credentials - id: migration_credentials - env: - DEV_DATABASE_NAME: ${{ secrets.DEV_DATABASE_NAME }} - DEV_DATABASE_USER: ${{ secrets.DEV_DATABASE_USER }} - DEV_MIGRATION_DATABASE_USER: ${{ secrets.DEV_MIGRATION_DATABASE_USER }} - DEV_MIGRATION_DATABASE_PASSWORD: ${{ secrets.DEV_MIGRATION_DATABASE_PASSWORD }} - run: python3 scripts/dev_ci_migrate.py --prepare-payload - - uses: actions/setup-node@v4 - with: - node-version: "24" - - name: Install DEV SSH transport - run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 - - name: Deploy to server - env: - NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules - DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} - DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} - DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} - DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} - DEV_DEPLOY_SCRIPT: | - set +x - set -eu - exec 2>&1 - - cd /root/api - git fetch --all --tags --prune - git branch -r --contains "${{ github.sha }}" | grep -q 'origin/dev' - git checkout --detach "${{ github.sha }}" - git reset --hard "${{ github.sha }}" - test "$(git rev-parse HEAD)" = "${{ github.sha }}" - if [ -n "$(git status --porcelain --untracked-files=no)" ]; then - echo "Tracked working tree is dirty after reset" >&2 - git status --short --untracked-files=no >&2 - exit 1 - fi - expected_serializer_blob="$(git rev-parse HEAD:vacancy/serializers.py)" - actual_serializer_blob="$(git hash-object vacancy/serializers.py)" - if [ "$actual_serializer_blob" != "$expected_serializer_blob" ]; then - echo "Server serializers.py does not match HEAD blob" >&2 - exit 1 - fi - - echo "Deploy source HEAD: $(git rev-parse HEAD)" - echo "vacancy/serializers.py blob: ${actual_serializer_blob}" - - export IMAGE_TAG="${{ github.sha }}" - SOURCE_IMAGE="procollab-dev-api:${IMAGE_TAG}" - export DEPLOY_IMAGE="$SOURCE_IMAGE" - - rm -f .env - touch .env - - echo "DJANGO_SECRET_KEY=${{ secrets.DEV_DJANGO_SECRET_KEY }}" >> .env - - echo "DATABASE_NAME=${{ secrets.DEV_DATABASE_NAME }}" >> .env - echo "DATABASE_PASSWORD=${{ secrets.DEV_DATABASE_PASSWORD }}" >> .env - echo "DATABASE_USER=${{ secrets.DEV_DATABASE_USER }}" >> .env - echo "DATABASE_HOST=${{ secrets.DEV_DATABASE_HOST }}" >> .env - echo "DATABASE_PORT=${{ secrets.DEV_DATABASE_PORT }}" >> .env - - echo "SELECTEL_ACCOUNT_ID=${{ secrets.SELECTEL_ACCOUNT_ID }}" >> .env - echo "SELECTEL_CONTAINER_NAME=${{ secrets.SELECTEL_CONTAINER_NAME }}" >> .env - echo "SELECTEL_CONTAINER_PASSWORD=${{ secrets.SELECTEL_CONTAINER_PASSWORD }}" >> .env - echo "SELECTEL_CONTAINER_USERNAME=${{ secrets.SELECTEL_CONTAINER_USERNAME }}" >> .env - - echo "EMAIL_USER=${{ secrets.EMAIL_USER }}" >> .env - echo "UNISENDER_GO_API_KEY=${{ secrets.UNISENDER_GO_API_KEY }}" >> .env - - chmod 600 .env - - docker compose -f docker-compose.dev-ci.yml config >/dev/null - docker compose -f docker-compose.dev-ci.yml build web - - BUILT_IMAGE_ID="$( - docker image inspect \ - -f '{{.Id}}' \ - "$SOURCE_IMAGE" - )" - - echo "Built image: ${SOURCE_IMAGE}" - echo "Built image ID: ${BUILT_IMAGE_ID}" - - case "$BUILT_IMAGE_ID" in - sha256:*) ;; - *) - echo "Invalid built image ID: ${BUILT_IMAGE_ID}" >&2 - exit 1 - ;; - esac - - BUILT_IMAGE_SHORT="$( - printf '%s' "${BUILT_IMAGE_ID#sha256:}" | - cut -c1-12 - )" - RUNTIME_IMAGE_TAG="${IMAGE_TAG}-${BUILT_IMAGE_SHORT}" - RUNTIME_IMAGE="procollab-dev-api:${RUNTIME_IMAGE_TAG}" - - docker image rm "$RUNTIME_IMAGE" >/dev/null 2>&1 || true - docker tag "$BUILT_IMAGE_ID" "$RUNTIME_IMAGE" - - RUNTIME_IMAGE_ID="$( - docker image inspect -f '{{.Id}}' "$RUNTIME_IMAGE" - )" - test "$RUNTIME_IMAGE_ID" = "$BUILT_IMAGE_ID" - - export DEPLOY_IMAGE="$RUNTIME_IMAGE" - - echo "Pinned runtime image: ${RUNTIME_IMAGE}" - echo "Pinned runtime image ID: ${RUNTIME_IMAGE_ID}" - docker compose -f docker-compose.dev-ci.yml config >/dev/null - docker compose -f docker-compose.dev-ci.yml config --format json | - docker run --rm -i \ - --entrypoint python \ - "$RUNTIME_IMAGE" \ - -c ' - import json - import sys - - expected = sys.argv[1] - services = json.load(sys.stdin)["services"] - for service_name in ("web", "celerys"): - actual = services[service_name]["image"] - print(f"Compose {service_name} image: {actual}") - if actual != expected: - raise SystemExit( - f"Compose {service_name} does not use {expected}" - ) - ' "$RUNTIME_IMAGE" - - BUILT_SERIALIZER_SHA="$( - docker run --rm \ - --entrypoint sha256sum \ - "$RUNTIME_IMAGE" \ - /procollab/vacancy/serializers.py | - awk '{print $1}' - )" - test -n "$BUILT_SERIALIZER_SHA" - echo "Built vacancy/serializers.py SHA-256: ${BUILT_SERIALIZER_SHA}" - - docker compose -f docker-compose.dev-ci.yml run --rm -T web python - <<'PY' - import os - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") - - import django - - django.setup() - - from vacancy.serializers import ( - ProjectVacancyCreateListSerializer, - VacancyDetailSerializer, - ) - - create_fields = ProjectVacancyCreateListSerializer.Meta.fields - detail_fields = VacancyDetailSerializer.Meta.fields - - print("Compose run create serializer fields:", create_fields) - print("Compose run detail serializer fields:", detail_fields) - - if "city" not in create_fields: - raise SystemExit("Compose run create serializer uses stale image") - - if "city" not in detail_fields: - raise SystemExit("Compose run detail serializer uses stale image") - PY - - docker compose -f docker-compose.dev-ci.yml run --rm -T --no-deps \ - --entrypoint python web scripts/dev_ci_migrate.py <<'DEV_MIGRATION_PAYLOAD' - ${{ steps.migration_credentials.outputs.payload }} - DEV_MIGRATION_PAYLOAD - docker compose -f docker-compose.dev-ci.yml run --rm -T --no-deps web \ - python manage.py migrate --check < /dev/null - docker compose -f docker-compose.dev-ci.yml run --rm -T web \ - python manage.py showmigrations vacancy < /dev/null - - docker compose -f docker-compose.dev-ci.yml rm -sf web celerys || true - docker compose -f docker-compose.dev-ci.yml up \ - -d \ - --no-build \ - --force-recreate - - for service in web celerys; do - container="$(docker compose -f docker-compose.dev-ci.yml ps -q "$service")" - if [ -z "$container" ]; then - echo "Service ${service} has no running container" >&2 - docker compose -f docker-compose.dev-ci.yml ps >&2 || true - exit 1 - fi - - configured_image="$(docker inspect -f '{{.Config.Image}}' "$container")" - actual_image_id="$(docker inspect -f '{{.Image}}' "$container")" - echo "Service ${service}: container=${container} reference=${configured_image} image_id=${actual_image_id}" - if [ "$actual_image_id" != "$BUILT_IMAGE_ID" ]; then - echo "Service ${service} uses wrong image ID" >&2 - echo "Expected: ${BUILT_IMAGE_ID}" >&2 - echo "Actual: ${actual_image_id}" >&2 - echo "Reference: ${configured_image}" >&2 - exit 1 - fi - done - - RUNNING_SERIALIZER_SHA="$( - docker compose -f docker-compose.dev-ci.yml exec -T web \ - sha256sum /procollab/vacancy/serializers.py | - awk '{print $1}' - )" - echo "Running vacancy/serializers.py SHA-256: ${RUNNING_SERIALIZER_SHA}" - if [ "$RUNNING_SERIALIZER_SHA" != "$BUILT_SERIALIZER_SHA" ]; then - echo "Running serializer source does not match built image" >&2 - echo "Expected: ${BUILT_SERIALIZER_SHA}" >&2 - echo "Actual: ${RUNNING_SERIALIZER_SHA}" >&2 - exit 1 - fi - - docker compose -f docker-compose.dev-ci.yml exec -T web python - <<'PY' - import os - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") - - import django - - django.setup() - - from django.db import connection - from django.db.migrations.recorder import MigrationRecorder - - applied = { - (migration.app, migration.name) - for migration in MigrationRecorder(connection).migration_qs - } - - if ("vacancy", "0010_vacancy_city") not in applied: - raise SystemExit("vacancy.0010_vacancy_city is not applied") - - print("vacancy.0010_vacancy_city applied") - PY - - docker compose -f docker-compose.dev-ci.yml exec -T web python - <<'PY' - import os - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") - - import django - - django.setup() - - from vacancy.serializers import ( - ProjectVacancyCreateListSerializer, - VacancyDetailSerializer, - ) - - create_fields = set(ProjectVacancyCreateListSerializer().fields.keys()) - detail_fields = set(VacancyDetailSerializer().fields.keys()) - - print("Runtime create serializer fields:", sorted(create_fields)) - print("Runtime detail serializer fields:", sorted(detail_fields)) - - if "city" not in create_fields: - raise SystemExit("Runtime create serializer does not contain city") - - if "city" not in detail_fields: - raise SystemExit("Runtime detail serializer does not contain city") - - print("Runtime serializers contain city") - PY - - install -d /etc/nginx/procollab/includes && - install -m 644 deploy/nginx/host/includes/proxy_app.inc /etc/nginx/procollab/includes/proxy_app.inc && - install -m 644 deploy/nginx/host/dev/dev.procollab.ru /etc/nginx/sites-available/dev.procollab.ru && - ln -sfn /etc/nginx/sites-available/dev.procollab.ru /etc/nginx/sites-enabled/dev.procollab.ru && - if [ "$(id -u)" -eq 0 ]; then - nginx -t && - systemctl reload nginx - else - sudo nginx -t && - sudo systemctl reload nginx - fi && - - for attempt in $(seq 1 24); do - admin_status="$(curl -s -o /dev/null -w '%{http_code}' https://dev.procollab.ru/admin/login/ || true)" && - - if [ "$admin_status" = "200" ]; then - echo "Smoke check passed on attempt ${attempt}" && - break - fi - - sleep 5 - done && - - if [ "$admin_status" != "200" ]; then - echo "Smoke check failed: /admin/login/=${admin_status}" >&2 && - exit 1 - fi && - - if ! python3 scripts/check_dev_routes.py https://dev.procollab.ru; then - echo "Frontend/API routing smoke check failed" >&2 - exit 1 - fi && - - docker compose -f docker-compose.dev-ci.yml ps - - celery_status="" - celery_ping="" - celery_container="" - for attempt in $(seq 1 12); do - celery_container="$(docker compose -f docker-compose.dev-ci.yml ps -q celerys 2>/dev/null || true)" - if [ -n "$celery_container" ]; then - celery_status="$(docker inspect -f '{{.State.Status}}' "$celery_container" 2>/dev/null || true)" - else - celery_status="missing" - fi - - echo "Celery check attempt ${attempt}: container=${celery_container:-missing} status=${celery_status}" - if [ "$celery_status" = "running" ]; then - celery_ping="$(docker compose -f docker-compose.dev-ci.yml exec -T celerys sh -lc 'celery -A procollab inspect ping --timeout=15' 2>&1 || true)" - printf '%s\n' "$celery_ping" - if printf '%s\n' "$celery_ping" | grep -q 'pong'; then - echo "Celery check passed on attempt ${attempt}" - break - fi - fi - - sleep 5 - done - - if [ "$celery_status" != "running" ] || ! printf '%s\n' "$celery_ping" | grep -q 'pong'; then - echo "Celery check failed: status=${celery_status}" >&2 - docker compose -f docker-compose.dev-ci.yml ps >&2 || true - docker compose -f docker-compose.dev-ci.yml logs --tail=200 celerys >&2 || true - docker compose -f docker-compose.dev-ci.yml logs --tail=100 redis >&2 || true - docker compose -f docker-compose.dev-ci.yml logs --tail=100 web >&2 || true - exit 1 - fi - run: node scripts/dev_ci_ssh.cjs + recovery: + if: github.ref == 'refs/heads/ops/dev-results-disk-recovery-20261019' && inputs.confirmation == 'RESULTS_DEV_CACHE' && (inputs.mode == 'inspect' || inputs.mode == 'cache-only') + environment: dev + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Install existing verified SSH transport + run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 + - name: Inspect DEV or remove unused build cache only + env: + NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules + DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} + DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} + DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} + DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} + DEV_DEPLOY_SCRIPT: | + set +x + set -eu + exec 2>&1 + test "$(id -u)" = 0 + test "$(readlink -f /root/api)" = /root/api + test "$(git -C /root/api rev-parse HEAD)" = beef71645567128398d0afa5a2eb80dc6bc8dac7 + grep -Eq 'server_name[[:space:]]+dev\.procollab\.ru;' /etc/nginx/sites-enabled/dev.procollab.ru + mode='${{ inputs.mode }}' + case "$mode" in inspect|cache-only) ;; *) exit 1;; esac + containers() { + for id in $(docker ps -aq --no-trunc); do + docker inspect -f '{{.Id}} {{.Image}} {{.State.Status}} {{.State.StartedAt}} {{.RestartCount}}' "$id" + done | LC_ALL=C sort | sha256sum + } + images() { docker image ls -a --no-trunc --digests --format '{{.ID}} {{.Repository}} {{.Tag}} {{.Digest}}' | LC_ALL=C sort | sha256sum; } + volumes() { docker volume ls --format '{{.Driver}} {{.Name}}' | LC_ALL=C sort | sha256sum; } + before_containers=$(containers) + before_images=$(images) + before_volumes=$(volumes) + printf 'DISK_BEFORE\n' + df -h / /home/front/app + df -i / + docker system df + du -xsh /var/lib/docker /var/log /root/api /home/front/app + if test "$mode" = cache-only; then + docker builder prune --all --force + fi + test "$(containers)" = "$before_containers" + test "$(images)" = "$before_images" + test "$(volumes)" = "$before_volumes" + test "$(git -C /root/api rev-parse HEAD)" = beef71645567128398d0afa5a2eb80dc6bc8dac7 + printf 'CONTAINERS_UNCHANGED=true\nIMAGES_UNCHANGED=true\nVOLUMES_UNCHANGED=true\nDISK_AFTER\n' + df -h / + docker system df + run: node scripts/dev_ci_ssh.cjs From ce99f17828bbcfa027facdb262aee6741036fab4 Mon Sep 17 00:00:00 2001 From: DarKylX Date: Mon, 19 Oct 2026 12:15:59 +0300 Subject: [PATCH 2/2] ops: run guarded cache recovery through protected DEV environment --- .github/workflows/dev-ci.yml | 413 +++++++++++++++--- .../workflows/dev-results-disk-recovery.yml | 85 ++++ 2 files changed, 431 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/dev-results-disk-recovery.yml diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml index 9501269c..c2afc0bd 100644 --- a/.github/workflows/dev-ci.yml +++ b/.github/workflows/dev-ci.yml @@ -1,18 +1,10 @@ -name: DEV Results disk diagnostics and build cache recovery +name: Deploy Dev on: + push: + branches: + - dev workflow_dispatch: - inputs: - mode: - description: Read-only inspection or unused build cache only - type: choice - options: [inspect, cache-only] - default: inspect - required: true - confirmation: - description: RESULTS_DEV_CACHE - type: string - required: true permissions: contents: read @@ -22,58 +14,345 @@ concurrency: cancel-in-progress: false jobs: - recovery: - if: github.ref == 'refs/heads/ops/dev-results-disk-recovery-20261019' && inputs.confirmation == 'RESULTS_DEV_CACHE' && (inputs.mode == 'inspect' || inputs.mode == 'cache-only') - environment: dev - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '24' - - name: Install existing verified SSH transport - run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 - - name: Inspect DEV or remove unused build cache only - env: - NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules - DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} - DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} - DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} - DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} - DEV_DEPLOY_SCRIPT: | - set +x - set -eu - exec 2>&1 - test "$(id -u)" = 0 - test "$(readlink -f /root/api)" = /root/api - test "$(git -C /root/api rev-parse HEAD)" = beef71645567128398d0afa5a2eb80dc6bc8dac7 - grep -Eq 'server_name[[:space:]]+dev\.procollab\.ru;' /etc/nginx/sites-enabled/dev.procollab.ru - mode='${{ inputs.mode }}' - case "$mode" in inspect|cache-only) ;; *) exit 1;; esac - containers() { - for id in $(docker ps -aq --no-trunc); do - docker inspect -f '{{.Id}} {{.Image}} {{.State.Status}} {{.State.StartedAt}} {{.RestartCount}}' "$id" - done | LC_ALL=C sort | sha256sum - } - images() { docker image ls -a --no-trunc --digests --format '{{.ID}} {{.Repository}} {{.Tag}} {{.Digest}}' | LC_ALL=C sort | sha256sum; } - volumes() { docker volume ls --format '{{.Driver}} {{.Name}}' | LC_ALL=C sort | sha256sum; } - before_containers=$(containers) - before_images=$(images) - before_volumes=$(volumes) - printf 'DISK_BEFORE\n' - df -h / /home/front/app - df -i / - docker system df - du -xsh /var/lib/docker /var/log /root/api /home/front/app - if test "$mode" = cache-only; then - docker builder prune --all --force - fi - test "$(containers)" = "$before_containers" - test "$(images)" = "$before_images" - test "$(volumes)" = "$before_volumes" - test "$(git -C /root/api rev-parse HEAD)" = beef71645567128398d0afa5a2eb80dc6bc8dac7 - printf 'CONTAINERS_UNCHANGED=true\nIMAGES_UNCHANGED=true\nVOLUMES_UNCHANGED=true\nDISK_AFTER\n' - df -h / - docker system df - run: node scripts/dev_ci_ssh.cjs + deploy: + name: Deploy + if: github.ref == 'refs/heads/dev' + environment: dev + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Prepare isolated DEV migration credentials + id: migration_credentials + env: + DEV_DATABASE_NAME: ${{ secrets.DEV_DATABASE_NAME }} + DEV_DATABASE_USER: ${{ secrets.DEV_DATABASE_USER }} + DEV_MIGRATION_DATABASE_USER: ${{ secrets.DEV_MIGRATION_DATABASE_USER }} + DEV_MIGRATION_DATABASE_PASSWORD: ${{ secrets.DEV_MIGRATION_DATABASE_PASSWORD }} + run: python3 scripts/dev_ci_migrate.py --prepare-payload + - uses: actions/setup-node@v4 + with: + node-version: "24" + - name: Install DEV SSH transport + run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 + - name: Deploy to server + env: + NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules + DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} + DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} + DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} + DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} + DEV_DEPLOY_SCRIPT: | + set +x + set -eu + exec 2>&1 + + cd /root/api + git fetch --all --tags --prune + git branch -r --contains "${{ github.sha }}" | grep -q 'origin/dev' + git checkout --detach "${{ github.sha }}" + git reset --hard "${{ github.sha }}" + test "$(git rev-parse HEAD)" = "${{ github.sha }}" + if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + echo "Tracked working tree is dirty after reset" >&2 + git status --short --untracked-files=no >&2 + exit 1 + fi + expected_serializer_blob="$(git rev-parse HEAD:vacancy/serializers.py)" + actual_serializer_blob="$(git hash-object vacancy/serializers.py)" + if [ "$actual_serializer_blob" != "$expected_serializer_blob" ]; then + echo "Server serializers.py does not match HEAD blob" >&2 + exit 1 + fi + + echo "Deploy source HEAD: $(git rev-parse HEAD)" + echo "vacancy/serializers.py blob: ${actual_serializer_blob}" + + export IMAGE_TAG="${{ github.sha }}" + SOURCE_IMAGE="procollab-dev-api:${IMAGE_TAG}" + export DEPLOY_IMAGE="$SOURCE_IMAGE" + + rm -f .env + touch .env + + echo "DJANGO_SECRET_KEY=${{ secrets.DEV_DJANGO_SECRET_KEY }}" >> .env + + echo "DATABASE_NAME=${{ secrets.DEV_DATABASE_NAME }}" >> .env + echo "DATABASE_PASSWORD=${{ secrets.DEV_DATABASE_PASSWORD }}" >> .env + echo "DATABASE_USER=${{ secrets.DEV_DATABASE_USER }}" >> .env + echo "DATABASE_HOST=${{ secrets.DEV_DATABASE_HOST }}" >> .env + echo "DATABASE_PORT=${{ secrets.DEV_DATABASE_PORT }}" >> .env + + echo "SELECTEL_ACCOUNT_ID=${{ secrets.SELECTEL_ACCOUNT_ID }}" >> .env + echo "SELECTEL_CONTAINER_NAME=${{ secrets.SELECTEL_CONTAINER_NAME }}" >> .env + echo "SELECTEL_CONTAINER_PASSWORD=${{ secrets.SELECTEL_CONTAINER_PASSWORD }}" >> .env + echo "SELECTEL_CONTAINER_USERNAME=${{ secrets.SELECTEL_CONTAINER_USERNAME }}" >> .env + + echo "EMAIL_USER=${{ secrets.EMAIL_USER }}" >> .env + echo "UNISENDER_GO_API_KEY=${{ secrets.UNISENDER_GO_API_KEY }}" >> .env + + chmod 600 .env + + docker compose -f docker-compose.dev-ci.yml config >/dev/null + docker compose -f docker-compose.dev-ci.yml build web + + BUILT_IMAGE_ID="$( + docker image inspect \ + -f '{{.Id}}' \ + "$SOURCE_IMAGE" + )" + + echo "Built image: ${SOURCE_IMAGE}" + echo "Built image ID: ${BUILT_IMAGE_ID}" + + case "$BUILT_IMAGE_ID" in + sha256:*) ;; + *) + echo "Invalid built image ID: ${BUILT_IMAGE_ID}" >&2 + exit 1 + ;; + esac + + BUILT_IMAGE_SHORT="$( + printf '%s' "${BUILT_IMAGE_ID#sha256:}" | + cut -c1-12 + )" + RUNTIME_IMAGE_TAG="${IMAGE_TAG}-${BUILT_IMAGE_SHORT}" + RUNTIME_IMAGE="procollab-dev-api:${RUNTIME_IMAGE_TAG}" + + docker image rm "$RUNTIME_IMAGE" >/dev/null 2>&1 || true + docker tag "$BUILT_IMAGE_ID" "$RUNTIME_IMAGE" + + RUNTIME_IMAGE_ID="$( + docker image inspect -f '{{.Id}}' "$RUNTIME_IMAGE" + )" + test "$RUNTIME_IMAGE_ID" = "$BUILT_IMAGE_ID" + + export DEPLOY_IMAGE="$RUNTIME_IMAGE" + + echo "Pinned runtime image: ${RUNTIME_IMAGE}" + echo "Pinned runtime image ID: ${RUNTIME_IMAGE_ID}" + docker compose -f docker-compose.dev-ci.yml config >/dev/null + docker compose -f docker-compose.dev-ci.yml config --format json | + docker run --rm -i \ + --entrypoint python \ + "$RUNTIME_IMAGE" \ + -c ' + import json + import sys + + expected = sys.argv[1] + services = json.load(sys.stdin)["services"] + for service_name in ("web", "celerys"): + actual = services[service_name]["image"] + print(f"Compose {service_name} image: {actual}") + if actual != expected: + raise SystemExit( + f"Compose {service_name} does not use {expected}" + ) + ' "$RUNTIME_IMAGE" + + BUILT_SERIALIZER_SHA="$( + docker run --rm \ + --entrypoint sha256sum \ + "$RUNTIME_IMAGE" \ + /procollab/vacancy/serializers.py | + awk '{print $1}' + )" + test -n "$BUILT_SERIALIZER_SHA" + echo "Built vacancy/serializers.py SHA-256: ${BUILT_SERIALIZER_SHA}" + + docker compose -f docker-compose.dev-ci.yml run --rm -T web python - <<'PY' + import os + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") + + import django + + django.setup() + + from vacancy.serializers import ( + ProjectVacancyCreateListSerializer, + VacancyDetailSerializer, + ) + + create_fields = ProjectVacancyCreateListSerializer.Meta.fields + detail_fields = VacancyDetailSerializer.Meta.fields + + print("Compose run create serializer fields:", create_fields) + print("Compose run detail serializer fields:", detail_fields) + + if "city" not in create_fields: + raise SystemExit("Compose run create serializer uses stale image") + + if "city" not in detail_fields: + raise SystemExit("Compose run detail serializer uses stale image") + PY + + docker compose -f docker-compose.dev-ci.yml run --rm -T --no-deps \ + --entrypoint python web scripts/dev_ci_migrate.py <<'DEV_MIGRATION_PAYLOAD' + ${{ steps.migration_credentials.outputs.payload }} + DEV_MIGRATION_PAYLOAD + docker compose -f docker-compose.dev-ci.yml run --rm -T --no-deps web \ + python manage.py migrate --check < /dev/null + docker compose -f docker-compose.dev-ci.yml run --rm -T web \ + python manage.py showmigrations vacancy < /dev/null + + docker compose -f docker-compose.dev-ci.yml rm -sf web celerys || true + docker compose -f docker-compose.dev-ci.yml up \ + -d \ + --no-build \ + --force-recreate + + for service in web celerys; do + container="$(docker compose -f docker-compose.dev-ci.yml ps -q "$service")" + if [ -z "$container" ]; then + echo "Service ${service} has no running container" >&2 + docker compose -f docker-compose.dev-ci.yml ps >&2 || true + exit 1 + fi + + configured_image="$(docker inspect -f '{{.Config.Image}}' "$container")" + actual_image_id="$(docker inspect -f '{{.Image}}' "$container")" + echo "Service ${service}: container=${container} reference=${configured_image} image_id=${actual_image_id}" + if [ "$actual_image_id" != "$BUILT_IMAGE_ID" ]; then + echo "Service ${service} uses wrong image ID" >&2 + echo "Expected: ${BUILT_IMAGE_ID}" >&2 + echo "Actual: ${actual_image_id}" >&2 + echo "Reference: ${configured_image}" >&2 + exit 1 + fi + done + + RUNNING_SERIALIZER_SHA="$( + docker compose -f docker-compose.dev-ci.yml exec -T web \ + sha256sum /procollab/vacancy/serializers.py | + awk '{print $1}' + )" + echo "Running vacancy/serializers.py SHA-256: ${RUNNING_SERIALIZER_SHA}" + if [ "$RUNNING_SERIALIZER_SHA" != "$BUILT_SERIALIZER_SHA" ]; then + echo "Running serializer source does not match built image" >&2 + echo "Expected: ${BUILT_SERIALIZER_SHA}" >&2 + echo "Actual: ${RUNNING_SERIALIZER_SHA}" >&2 + exit 1 + fi + + docker compose -f docker-compose.dev-ci.yml exec -T web python - <<'PY' + import os + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") + + import django + + django.setup() + + from django.db import connection + from django.db.migrations.recorder import MigrationRecorder + + applied = { + (migration.app, migration.name) + for migration in MigrationRecorder(connection).migration_qs + } + + if ("vacancy", "0010_vacancy_city") not in applied: + raise SystemExit("vacancy.0010_vacancy_city is not applied") + + print("vacancy.0010_vacancy_city applied") + PY + + docker compose -f docker-compose.dev-ci.yml exec -T web python - <<'PY' + import os + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "procollab.settings") + + import django + + django.setup() + + from vacancy.serializers import ( + ProjectVacancyCreateListSerializer, + VacancyDetailSerializer, + ) + + create_fields = set(ProjectVacancyCreateListSerializer().fields.keys()) + detail_fields = set(VacancyDetailSerializer().fields.keys()) + + print("Runtime create serializer fields:", sorted(create_fields)) + print("Runtime detail serializer fields:", sorted(detail_fields)) + + if "city" not in create_fields: + raise SystemExit("Runtime create serializer does not contain city") + + if "city" not in detail_fields: + raise SystemExit("Runtime detail serializer does not contain city") + + print("Runtime serializers contain city") + PY + + install -d /etc/nginx/procollab/includes && + install -m 644 deploy/nginx/host/includes/proxy_app.inc /etc/nginx/procollab/includes/proxy_app.inc && + install -m 644 deploy/nginx/host/dev/dev.procollab.ru /etc/nginx/sites-available/dev.procollab.ru && + ln -sfn /etc/nginx/sites-available/dev.procollab.ru /etc/nginx/sites-enabled/dev.procollab.ru && + if [ "$(id -u)" -eq 0 ]; then + nginx -t && + systemctl reload nginx + else + sudo nginx -t && + sudo systemctl reload nginx + fi && + + for attempt in $(seq 1 24); do + admin_status="$(curl -s -o /dev/null -w '%{http_code}' https://dev.procollab.ru/admin/login/ || true)" && + + if [ "$admin_status" = "200" ]; then + echo "Smoke check passed on attempt ${attempt}" && + break + fi + + sleep 5 + done && + + if [ "$admin_status" != "200" ]; then + echo "Smoke check failed: /admin/login/=${admin_status}" >&2 && + exit 1 + fi && + + if ! python3 scripts/check_dev_routes.py https://dev.procollab.ru; then + echo "Frontend/API routing smoke check failed" >&2 + exit 1 + fi && + + docker compose -f docker-compose.dev-ci.yml ps + + celery_status="" + celery_ping="" + celery_container="" + for attempt in $(seq 1 12); do + celery_container="$(docker compose -f docker-compose.dev-ci.yml ps -q celerys 2>/dev/null || true)" + if [ -n "$celery_container" ]; then + celery_status="$(docker inspect -f '{{.State.Status}}' "$celery_container" 2>/dev/null || true)" + else + celery_status="missing" + fi + + echo "Celery check attempt ${attempt}: container=${celery_container:-missing} status=${celery_status}" + if [ "$celery_status" = "running" ]; then + celery_ping="$(docker compose -f docker-compose.dev-ci.yml exec -T celerys sh -lc 'celery -A procollab inspect ping --timeout=15' 2>&1 || true)" + printf '%s\n' "$celery_ping" + if printf '%s\n' "$celery_ping" | grep -q 'pong'; then + echo "Celery check passed on attempt ${attempt}" + break + fi + fi + + sleep 5 + done + + if [ "$celery_status" != "running" ] || ! printf '%s\n' "$celery_ping" | grep -q 'pong'; then + echo "Celery check failed: status=${celery_status}" >&2 + docker compose -f docker-compose.dev-ci.yml ps >&2 || true + docker compose -f docker-compose.dev-ci.yml logs --tail=200 celerys >&2 || true + docker compose -f docker-compose.dev-ci.yml logs --tail=100 redis >&2 || true + docker compose -f docker-compose.dev-ci.yml logs --tail=100 web >&2 || true + exit 1 + fi + run: node scripts/dev_ci_ssh.cjs diff --git a/.github/workflows/dev-results-disk-recovery.yml b/.github/workflows/dev-results-disk-recovery.yml new file mode 100644 index 00000000..9af2c1c3 --- /dev/null +++ b/.github/workflows/dev-results-disk-recovery.yml @@ -0,0 +1,85 @@ +name: DEV Results disk diagnostics and build cache recovery + +on: + push: + branches: [dev] + paths: [.github/workflows/dev-results-disk-recovery.yml] + workflow_dispatch: + inputs: + mode: + description: Read-only inspection or unused build cache only + type: choice + options: [inspect, cache-only] + default: inspect + required: true + confirmation: + description: RESULTS_DEV_CACHE + type: string + required: true + +permissions: + contents: read + +concurrency: + group: api-deploy-dev + cancel-in-progress: false + +jobs: + recovery: + if: github.ref == 'refs/heads/dev' && (github.event_name == 'push' || (inputs.confirmation == 'RESULTS_DEV_CACHE' && (inputs.mode == 'inspect' || inputs.mode == 'cache-only'))) + environment: dev + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Install existing verified SSH transport + run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 + - name: Inspect DEV or remove unused build cache only + env: + NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules + DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} + DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} + DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} + DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} + DEV_DEPLOY_SCRIPT: | + set +x + set -euo pipefail + exec 2>&1 + test "$(id -u)" = 0 + test "$(readlink -f /root/api)" = /root/api + git -C /root/api merge-base --is-ancestor beef71645567128398d0afa5a2eb80dc6bc8dac7 HEAD + grep -Eq 'server_name[[:space:]]+dev\.procollab\.ru;' /etc/nginx/sites-enabled/dev.procollab.ru + container=$(docker ps -q --no-trunc --filter 'label=com.docker.compose.project.working_dir=/root/api' --filter 'label=com.docker.compose.service=web') + test -n "$container" + test "$(docker inspect -f '{{.Image}}' "$container")" = sha256:471421701ce5fe4741eded480dfd94aa628398f307791fd78c59547d768a1e81 + mode='${{ github.event_name == 'push' && 'inspect' || inputs.mode }}' + case "$mode" in inspect|cache-only) ;; *) exit 1;; esac + containers() { + for id in $(docker ps -aq --no-trunc); do + docker inspect -f '{{.Id}} {{.Image}} {{.State.Status}} {{.State.StartedAt}} {{.RestartCount}}' "$id" + done | LC_ALL=C sort | sha256sum + } + images() { docker image ls -a --no-trunc --digests --format '{{.ID}} {{.Repository}} {{.Tag}} {{.Digest}}' | LC_ALL=C sort | sha256sum; } + volumes() { docker volume ls --format '{{.Driver}} {{.Name}}' | LC_ALL=C sort | sha256sum; } + before_containers=$(containers) + before_images=$(images) + before_volumes=$(volumes) + printf 'DISK_BEFORE\n' + df -h / /home/front/app + df -i / + docker system df + du -xsh /var/lib/docker /var/log /root/api /home/front/app + if test "$mode" = cache-only; then + docker builder prune --all --force + fi + test "$(containers)" = "$before_containers" + test "$(images)" = "$before_images" + test "$(volumes)" = "$before_volumes" + git -C /root/api merge-base --is-ancestor beef71645567128398d0afa5a2eb80dc6bc8dac7 HEAD + printf 'CONTAINERS_UNCHANGED=true\nIMAGES_UNCHANGED=true\nVOLUMES_UNCHANGED=true\nDISK_AFTER\n' + df -h / + docker system df + run: node scripts/dev_ci_ssh.cjs