Добавить резервную копию и проверку данных dev перед обновлением зависимостей

.github/scripts/dependency_dev_preflight.py

@@ -0,0 +1,293 @@
+"""DEV-only read-only data checks and a protected, validated database backup."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+from datetime import datetime, timezone
+from uuid import uuid4
+
+REPO = Path("/root/api")
+BACKUPS = Path("/root/procollab-dependency-backups")
+KEY = re.compile(r"dependency-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}")
+
+PROBE = r"""
+import hashlib, json, os
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procollab.settings')
+import django
+django.setup()
+from django.db import connection, transaction
+payload = {'django': django.get_version(), 'tables': {}}
+with transaction.atomic():
+    with connection.cursor() as cursor:
+        cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY')
+        cursor.execute("SET LOCAL statement_timeout = '60s'")
+        cursor.execute("SET LOCAL lock_timeout = '10s'")
+        cursor.execute(
+            "SELECT current_database(), current_setting('server_version_num')::int, "
+            "pg_database_size(current_database())"
+        )
+        name, version, size = cursor.fetchone()
+        if name != 'procollab_db_dev' or not 140000 <= version < 190000:
+            raise RuntimeError('Unexpected DEV database or unsupported PostgreSQL version')
+        payload.update(server_major=version // 10000, database_bytes=size)
+        cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename")
+        tables = [
+            row[0] for row in cursor.fetchall()
+            if row[0].startswith(
+                ('users_', 'projects_', 'partner_programs_', 'project_rates_', 'files_')
+            )
+        ]
+        for table in tables:
+            cursor.execute(
+                'SELECT a.attname FROM pg_index i JOIN pg_attribute a '
+                'ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) '
+                'WHERE i.indrelid=to_regclass(%s) AND i.indisprimary ORDER BY a.attnum',
+                [table],
+            )
+            primary = [row[0] for row in cursor.fetchall()]
+            if not primary:
+                raise RuntimeError('Missing primary key in a protected table')
+            identity = 'jsonb_build_array(' + ','.join(
+                't.' + connection.ops.quote_name(key) for key in primary
+            ) + ')::text'
+            record = "(to_jsonb(t) - 'last_activity' - 'last_login' - 'datetime_updated')::text"
+            hash_sql = lambda expression: "encode(sha256(convert_to(" + expression + ", 'UTF8')), 'hex')"
+            cursor.execute(
+                'SELECT ' + hash_sql(identity) + ',' + hash_sql(record)
+                + ' FROM ' + connection.ops.quote_name(table) + ' t'
+            )
+            records = {}
+            while True:
+                batch = cursor.fetchmany(500)
+                if not batch:
+                    break
+                for identity, record_hash in batch:
+                    records[identity] = record_hash
+            payload['tables'][table] = records
+print(json.dumps(payload, ensure_ascii=False))
+"""
+
+
+def emit(message, *, file=sys.stdout, flush=True):
+    file.write(message + "\n")
+    if flush:
+        file.flush()
+
+
+def command(args, **kwargs):
+    try:
+        return subprocess.run(args, check=True, stderr=subprocess.PIPE, **kwargs)
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        raise RuntimeError(
+            "A DEV backup/check command failed; no credentials are printed"
+        ) from None
+
+
+def output(args, **kwargs):
+    return command(args, stdout=subprocess.PIPE, **kwargs).stdout
+
+
+def inspect_container(container):
+    value = json.loads(output(["docker", "inspect", container]))[0]
+    labels = value["Config"].get("Labels", {})
+    paths = labels.get("com.docker.compose.project.config_files", "").split(",")
+    if str(REPO / "docker-compose.dev-ci.yml") not in paths:
+        raise RuntimeError("Container does not belong to the DEV compose configuration")
+    return value
+
+
+def snapshot(web):
+    return json.loads(
+        output(
+            ["docker", "exec", "-i", web, "python", "-"],
+            input=PROBE.encode(),
+            timeout=180,
+        )
+    )
+
+
+def digest(path):
+    value = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+
+def main():
+    phase = sys.argv[1]
+    key = sys.argv[2] if len(sys.argv) > 2 else ""
+    if phase not in ("backup", "verify") or REPO.resolve() != REPO:
+        raise RuntimeError("Invalid operation or repository location")
+    os.umask(0o077)
+    os.chdir(REPO)
+    web = output(
+        ["docker", "compose", "-f", "docker-compose.dev-ci.yml", "ps", "-q", "web"],
+        text=True,
+    ).strip()
+    worker = output(
+        ["docker", "compose", "-f", "docker-compose.dev-ci.yml", "ps", "-q", "celerys"],
+        text=True,
+    ).strip()
+    if not re.fullmatch(r"[a-f0-9]{12,64}", web) or not re.fullmatch(
+        r"[a-f0-9]{12,64}", worker
+    ):
+        raise RuntimeError("Expected exactly one running DEV web and worker")
+    runtime = inspect_container(web)
+    worker_runtime = inspect_container(worker)
+    if runtime["Image"] != worker_runtime["Image"]:
+        raise RuntimeError("DEV web and worker use different images")
+    current = snapshot(web)
+    emit("POSTGRESQL_MAJOR=" + str(current["server_major"]), flush=True)
+    emit("DJANGO_RUNTIME=" + current["django"], flush=True)
+
+    if phase == "backup":
+        required = max(2 * 1024**3, current["database_bytes"] * 2 + 512 * 1024**2)
+        if shutil.disk_usage("/root").free < required:
+            raise RuntimeError(
+                "Insufficient free space for a protected DEV backup and rollout"
+            )
+        key = (
+            "dependency-"
+            + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+            + "-"
+            + uuid4().hex[:8]
+        )
+        BACKUPS.mkdir(mode=0o700, exist_ok=True)
+        folder = BACKUPS / key
+        folder.mkdir(mode=0o700)
+        dump = folder / "database.dump"
+        environment_script = (
+            "import json,os; print(json.dumps({k:os.environ[k] for k in "
+            '("DATABASE_HOST","DATABASE_PORT","DATABASE_NAME",'
+            '"DATABASE_USER","DATABASE_PASSWORD")}))'
+        )
+        config = json.loads(
+            output(["docker", "exec", web, "python", "-c", environment_script])
+        )
+        if config["DATABASE_NAME"] != "procollab_db_dev":
+            raise RuntimeError("Unexpected DEV database configuration")
+        environment = dict(os.environ)
+        for destination, source in (
+            ("PGHOST", "DATABASE_HOST"),
+            ("PGPORT", "DATABASE_PORT"),
+            ("PGDATABASE", "DATABASE_NAME"),
+            ("PGUSER", "DATABASE_USER"),
+            ("PGPASSWORD", "DATABASE_PASSWORD"),
+        ):
+            environment[destination] = config[source]
+        client = "postgres:" + str(current["server_major"]) + "-alpine"
+        container_name = "dependency-backup-" + uuid4().hex[:12]
+        try:
+            with dump.open("xb") as handle:
+                command(
+                    [
+                        "docker",
+                        "run",
+                        "--rm",
+                        "--name",
+                        container_name,
+                        "--network",
+                        "container:" + web,
+                        "--env",
+                        "PGHOST",
+                        "--env",
+                        "PGPORT",
+                        "--env",
+                        "PGDATABASE",
+                        "--env",
+                        "PGUSER",
+                        "--env",
+                        "PGPASSWORD",
+                        client,
+                        "pg_dump",
+                        "--format=custom",
+                        "--no-owner",
+                        "--no-acl",
+                        "--lock-wait-timeout=30s",
+                    ],
+                    env=environment,
+                    stdout=handle,
+                    timeout=600,
+                )
+        finally:
+            subprocess.run(
+                ["docker", "rm", "-f", container_name],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        with dump.open("rb") as handle:
+            if handle.read(5) != b"PGDMP":
+                raise RuntimeError("Invalid backup archive")
+            handle.seek(0)
+            command(
+                ["docker", "run", "--rm", "-i", client, "pg_restore", "--file=/dev/null"],
+                stdin=handle,
+                stdout=subprocess.DEVNULL,
+                timeout=600,
+            )
+        rollback = "procollab-dependency-rollback:" + key
+        command(["docker", "tag", runtime["Image"], rollback], stdout=subprocess.DEVNULL)
+        shutil.copyfile(REPO / ".env", folder / "runtime.env")
+        os.chmod(folder / "runtime.env", 0o600)
+        metadata = {
+            "snapshot": current,
+            "image_id": runtime["Image"],
+            "image": runtime["Config"]["Image"],
+            "rollback_tag": rollback,
+            "source_sha": output(["git", "rev-parse", "HEAD"], text=True).strip(),
+            "dump_sha256": digest(dump),
+            "dump_bytes": dump.stat().st_size,
+        }
+        (folder / "manifest.json").write_text(json.dumps(metadata), encoding="utf8")
+        emit("BACKUP_KEY=" + key)
+        emit("BACKUP_BYTES=" + str(metadata["dump_bytes"]))
+        emit("BACKUP_SHA256=" + metadata["dump_sha256"])
+        emit("ROLLBACK_IMAGE=" + rollback)
+    else:
+        if KEY.fullmatch(key) is None:
+            raise RuntimeError("Invalid backup key")
+        folder = BACKUPS / key
+        if folder.resolve().parent != BACKUPS.resolve():
+            raise RuntimeError("Invalid backup location")
+        before = json.loads((folder / "manifest.json").read_text(encoding="utf8"))
+        if digest(folder / "database.dump") != before["dump_sha256"]:
+            raise RuntimeError("Backup digest mismatch")
+        removed, changed = {}, {}
+        for table, records in before["snapshot"]["tables"].items():
+            after = current["tables"].get(table, {})
+            missing = set(records) - set(after)
+            different = [
+                identity
+                for identity in set(records) & set(after)
+                if records[identity] != after[identity]
+            ]
+            if missing:
+                removed[table] = len(missing)
+            if different:
+                changed[table] = len(different)
+        emit("REMOVED_EXISTING_RECORDS=" + json.dumps(removed))
+        emit("CHANGED_EXISTING_RECORDS=" + json.dumps(changed))
+        if removed or changed:
+            raise RuntimeError("Existing DEV records changed; inspect before continuing")
+        emit("EXISTING_DEV_DATA_PRESERVED=true")
+        emit("CURRENT_IMAGE=" + runtime["Config"]["Image"])
+    emit(
+        "PROTECTED_TABLE_COUNTS="
+        + json.dumps({name: len(rows) for name, rows in current["tables"].items()})
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as error:
+        emit("DEV_DEPENDENCY_PREFLIGHT_FAILED=" + type(error).__name__, file=sys.stderr)
+        if isinstance(error, RuntimeError):
+            emit(str(error), file=sys.stderr)
+        raise SystemExit(1)

.github/workflows/dev-ci.yml

@@ -4,13 +4,20 @@ on:
   push:
     branches: 
       - dev
+    paths-ignore:
+      - .github/workflows/dev-ci.yml
+      - .github/scripts/dependency_dev_preflight.py
   workflow_dispatch:
     inputs:
       operation:
         description: "DEV operation"
         type: choice
         default: deploy
-        options: [deploy, verify_solution_rollback, prepare_solution_browser, finalize_solution_browser, verify_program_publication, grant_program_creator_11]
+        options: [deploy, verify_solution_rollback, prepare_solution_browser, finalize_solution_browser, verify_program_publication, grant_program_creator_11, dependency_backup, dependency_verify]
+      dependency_backup_key:
+        description: "Ключ резервной копии для проверки сохранности данных dev"
+        type: string
+        default: ''
       fixture_key:
         description: "Unique acceptance fixture key"
         default: release-20261026
@@ -372,7 +379,7 @@ jobs:
           run: node scripts/dev_ci_ssh.cjs
   solution-acceptance:
     name: Solution acceptance
-    if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && inputs.operation != 'deploy' && inputs.operation != ''
+    if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && inputs.operation != 'deploy' && inputs.operation != '' && inputs.operation != 'dependency_backup' && inputs.operation != 'dependency_verify'
     environment: dev
     runs-on: ubuntu-latest
     timeout-minutes: 12
@@ -434,3 +441,45 @@ jobs:
             timeoutMs: 9 * 60 * 1000
           }).catch(() => { console.error('DEV solution acceptance failed; inspect masked output.'); process.exitCode = 1; });
           NODE
+
+  dependency-preflight:
+    if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && (inputs.operation == 'dependency_backup' || inputs.operation == 'dependency_verify')
+    environment: dev
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+      - name: Install pinned SSH transport
+        run: npm install --prefix "$RUNNER_TEMP/dependency-dev-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund [email protected]
+      - name: Back up or verify DEV data
+        env:
+          NODE_PATH: ${{ runner.temp }}/dependency-dev-ssh/node_modules
+          DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }}
+          DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }}
+          DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }}
+          DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }}
+          PREFLIGHT_PHASE: ${{ inputs.operation == 'dependency_backup' && 'backup' || 'verify' }}
+          PREFLIGHT_BACKUP_KEY: ${{ inputs.dependency_backup_key }}
+        run: |
+          node <<'NODE'
+          const fs = require('node:fs');
+          const { deploy } = require('./scripts/dev_ci_ssh.cjs');
+          const phase = process.env.PREFLIGHT_PHASE;
+          const key = process.env.PREFLIGHT_BACKUP_KEY || '';
+          if (!['backup', 'verify'].includes(phase) || (key && !/^dependency-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}$/.test(key))) throw new Error('Invalid preflight arguments');
+          const source = fs.readFileSync('.github/scripts/dependency_dev_preflight.py', 'utf8');
+          const script = [
+            'set +x', 'set -eu', 'cd /root/api',
+            'test "$(readlink -f /root/api)" = /root/api',
+            'grep -Eq "server_name[[:space:]]+dev\\.procollab\\.ru;" /etc/nginx/sites-enabled/dev.procollab.ru',
+            `python3 - '${phase}' '${key}' <<'DEPENDENCY_DEV_PY'`,
+            source, 'DEPENDENCY_DEV_PY',
+          ].join('\n') + '\n';
+          deploy({ host: process.env.DEV_SERVER_HOST, username: process.env.DEV_SERVER_USER,
+            password: process.env.DEV_SERVER_PASSWORD, hostFingerprint: process.env.DEV_SSH_HOST_KEY_SHA256,
+            script, timeoutMs: 22 * 60 * 1000,
+          }).catch(() => { console.error('DEV dependency preflight failed; inspect protected output.'); process.exitCode = 1; });
+          NODE