From ecde2889fd14239e88f98ec85b9f3628b5381aad Mon Sep 17 00:00:00 2001 From: Fedor Date: Tue, 3 Nov 2026 00:13:56 +0300 Subject: [PATCH] ops: add guarded DEV backup and data verification commands --- .github/scripts/dependency_dev_preflight.py | 293 ++++++++++++++++++++ .github/workflows/dev-ci.yml | 53 +++- 2 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/dependency_dev_preflight.py diff --git a/.github/scripts/dependency_dev_preflight.py b/.github/scripts/dependency_dev_preflight.py new file mode 100644 index 00000000..7152117d --- /dev/null +++ b/.github/scripts/dependency_dev_preflight.py @@ -0,0 +1,293 @@ +"""DEV-only read-only data checks and a protected, validated database backup.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from uuid import uuid4 + +REPO = Path("/root/api") +BACKUPS = Path("/root/procollab-dependency-backups") +KEY = re.compile(r"dependency-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}") + +PROBE = r""" +import hashlib, json, os +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procollab.settings') +import django +django.setup() +from django.db import connection, transaction +payload = {'django': django.get_version(), 'tables': {}} +with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') + cursor.execute("SET LOCAL statement_timeout = '60s'") + cursor.execute("SET LOCAL lock_timeout = '10s'") + cursor.execute( + "SELECT current_database(), current_setting('server_version_num')::int, " + "pg_database_size(current_database())" + ) + name, version, size = cursor.fetchone() + if name != 'procollab_db_dev' or not 140000 <= version < 190000: + raise RuntimeError('Unexpected DEV database or unsupported PostgreSQL version') + payload.update(server_major=version // 10000, database_bytes=size) + cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename") + tables = [ + row[0] for row in cursor.fetchall() + if row[0].startswith( + ('users_', 'projects_', 'partner_programs_', 'project_rates_', 'files_') + ) + ] + for table in tables: + cursor.execute( + 'SELECT a.attname FROM pg_index i JOIN pg_attribute a ' + 'ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) ' + 'WHERE i.indrelid=to_regclass(%s) AND i.indisprimary ORDER BY a.attnum', + [table], + ) + primary = [row[0] for row in cursor.fetchall()] + if not primary: + raise RuntimeError('Missing primary key in a protected table') + identity = 'jsonb_build_array(' + ','.join( + 't.' + connection.ops.quote_name(key) for key in primary + ) + ')::text' + record = "(to_jsonb(t) - 'last_activity' - 'last_login' - 'datetime_updated')::text" + hash_sql = lambda expression: "encode(sha256(convert_to(" + expression + ", 'UTF8')), 'hex')" + cursor.execute( + 'SELECT ' + hash_sql(identity) + ',' + hash_sql(record) + + ' FROM ' + connection.ops.quote_name(table) + ' t' + ) + records = {} + while True: + batch = cursor.fetchmany(500) + if not batch: + break + for identity, record_hash in batch: + records[identity] = record_hash + payload['tables'][table] = records +print(json.dumps(payload, ensure_ascii=False)) +""" + + +def emit(message, *, file=sys.stdout, flush=True): + file.write(message + "\n") + if flush: + file.flush() + + +def command(args, **kwargs): + try: + return subprocess.run(args, check=True, stderr=subprocess.PIPE, **kwargs) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + raise RuntimeError( + "A DEV backup/check command failed; no credentials are printed" + ) from None + + +def output(args, **kwargs): + return command(args, stdout=subprocess.PIPE, **kwargs).stdout + + +def inspect_container(container): + value = json.loads(output(["docker", "inspect", container]))[0] + labels = value["Config"].get("Labels", {}) + paths = labels.get("com.docker.compose.project.config_files", "").split(",") + if str(REPO / "docker-compose.dev-ci.yml") not in paths: + raise RuntimeError("Container does not belong to the DEV compose configuration") + return value + + +def snapshot(web): + return json.loads( + output( + ["docker", "exec", "-i", web, "python", "-"], + input=PROBE.encode(), + timeout=180, + ) + ) + + +def digest(path): + value = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def main(): + phase = sys.argv[1] + key = sys.argv[2] if len(sys.argv) > 2 else "" + if phase not in ("backup", "verify") or REPO.resolve() != REPO: + raise RuntimeError("Invalid operation or repository location") + os.umask(0o077) + os.chdir(REPO) + web = output( + ["docker", "compose", "-f", "docker-compose.dev-ci.yml", "ps", "-q", "web"], + text=True, + ).strip() + worker = output( + ["docker", "compose", "-f", "docker-compose.dev-ci.yml", "ps", "-q", "celerys"], + text=True, + ).strip() + if not re.fullmatch(r"[a-f0-9]{12,64}", web) or not re.fullmatch( + r"[a-f0-9]{12,64}", worker + ): + raise RuntimeError("Expected exactly one running DEV web and worker") + runtime = inspect_container(web) + worker_runtime = inspect_container(worker) + if runtime["Image"] != worker_runtime["Image"]: + raise RuntimeError("DEV web and worker use different images") + current = snapshot(web) + emit("POSTGRESQL_MAJOR=" + str(current["server_major"]), flush=True) + emit("DJANGO_RUNTIME=" + current["django"], flush=True) + + if phase == "backup": + required = max(2 * 1024**3, current["database_bytes"] * 2 + 512 * 1024**2) + if shutil.disk_usage("/root").free < required: + raise RuntimeError( + "Insufficient free space for a protected DEV backup and rollout" + ) + key = ( + "dependency-" + + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + + uuid4().hex[:8] + ) + BACKUPS.mkdir(mode=0o700, exist_ok=True) + folder = BACKUPS / key + folder.mkdir(mode=0o700) + dump = folder / "database.dump" + environment_script = ( + "import json,os; print(json.dumps({k:os.environ[k] for k in " + '("DATABASE_HOST","DATABASE_PORT","DATABASE_NAME",' + '"DATABASE_USER","DATABASE_PASSWORD")}))' + ) + config = json.loads( + output(["docker", "exec", web, "python", "-c", environment_script]) + ) + if config["DATABASE_NAME"] != "procollab_db_dev": + raise RuntimeError("Unexpected DEV database configuration") + environment = dict(os.environ) + for destination, source in ( + ("PGHOST", "DATABASE_HOST"), + ("PGPORT", "DATABASE_PORT"), + ("PGDATABASE", "DATABASE_NAME"), + ("PGUSER", "DATABASE_USER"), + ("PGPASSWORD", "DATABASE_PASSWORD"), + ): + environment[destination] = config[source] + client = "postgres:" + str(current["server_major"]) + "-alpine" + container_name = "dependency-backup-" + uuid4().hex[:12] + try: + with dump.open("xb") as handle: + command( + [ + "docker", + "run", + "--rm", + "--name", + container_name, + "--network", + "container:" + web, + "--env", + "PGHOST", + "--env", + "PGPORT", + "--env", + "PGDATABASE", + "--env", + "PGUSER", + "--env", + "PGPASSWORD", + client, + "pg_dump", + "--format=custom", + "--no-owner", + "--no-acl", + "--lock-wait-timeout=30s", + ], + env=environment, + stdout=handle, + timeout=600, + ) + finally: + subprocess.run( + ["docker", "rm", "-f", container_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + with dump.open("rb") as handle: + if handle.read(5) != b"PGDMP": + raise RuntimeError("Invalid backup archive") + handle.seek(0) + command( + ["docker", "run", "--rm", "-i", client, "pg_restore", "--file=/dev/null"], + stdin=handle, + stdout=subprocess.DEVNULL, + timeout=600, + ) + rollback = "procollab-dependency-rollback:" + key + command(["docker", "tag", runtime["Image"], rollback], stdout=subprocess.DEVNULL) + shutil.copyfile(REPO / ".env", folder / "runtime.env") + os.chmod(folder / "runtime.env", 0o600) + metadata = { + "snapshot": current, + "image_id": runtime["Image"], + "image": runtime["Config"]["Image"], + "rollback_tag": rollback, + "source_sha": output(["git", "rev-parse", "HEAD"], text=True).strip(), + "dump_sha256": digest(dump), + "dump_bytes": dump.stat().st_size, + } + (folder / "manifest.json").write_text(json.dumps(metadata), encoding="utf8") + emit("BACKUP_KEY=" + key) + emit("BACKUP_BYTES=" + str(metadata["dump_bytes"])) + emit("BACKUP_SHA256=" + metadata["dump_sha256"]) + emit("ROLLBACK_IMAGE=" + rollback) + else: + if KEY.fullmatch(key) is None: + raise RuntimeError("Invalid backup key") + folder = BACKUPS / key + if folder.resolve().parent != BACKUPS.resolve(): + raise RuntimeError("Invalid backup location") + before = json.loads((folder / "manifest.json").read_text(encoding="utf8")) + if digest(folder / "database.dump") != before["dump_sha256"]: + raise RuntimeError("Backup digest mismatch") + removed, changed = {}, {} + for table, records in before["snapshot"]["tables"].items(): + after = current["tables"].get(table, {}) + missing = set(records) - set(after) + different = [ + identity + for identity in set(records) & set(after) + if records[identity] != after[identity] + ] + if missing: + removed[table] = len(missing) + if different: + changed[table] = len(different) + emit("REMOVED_EXISTING_RECORDS=" + json.dumps(removed)) + emit("CHANGED_EXISTING_RECORDS=" + json.dumps(changed)) + if removed or changed: + raise RuntimeError("Existing DEV records changed; inspect before continuing") + emit("EXISTING_DEV_DATA_PRESERVED=true") + emit("CURRENT_IMAGE=" + runtime["Config"]["Image"]) + emit( + "PROTECTED_TABLE_COUNTS=" + + json.dumps({name: len(rows) for name, rows in current["tables"].items()}) + ) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + emit("DEV_DEPENDENCY_PREFLIGHT_FAILED=" + type(error).__name__, file=sys.stderr) + if isinstance(error, RuntimeError): + emit(str(error), file=sys.stderr) + raise SystemExit(1) diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml index 9ba1e037..6c54482c 100644 --- a/.github/workflows/dev-ci.yml +++ b/.github/workflows/dev-ci.yml @@ -4,13 +4,20 @@ on: push: branches: - dev + paths-ignore: + - .github/workflows/dev-ci.yml + - .github/scripts/dependency_dev_preflight.py workflow_dispatch: inputs: operation: description: "DEV operation" type: choice default: deploy - options: [deploy, verify_solution_rollback, prepare_solution_browser, finalize_solution_browser, verify_program_publication, grant_program_creator_11] + options: [deploy, verify_solution_rollback, prepare_solution_browser, finalize_solution_browser, verify_program_publication, grant_program_creator_11, dependency_backup, dependency_verify] + dependency_backup_key: + description: "Ключ резервной копии для проверки сохранности данных dev" + type: string + default: '' fixture_key: description: "Unique acceptance fixture key" default: release-20261026 @@ -372,7 +379,7 @@ jobs: run: node scripts/dev_ci_ssh.cjs solution-acceptance: name: Solution acceptance - if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && inputs.operation != 'deploy' && inputs.operation != '' + if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && inputs.operation != 'deploy' && inputs.operation != '' && inputs.operation != 'dependency_backup' && inputs.operation != 'dependency_verify' environment: dev runs-on: ubuntu-latest timeout-minutes: 12 @@ -434,3 +441,45 @@ jobs: timeoutMs: 9 * 60 * 1000 }).catch(() => { console.error('DEV solution acceptance failed; inspect masked output.'); process.exitCode = 1; }); NODE + + dependency-preflight: + if: github.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && (inputs.operation == 'dependency_backup' || inputs.operation == 'dependency_verify') + environment: dev + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Install pinned SSH transport + run: npm install --prefix "$RUNNER_TEMP/dependency-dev-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0 + - name: Back up or verify DEV data + env: + NODE_PATH: ${{ runner.temp }}/dependency-dev-ssh/node_modules + DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }} + DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }} + DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }} + DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }} + PREFLIGHT_PHASE: ${{ inputs.operation == 'dependency_backup' && 'backup' || 'verify' }} + PREFLIGHT_BACKUP_KEY: ${{ inputs.dependency_backup_key }} + run: | + node <<'NODE' + const fs = require('node:fs'); + const { deploy } = require('./scripts/dev_ci_ssh.cjs'); + const phase = process.env.PREFLIGHT_PHASE; + const key = process.env.PREFLIGHT_BACKUP_KEY || ''; + if (!['backup', 'verify'].includes(phase) || (key && !/^dependency-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}$/.test(key))) throw new Error('Invalid preflight arguments'); + const source = fs.readFileSync('.github/scripts/dependency_dev_preflight.py', 'utf8'); + const script = [ + 'set +x', 'set -eu', 'cd /root/api', + 'test "$(readlink -f /root/api)" = /root/api', + 'grep -Eq "server_name[[:space:]]+dev\\.procollab\\.ru;" /etc/nginx/sites-enabled/dev.procollab.ru', + `python3 - '${phase}' '${key}' <<'DEPENDENCY_DEV_PY'`, + source, 'DEPENDENCY_DEV_PY', + ].join('\n') + '\n'; + deploy({ host: process.env.DEV_SERVER_HOST, username: process.env.DEV_SERVER_USER, + password: process.env.DEV_SERVER_PASSWORD, hostFingerprint: process.env.DEV_SSH_HOST_KEY_SHA256, + script, timeoutMs: 22 * 60 * 1000, + }).catch(() => { console.error('DEV dependency preflight failed; inspect protected output.'); process.exitCode = 1; }); + NODE