From 746df19b26fc9a5e08077309686aed894fb7f6b7 Mon Sep 17 00:00:00 2001 From: mshqq Date: Sun, 16 Aug 2026 03:10:53 +0900 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=84=D0=BE=D0=BD=D0=BE=D0=B2=D1=8B=D0=B9?= =?UTF-8?q?=20=D0=B2=D0=BE=D1=80=D0=BA=D0=B5=D1=80=20=D1=81=D0=BA=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20(scan?= =?UTF-8?q?=5Fworker)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Реализует run_scan/worker_function: клонирование репо в фоновом потоке, прогон правил, запись Scan/Finding, статусы running/done/failed. --- app/scanner/scan_worker.py | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 app/scanner/scan_worker.py diff --git a/app/scanner/scan_worker.py b/app/scanner/scan_worker.py new file mode 100644 index 0000000..4e22ae1 --- /dev/null +++ b/app/scanner/scan_worker.py @@ -0,0 +1,83 @@ +from concurrent.futures import Future, ThreadPoolExecutor + +from sqlalchemy.orm import joinedload + +from app.extensions import db +from app.models.finding import Finding +from app.models.scan import Scan +from app.scanner.base import ScanContext +from app.scanner.clone import cleanup, clone_repo +from app.scanner.files import list_files +from app.scanner.runner import ScanResult, run_rules +from app.utils import utcnow + +executor = ThreadPoolExecutor(max_workers=4) + + +def run_scan(app, scan_id: int) -> Future: + return executor.submit(worker_function, app, scan_id) + + +def worker_function(app, scan_id) -> ScanResult | None: + with app.app_context(): + scan: Scan | None = ( + Scan.query.options(joinedload(Scan.project)) + .filter(Scan.id == scan_id) + .first() + ) + repo_url: str = scan.project.repo_url + repo_path: str | None = None + print(f"Начинаю обработку {repo_url}") + scan.status = "running" + scan.started_at = utcnow() + db.session.commit() + + try: + repo_path, resolved_sha = clone_repo(repo_url) + + ctx = ScanContext(repo_path, resolved_sha, list_files(repo_path)) + result: ScanResult = run_rules(ctx) + + scan.status = "done" + scan.truncated = result.truncated + scan.error_message = "; ".join(result.errors) + + for f in result.findings: + # pyrefly: ignore [unexpected-keyword] + finding = Finding( + # pyrefly: ignore [unexpected-keyword] + scan_id=scan_id, + # pyrefly: ignore [unexpected-keyword] + rule_id=f["rule_id"], + # pyrefly: ignore [unexpected-keyword] + severity=f["severity"], + # pyrefly: ignore [unexpected-keyword] + confidence=f["confidence"], + # pyrefly: ignore [unexpected-keyword] + source=f["source"], + # pyrefly: ignore [unexpected-keyword] + file_path=f["file_path"], + # pyrefly: ignore [unexpected-keyword] + line_no=f["line_no"], + # pyrefly: ignore [unexpected-keyword] + commit_sha=f["commit_sha"], + # pyrefly: ignore [unexpected-keyword] + masked_value=f["masked_value"], + # pyrefly: ignore [unexpected-keyword] + context=f["context"], + ) + db.session.add(finding) + + scan.finished_at = utcnow() + db.session.commit() + + except Exception as e: + scan.status = "failed" + scan.error_message = str(e) + db.session.commit() + return + finally: + if repo_path: + cleanup(repo_path) + + print("Поток завершил работу") From 0a08db11a696b5c7246007f978c6182548851b74 Mon Sep 17 00:00:00 2001 From: mshqq Date: Sun, 16 Aug 2026 03:12:31 +0900 Subject: [PATCH 2/3] =?UTF-8?q?test:=20=D0=BF=D0=BE=D0=BA=D1=80=D1=8B?= =?UTF-8?q?=D1=82=D1=8C=20scan=5Fworker=20=D1=82=D0=B5=D1=81=D1=82=D0=BE?= =?UTF-8?q?=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Общие фикстуры app/user/project/scan + test_concurrent, проверяющий полный цикл run_scan end-to-end с ожиданием future.result(). --- tests/conftest.py | 68 +++++++++++++++++++++++++++++++++++++++ tests/test_scan_worker.py | 16 +++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/test_scan_worker.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d9e22c1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,68 @@ +import os + +import pytest + +from app import create_app +from app.extensions import db +from app.models.project import Project +from app.models.scan import Scan +from app.models.user import User +from app.utils import utcnow + +os.environ.setdefault("FLASK_SECRET", "test-secret") + + +@pytest.fixture +def app(): + # pyrefly: ignore [unexpected-keyword] + app = create_app() + app.config.update( + { + "TESTING": True, + "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", + "SECRET_KEY": "test-secret-key", + } + ) + + with app.app_context(): + db.create_all() + yield app + db.drop_all() + + +@pytest.fixture +def user(app): + u = User(email="mshqq@email.test") + u.set_password("TEST_PASSWORD") + db.session.add(u) + db.session.commit() + return u + + +@pytest.fixture +def project(app, user): + p = Project( + owner_id=user.id, + title="Тестовый проект", + repo_url="https://github.com/mshqq/ZabGU-DevSecOps-Hub", + provider="github", + ) + p.create_ownership_token() + db.session.add(p) + db.session.commit() + return p + + +@pytest.fixture +def scan(app, user, project): + s = Scan( + project_id=project.id, + status="done", + started_at=utcnow(), + finished_at=utcnow(), + commit_sha="0" * 40, + truncated=False, + ) + db.session.add(s) + db.session.commit() + return s diff --git a/tests/test_scan_worker.py b/tests/test_scan_worker.py new file mode 100644 index 0000000..77968fc --- /dev/null +++ b/tests/test_scan_worker.py @@ -0,0 +1,16 @@ +from app.models.finding import Finding +from app.models.scan import Scan +from app.scanner.scan_worker import run_scan + + +def test_concurrent(app, user, project, scan): + future = run_scan(app, scan.id) + future.result(timeout=60) + + with app.app_context(): + completed_scan = Scan.query.filter(Scan.id == scan.id).first() + findings = Finding.query.all() + + assert completed_scan.status == "done" + assert findings + assert all(f.rule_id == "ENV_FILE_COMMITED" for f in findings) From 24fec3a977a768425034c2c4dca59d71b9151b4f Mon Sep 17 00:00:00 2001 From: mshqq Date: Sun, 16 Aug 2026 03:15:44 +0900 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=81=D0=BA=D1=80=D0=B8=D0=BF=D1=82=20?= =?UTF-8?q?=D1=81=D0=BE=D0=B7=D0=B4=D0=B0=D0=BD=D0=B8=D1=8F=20=D1=82=D0=B5?= =?UTF-8?q?=D1=81=D1=82=D0=BE=D0=B2=D1=8B=D1=85=20=D0=B4=D0=B0=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/create_test_data.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 scripts/create_test_data.py diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py new file mode 100644 index 0000000..c3211d3 --- /dev/null +++ b/scripts/create_test_data.py @@ -0,0 +1,96 @@ +from dotenv import load_dotenv + +from app import create_app +from app.extensions import db +from app.models.finding import Finding +from app.models.project import Project +from app.models.scan import Scan +from app.models.user import User +from app.utils import utcnow + +TEST_EMAIL = "mshqq@email.test" +TEST_PASSWORD = "TestPassword123!" +TEST_REPO_URL = "https://github.com/mshqq/mshqq" + + +def create_test_user() -> User: + user = User.query.filter_by(email=TEST_EMAIL).first() + if user is not None: + print(f"Пользователь уже существует: {user.email}") + return user + + user = User(email=TEST_EMAIL) + user.set_password(TEST_PASSWORD) + db.session.add(user) + db.session.flush() + print(f"Создан пользователь: {user.email} (пароль: {TEST_PASSWORD})") + return user + + +def create_test_project(user: User) -> Project: + project = Project.query.filter_by(owner_id=user.id, repo_url=TEST_REPO_URL).first() + if project is not None: + print(f"Проект уже существует: {project.title} (id={project.id})") + return project + + project = Project( + owner_id=user.id, + title="Тестовый проект", + repo_url=TEST_REPO_URL, + provider="github", + ) + project.create_ownership_token() + project.ownership_verified_at = utcnow() + db.session.add(project) + db.session.flush() + print(f"Создан проект: {project.title} (id={project.id})") + return project + + +def create_test_scan(project: Project) -> Scan: + scan = Scan( + project_id=project.id, + status="done", + started_at=utcnow(), + finished_at=utcnow(), + commit_sha="0" * 40, + truncated=False, + ) + db.session.add(scan) + db.session.flush() + print(f"Создан скан: id={scan.id}, status={scan.status}") + return scan + + +def create_test_finding(scan: Scan) -> Finding: + finding = Finding( + scan_id=scan.id, + rule_id="ENV_FILE_COMMITED", + severity="P0", + confidence="high", + source="regex", + file_path=".env", + line_no=3, + commit_sha=scan.commit_sha, + masked_value="test****test", + context={"key": "API_KEY"}, + ) + db.session.add(finding) + db.session.flush() + print(f"Создан finding: id={finding.id}, rule_id={finding.rule_id}") + return finding + + +def seed() -> None: + user = create_test_user() + project = create_test_project(user) + scan = create_test_scan(project) + create_test_finding(scan) + db.session.commit() + + +if __name__ == "__main__": + load_dotenv() + app = create_app() + with app.app_context(): + seed()