feat/scan-worker

app/scanner/scan_worker.py

@@ -0,0 +1,83 @@
+from concurrent.futures import Future, ThreadPoolExecutor
+
+from sqlalchemy.orm import joinedload
+
+from app.extensions import db
+from app.models.finding import Finding
+from app.models.scan import Scan
+from app.scanner.base import ScanContext
+from app.scanner.clone import cleanup, clone_repo
+from app.scanner.files import list_files
+from app.scanner.runner import ScanResult, run_rules
+from app.utils import utcnow
+
+executor = ThreadPoolExecutor(max_workers=4)
+
+
+def run_scan(app, scan_id: int) -> Future:
+    return executor.submit(worker_function, app, scan_id)
+
+
+def worker_function(app, scan_id) -> ScanResult | None:
+    with app.app_context():
+        scan: Scan | None = (
+            Scan.query.options(joinedload(Scan.project))
+            .filter(Scan.id == scan_id)
+            .first()
+        )
+        repo_url: str = scan.project.repo_url
+        repo_path: str | None = None
+        print(f"Начинаю обработку {repo_url}")
+        scan.status = "running"
+        scan.started_at = utcnow()
+        db.session.commit()
+
+        try:
+            repo_path, resolved_sha = clone_repo(repo_url)
+
+            ctx = ScanContext(repo_path, resolved_sha, list_files(repo_path))
+            result: ScanResult = run_rules(ctx)
+
+            scan.status = "done"
+            scan.truncated = result.truncated
+            scan.error_message = "; ".join(result.errors)
+
+            for f in result.findings:
+                # pyrefly: ignore [unexpected-keyword]
+                finding = Finding(
+                    # pyrefly: ignore [unexpected-keyword]
+                    scan_id=scan_id,
+                    # pyrefly: ignore [unexpected-keyword]
+                    rule_id=f["rule_id"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    severity=f["severity"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    confidence=f["confidence"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    source=f["source"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    file_path=f["file_path"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    line_no=f["line_no"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    commit_sha=f["commit_sha"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    masked_value=f["masked_value"],
+                    # pyrefly: ignore [unexpected-keyword]
+                    context=f["context"],
+                )
+                db.session.add(finding)
+
+            scan.finished_at = utcnow()
+            db.session.commit()
+
+        except Exception as e:
+            scan.status = "failed"
+            scan.error_message = str(e)
+            db.session.commit()
+            return
+        finally:
+            if repo_path:
+                cleanup(repo_path)
+
+    print("Поток завершил работу")

scripts/create_test_data.py

@@ -0,0 +1,96 @@
+from dotenv import load_dotenv
+
+from app import create_app
+from app.extensions import db
+from app.models.finding import Finding
+from app.models.project import Project
+from app.models.scan import Scan
+from app.models.user import User
+from app.utils import utcnow
+
+TEST_EMAIL = "[email protected]"
+TEST_PASSWORD = "TestPassword123!"
+TEST_REPO_URL = "https://github.com/mshqq/mshqq"
+
+
+def create_test_user() -> User:
+    user = User.query.filter_by(email=TEST_EMAIL).first()
+    if user is not None:
+        print(f"Пользователь уже существует: {user.email}")
+        return user
+
+    user = User(email=TEST_EMAIL)
+    user.set_password(TEST_PASSWORD)
+    db.session.add(user)
+    db.session.flush()
+    print(f"Создан пользователь: {user.email} (пароль: {TEST_PASSWORD})")
+    return user
+
+
+def create_test_project(user: User) -> Project:
+    project = Project.query.filter_by(owner_id=user.id, repo_url=TEST_REPO_URL).first()
+    if project is not None:
+        print(f"Проект уже существует: {project.title} (id={project.id})")
+        return project
+
+    project = Project(
+        owner_id=user.id,
+        title="Тестовый проект",
+        repo_url=TEST_REPO_URL,
+        provider="github",
+    )
+    project.create_ownership_token()
+    project.ownership_verified_at = utcnow()
+    db.session.add(project)
+    db.session.flush()
+    print(f"Создан проект: {project.title} (id={project.id})")
+    return project
+
+
+def create_test_scan(project: Project) -> Scan:
+    scan = Scan(
+        project_id=project.id,
+        status="done",
+        started_at=utcnow(),
+        finished_at=utcnow(),
+        commit_sha="0" * 40,
+        truncated=False,
+    )
+    db.session.add(scan)
+    db.session.flush()
+    print(f"Создан скан: id={scan.id}, status={scan.status}")
+    return scan
+
+
+def create_test_finding(scan: Scan) -> Finding:
+    finding = Finding(
+        scan_id=scan.id,
+        rule_id="ENV_FILE_COMMITED",
+        severity="P0",
+        confidence="high",
+        source="regex",
+        file_path=".env",
+        line_no=3,
+        commit_sha=scan.commit_sha,
+        masked_value="test****test",
+        context={"key": "API_KEY"},
+    )
+    db.session.add(finding)
+    db.session.flush()
+    print(f"Создан finding: id={finding.id}, rule_id={finding.rule_id}")
+    return finding
+
+
+def seed() -> None:
+    user = create_test_user()
+    project = create_test_project(user)
+    scan = create_test_scan(project)
+    create_test_finding(scan)
+    db.session.commit()
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    app = create_app()
+    with app.app_context():
+        seed()

tests/conftest.py

@@ -0,0 +1,68 @@
+import os
+
+import pytest
+
+from app import create_app
+from app.extensions import db
+from app.models.project import Project
+from app.models.scan import Scan
+from app.models.user import User
+from app.utils import utcnow
+
+os.environ.setdefault("FLASK_SECRET", "test-secret")
+
+
+@pytest.fixture
+def app():
+    # pyrefly: ignore [unexpected-keyword]
+    app = create_app()
+    app.config.update(
+        {
+            "TESTING": True,
+            "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
+            "SECRET_KEY": "test-secret-key",
+        }
+    )
+
+    with app.app_context():
+        db.create_all()
+        yield app
+        db.drop_all()
+
+
+@pytest.fixture
+def user(app):
+    u = User(email="[email protected]")
+    u.set_password("TEST_PASSWORD")
+    db.session.add(u)
+    db.session.commit()
+    return u
+
+
+@pytest.fixture
+def project(app, user):
+    p = Project(
+        owner_id=user.id,
+        title="Тестовый проект",
+        repo_url="https://github.com/mshqq/ZabGU-DevSecOps-Hub",
+        provider="github",
+    )
+    p.create_ownership_token()
+    db.session.add(p)
+    db.session.commit()
+    return p
+
+
+@pytest.fixture
+def scan(app, user, project):
+    s = Scan(
+        project_id=project.id,
+        status="done",
+        started_at=utcnow(),
+        finished_at=utcnow(),
+        commit_sha="0" * 40,
+        truncated=False,
+    )
+    db.session.add(s)
+    db.session.commit()
+    return s

tests/test_scan_worker.py

@@ -0,0 +1,16 @@
+from app.models.finding import Finding
+from app.models.scan import Scan
+from app.scanner.scan_worker import run_scan
+
+
+def test_concurrent(app, user, project, scan):
+    future = run_scan(app, scan.id)
+    future.result(timeout=60)
+
+    with app.app_context():
+        completed_scan = Scan.query.filter(Scan.id == scan.id).first()
+        findings = Finding.query.all()
+
+    assert completed_scan.status == "done"
+    assert findings
+    assert all(f.rule_id == "ENV_FILE_COMMITED" for f in findings)