From fe298d687680ca9bc93f2d3e45a925a338346b4e Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:01:15 +0900 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=B8=D1=81=D0=BA=D0=BB=D1=8E=D1=87=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20RepoCloneError=20=D0=B8=20RepoTooLargeErro?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/scanner/exceptions.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 app/scanner/exceptions.py diff --git a/app/scanner/exceptions.py b/app/scanner/exceptions.py new file mode 100644 index 0000000..b126515 --- /dev/null +++ b/app/scanner/exceptions.py @@ -0,0 +1,6 @@ +class RepoCloneError(Exception): + pass + + +class RepoTooLargeError(Exception): + pass From c9ec522bc6fb70a14d0545b557983b415140cff2 Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:01:31 +0900 Subject: [PATCH 2/9] =?UTF-8?q?feat:=20=D1=80=D0=B5=D0=B0=D0=BB=D0=B8?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BA=D0=BB=D0=BE=D0=BD?= =?UTF-8?q?=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=BE=D0=B7=D0=B8=D1=82=D0=BE=D1=80=D0=B8=D1=8F=20=D0=BF?= =?UTF-8?q?=D0=BE=20URL=20=D0=B8=20commit=5Fsha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - allowlist на github.com/gitlab.com, http(s) и git@ - shallow-клон через init/remote add/fetch --depth=1/checkout FETCH_HEAD - проверка размера клона против MAX_REPO_SIZE_MB - cleanup временной директории --- app/scanner/clone.py | 124 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 app/scanner/clone.py diff --git a/app/scanner/clone.py b/app/scanner/clone.py new file mode 100644 index 0000000..c93e82c --- /dev/null +++ b/app/scanner/clone.py @@ -0,0 +1,124 @@ +import os +import shutil +import subprocess +import tempfile +from urllib.parse import urlparse + +from app.scanner.exceptions import RepoCloneError, RepoTooLargeError + +SCAN_TEMP_DIR = "" + +MAX_REPO_SIZE_MB = 500 +GIT_TIMEOUT_SECONDS = 30 + +ALLOWED_HOSTS = ("github.com", "gitlab.com") + + +def _check_repo_size(path: str) -> int: + total = 0 + + for dirpath, dirnames, filenames in os.walk(path, followlinks=False): + for name in filenames: + file_path = os.path.join(dirpath, name) + if os.path.islink(file_path): + continue + try: + total += os.path.getsize(file_path) + except OSError: + continue + + return total + + +def _run_git(args: list[str], cwd: str, timeout: int) -> None: + try: + subprocess.run( + ["git", *args], + check=True, + cwd=cwd, + timeout=timeout, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + stderr = (e.stderr or "").strip() + raise RepoCloneError(f"git {' '.join(args)} failed: {stderr or e}") from e + except subprocess.TimeoutExpired as e: + raise RepoCloneError(f"git {' '.join(args)} timed out after {timeout}s") from e + + +def _is_allowed_url(url: str) -> bool: + if not url: + return False + + parsed = urlparse(url) + if parsed.scheme in ("http", "https"): + host = parsed.hostname + + return host is not None and host.lower() in ALLOWED_HOSTS + + if url.startswith("git@"): + rest = url[len("git@") :] + host, sep, path = rest.partition(":") + + return bool(sep) and bool(host) and bool(path) and host.lower() in ALLOWED_HOSTS + + return False + + +def clone_repo(url: str, commit_sha: str | None = None) -> tuple[str, str]: + + if not _is_allowed_url(url): + raise RepoCloneError("URL не разрешён (нужен http(s):// или git@host:...)") + + ref: str = commit_sha or "HEAD" + temp_folder_path: str = tempfile.mkdtemp(dir=SCAN_TEMP_DIR or None, prefix="scan_") + + try: + _run_git(["init"], temp_folder_path, GIT_TIMEOUT_SECONDS) + _run_git( + ["remote", "add", "origin", url], temp_folder_path, GIT_TIMEOUT_SECONDS + ) + _run_git( + ["fetch", "--depth=1", "origin", ref], temp_folder_path, GIT_TIMEOUT_SECONDS + ) + _run_git(["checkout", "FETCH_HEAD"], temp_folder_path, GIT_TIMEOUT_SECONDS) + + resolved_sha = _get_head_sha(temp_folder_path) + + size_bytes = _check_repo_size(temp_folder_path) + max_bytes = MAX_REPO_SIZE_MB * 1024 * 1024 + + if size_bytes > max_bytes: + raise RepoTooLargeError( + f"Размер репозитория {size_bytes / (1024 * 1024):.1f}MB " + f"превышает лимит {MAX_REPO_SIZE_MB}MB" + ) + except (RepoCloneError, RepoTooLargeError): + shutil.rmtree(temp_folder_path, ignore_errors=True) + raise + except Exception as e: + shutil.rmtree(temp_folder_path, ignore_errors=True) + raise RepoCloneError(f"Не удалось клонировать {url}: {e}") from e + + return temp_folder_path, resolved_sha + + +def _get_head_sha(repo_path: str) -> str: + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=True, + cwd=repo_path, + timeout=GIT_TIMEOUT_SECONDS, + capture_output=True, + text=True, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + raise RepoCloneError(f"Не удалось получить хэш коммита: {e}") from e + + return result.stdout.strip() + + +def cleanup(repo_path: str) -> None: + shutil.rmtree(repo_path, ignore_errors=True) From 89a55fc4a745d2733f8cfdf70a7b0209f17d57c2 Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:03:01 +0900 Subject: [PATCH 3/9] =?UTF-8?q?test:=20=D0=BF=D0=BE=D0=BA=D1=80=D1=8B?= =?UTF-8?q?=D1=82=D1=8C=20clone=5Frepo=20=D0=B8=20=5Fis=5Fallowed=5Furl=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D0=B0=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_clone.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/test_clone.py diff --git a/tests/test_clone.py b/tests/test_clone.py new file mode 100644 index 0000000..c21a849 --- /dev/null +++ b/tests/test_clone.py @@ -0,0 +1,61 @@ +import os + +import pytest + +from app.scanner.clone import _is_allowed_url, cleanup, clone_repo +from app.scanner.exceptions import RepoCloneError + + +def test_clone_repo_real_github(): + path, sha = clone_repo(url="https://github.com/mshqq/mshqq.git") + try: + assert os.path.isdir(path) + assert sha + assert len(sha) == 40 + finally: + cleanup(path) + + +def test_clone_nonexist_repo_raises(): + with pytest.raises(RepoCloneError): + clone_repo(url="https://github.com/nonexist/nonexist-repo.git") + + +@pytest.mark.parametrize( + "url, status", + [ + ("https://github.com/owner/repo.git", True), + ("http://github.com/owner/repo.git", True), + ("https://gitlab.com/owner/repo.git", True), + ("git@github.com:owner/repo.git", True), + ("git@gitlab.com:owner/repo", True), + ("file:///etc/passwd", False), + ("local/path/to/repo", False), + ("../relative/path", False), + ("https://badurl.com/owner/repo", False), + ("git@evil.com:owner/repo", False), + ("git@github.com", False), + ("", False), + ("https://github.com.badurl.com", False), + ("HTTPS://GITHUB.COM/owner/repo.git", True), + ("git@github.com:", False), + ], +) +def test_is_allowed_url(url, status): + assert _is_allowed_url(url) is status + + +def test_cleanup_removes_directory(tmp_path): + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "file.txt").write_text("data") + + cleanup(str(repo_dir)) + + assert not os.path.isdir(repo_dir) + + +def test_cleanup_nonexistent_path_does_not_raise(tmp_path): + missing_dir = tmp_path / "missing" + + cleanup(str(missing_dir)) From 8e11ad1aea006c4072fd3fb1cb1aa1caaef6494d Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:52:09 +0900 Subject: [PATCH 4/9] =?UTF-8?q?feat:=20=D1=80=D0=B5=D0=B0=D0=BB=D0=B8?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20list=5Ffiles=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=BE=D0=B1=D1=85=D0=BE=D0=B4=D0=B0=20=D1=84?= =?UTF-8?q?=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2=20=D0=BA=D0=BB=D0=BE=D0=BD=D0=B0?= =?UTF-8?q?=20=D1=80=D0=B5=D0=BF=D0=BE=D0=B7=D0=B8=D1=82=D0=BE=D1=80=D0=B8?= =?UTF-8?q?=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - рекурсивный обход через os.walk, .git пропускается целиком - относительные пути от repo_path, отсортированы - симлинки (на файлы и директории) не дают выйти за пределы repo_path --- app/scanner/files.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 app/scanner/files.py diff --git a/app/scanner/files.py b/app/scanner/files.py new file mode 100644 index 0000000..e200221 --- /dev/null +++ b/app/scanner/files.py @@ -0,0 +1,21 @@ +import os + + +def list_files(repo_path: str) -> list[str]: + files: list[str] = [] + + for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=False): + if ".git" in dirnames: + dirnames.remove(".git") + + for name in filenames: + file_path: str = os.path.join(dirpath, name) + + if os.path.islink(file_path): + continue + + rel_path: str = os.path.relpath(file_path, repo_path) + files.append(rel_path) + + files.sort() + return files From 0c3feb7a1a1b505e986195bf792fbe4aac0d12d5 Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:53:19 +0900 Subject: [PATCH 5/9] =?UTF-8?q?test:=20=D0=BF=D0=BE=D0=BA=D1=80=D1=8B?= =?UTF-8?q?=D1=82=D1=8C=20list=5Ffiles=20=D1=82=D0=B5=D1=81=D1=82=D0=B0?= =?UTF-8?q?=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - пустой репозиторий, файлы в корне и во вложенных директориях - исключение .git - симлинк на директорию и симлинк на файл за пределами repo_path - несуществующий путь --- tests/test_files.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_files.py diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..c38bdf2 --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,77 @@ +import os + +from app.scanner.files import list_files + + +def test_files(tmp_path): + assert list_files(str(tmp_path)) == [] + + +def test_single_file_in_root(tmp_path): + (tmp_path / "a.txt").write_text("X") + + assert list_files(str(tmp_path)) == ["a.txt"] + + +def test_multiple_file_in_root(tmp_path): + (tmp_path / "a.txt").write_text("X") + (tmp_path / "b.txt").write_text("X") + (tmp_path / "c.txt").write_text("X") + + assert list_files(str(tmp_path)) == ["a.txt", "b.txt", "c.txt"] + + +def test_nested_dir(tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "models").mkdir() + (tmp_path / "app" / "models" / "user.py").write_text("X") + (tmp_path / "app" / "config.py").write_text("X") + (tmp_path / ".env").write_text("X") + + result: list[str] = list_files(str(tmp_path)) + + assert result == sorted( + [ + ".env", + os.path.join("app", "models", "user.py"), + os.path.join("app", "config.py"), + ] + ) + + +def test_git_dir_excluded(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "config").write_text("X") + (tmp_path / "file.txt").write_text("X") + + result: list[str] = sorted(list_files(str(tmp_path))) + + assert result == ["file.txt"] + + +def test_symlink(tmp_path): + target_dir = tmp_path / "target" + target_dir.mkdir() + (target_dir / "inner.txt").write_text("x") + + link_dir = tmp_path / "link_dir" + link_dir.symlink_to(target_dir, target_is_directory=True) + + result: list[str] = list_files(str(tmp_path)) + + assert result == [os.path.join("target", "inner.txt")] + + +def test_symlink_to_file_outside_repo(tmp_path, tmp_path_factory): + outside = tmp_path_factory.mktemp("outside") / "secret.txt" + outside.write_text("X") + + (tmp_path / "link.txt").symlink_to(outside) + (tmp_path / "real.txt").write_text("X") + + assert list_files(str(tmp_path)) == ["real.txt"] + + +def test_nonexist_path(tmp_path): + missing = tmp_path / "does_not_exist" + assert list_files(str(missing)) == [] From f65fc9de8682c5e2c27405fdcda3b705009ae017 Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 05:57:39 +0900 Subject: [PATCH 6/9] =?UTF-8?q?test:=20=D0=BF=D0=BE=D0=BA=D1=80=D1=8B?= =?UTF-8?q?=D1=82=D1=8C=20RepoTooLargeError=20=D1=82=D0=B5=D1=81=D1=82?= =?UTF-8?q?=D0=BE=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MAX_REPO_SIZE_MB временно занижается до 0 через модуль, чтобы проверить превышение лимита без раздувания тестового репозитория. - значение восстанавливается в finally, иначе оно утечёт в остальные тесты процесса --- tests/test_clone.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_clone.py b/tests/test_clone.py index c21a849..2065287 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -2,8 +2,9 @@ import pytest +import app.scanner.clone as clone_module from app.scanner.clone import _is_allowed_url, cleanup, clone_repo -from app.scanner.exceptions import RepoCloneError +from app.scanner.exceptions import RepoCloneError, RepoTooLargeError def test_clone_repo_real_github(): @@ -59,3 +60,14 @@ def test_cleanup_nonexistent_path_does_not_raise(tmp_path): missing_dir = tmp_path / "missing" cleanup(str(missing_dir)) + + +def test_clone_repo_too_large(): + original_max = clone_module.MAX_REPO_SIZE_MB + clone_module.MAX_REPO_SIZE_MB = 0 + + try: + with pytest.raises(RepoTooLargeError): + clone_repo(url="https://github.com/mshqq/mshqq.git") + finally: + clone_module.MAX_REPO_SIZE_MB = original_max From de86872b67f8e94595faeb0c38bbf628d196d8a9 Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 06:10:25 +0900 Subject: [PATCH 7/9] =?UTF-8?q?fix:=20=D0=BD=D0=BE=D1=80=D0=BC=D0=B0=D0=BB?= =?UTF-8?q?=D0=B8=D0=B7=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D0=B4=D0=B5=D0=BB=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=20=D0=BF=D1=83?= =?UTF-8?q?=D1=82=D0=B5=D0=B9=20=D0=B2=20list=5Ffiles=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=20/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit os.path.relpath отдаёт os.sep, на Windows это "\", из-за чего _basename в env_file.py (жёстко режет по "/") не найдёт .env-файлы во вложенных папках. Явно приводим путь к "/" независимо от ОС. --- app/scanner/files.py | 2 +- tests/test_files.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/scanner/files.py b/app/scanner/files.py index e200221..70759da 100644 --- a/app/scanner/files.py +++ b/app/scanner/files.py @@ -15,7 +15,7 @@ def list_files(repo_path: str) -> list[str]: continue rel_path: str = os.path.relpath(file_path, repo_path) - files.append(rel_path) + files.append(rel_path.replace(os.sep, "/")) files.sort() return files diff --git a/tests/test_files.py b/tests/test_files.py index c38bdf2..522af91 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -1,5 +1,3 @@ -import os - from app.scanner.files import list_files @@ -33,8 +31,8 @@ def test_nested_dir(tmp_path): assert result == sorted( [ ".env", - os.path.join("app", "models", "user.py"), - os.path.join("app", "config.py"), + "app/models/user.py", + "app/config.py", ] ) @@ -59,7 +57,7 @@ def test_symlink(tmp_path): result: list[str] = list_files(str(tmp_path)) - assert result == [os.path.join("target", "inner.txt")] + assert result == ["target/inner.txt"] def test_symlink_to_file_outside_repo(tmp_path, tmp_path_factory): From 2a761d170e72625603cf27f7fcd56a10d75504dd Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 06:11:38 +0900 Subject: [PATCH 8/9] =?UTF-8?q?test:=20=D0=BF=D0=BE=D0=BA=D1=80=D1=8B?= =?UTF-8?q?=D1=82=D1=8C=20clone=5Frepo=20=D0=BA=D0=BB=D0=BE=D0=BD=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=D0=BC=20=D0=BD=D0=B0?= =?UTF-8?q?=20=D0=BA=D0=BE=D0=BD=D0=BA=D1=80=D0=B5=D1=82=D0=BD=D1=8B=D0=B9?= =?UTF-8?q?=20commit=5Fsha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Раньше проверялся только дефолтный HEAD и несуществующий репозиторий - явного клонирования на заданный коммит тесты не закрывали. --- tests/test_clone.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_clone.py b/tests/test_clone.py index 2065287..3bd6ada 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -17,6 +17,19 @@ def test_clone_repo_real_github(): cleanup(path) +def test_clone_repo_specific_commit(): + commit_sha = "bce38ad98aa578b9603d84ef74505c60ceb287ef" + + path, sha = clone_repo( + url="https://github.com/mshqq/mshqq.git", commit_sha=commit_sha + ) + try: + assert os.path.isdir(path) + assert sha == commit_sha + finally: + cleanup(path) + + def test_clone_nonexist_repo_raises(): with pytest.raises(RepoCloneError): clone_repo(url="https://github.com/nonexist/nonexist-repo.git") From 4e35a02154dbbc0d9b6e43175804db414417e64c Mon Sep 17 00:00:00 2001 From: mshqq Date: Fri, 14 Aug 2026 17:03:22 +0900 Subject: [PATCH 9/9] ruff: fix formatting --- app/scanner/clone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/scanner/clone.py b/app/scanner/clone.py index c93e82c..cb50dbe 100644 --- a/app/scanner/clone.py +++ b/app/scanner/clone.py @@ -67,7 +67,6 @@ def _is_allowed_url(url: str) -> bool: def clone_repo(url: str, commit_sha: str | None = None) -> tuple[str, str]: - if not _is_allowed_url(url): raise RepoCloneError("URL не разрешён (нужен http(s):// или git@host:...)")