diff --git a/.github/workflows/trusted-uv-materializer-quality-ci.yml b/.github/workflows/trusted-uv-materializer-quality-ci.yml index a3404232b..b447229d3 100644 --- a/.github/workflows/trusted-uv-materializer-quality-ci.yml +++ b/.github/workflows/trusted-uv-materializer-quality-ci.yml @@ -8,6 +8,7 @@ on: - "scripts/ci/materialize_base_python_requirements.py" - "tests/conftest.py" - "tests/test_materialize*.py" + - "tests/test_requirements_directory_lock_materialization.py" - "tests/test_trusted_uv*.py" - "tests/test_uv*.py" - "tests/test_repository_branch_coverage_*.py" @@ -20,6 +21,7 @@ on: - "scripts/ci/materialize_base_python_requirements.py" - "tests/conftest.py" - "tests/test_materialize*.py" + - "tests/test_requirements_directory_lock_materialization.py" - "tests/test_trusted_uv*.py" - "tests/test_uv*.py" - "tests/test_repository_branch_coverage_*.py" @@ -126,6 +128,7 @@ jobs: python -m coverage run -m pytest \ tests/test_materialize_base_python_requirements.py \ tests/test_materialize_uv_export_hash_contract.py \ + tests/test_requirements_directory_lock_materialization.py \ tests/test_trusted_uv_download_contract.py \ tests/test_trusted_uv_portability_and_streaming.py \ tests/test_uv_export_isolation_contract.py \ @@ -153,6 +156,7 @@ jobs: scripts/ci/materialize_base_python_requirements.py \ tests/test_materialize_base_python_requirements.py \ tests/test_materialize_uv_export_hash_contract.py \ + tests/test_requirements_directory_lock_materialization.py \ tests/test_trusted_uv_download_contract.py \ tests/test_trusted_uv_portability_and_streaming.py \ tests/test_uv_export_isolation_contract.py \ diff --git a/AGENTS.md b/AGENTS.md index bd6a96a11..91d0dd5be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,8 +2,8 @@ > **Agents: read the master context FIRST.** Before any work, read [`docs/CWL-MASTER-CONTEXT.md`](docs/CWL-MASTER-CONTEXT.md) (mission · naruon-as-platform + inter-component UML · cross-cutting disciplines · conventions · roadmap · current state), the live **GitHub Project #1** (work/roadmap source of truth), the full spec **ContextualWisdomLab/naruon#974**, and operate the Project per [`docs/agent-github-project-protocol.md`](docs/agent-github-project-protocol.md). The repo/Project — not any private agent memory — is the source of truth. - Materialize accepts only exact SHA-256 pins or a bounded relative `-r` include (no `.`/`..`); a lone `--require-hashes` directive is not trust evidence. See [`docs/doctoring/hourly-nvidia-nim-autofix.md`](docs/doctoring/hourly-nvidia-nim-autofix.md). Conflict-scope roots fail closed when the immediate parent directory is a symbolic link. OriginWeave hourly NVIDIA NIM repair is a thin caller at minute 10. See [`docs/doctoring/originweave-hourly-review-caller.md`](docs/doctoring/originweave-hourly-review-caller.md). nonnest2 hourly NVIDIA NIM repair is a thin caller at minute 16. See [`docs/doctoring/nonnest2-hourly-review-caller.md`](docs/doctoring/nonnest2-hourly-review-caller.md). +Trusted requirement includes must be normalized relative POSIX lock paths; `./` and `//` are rejected. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3e2e70b58..3784518dc 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -70,6 +70,29 @@ Product callers stagger Clearfolio at minute 23, DiskSage at minute 37, and fast-mlsirm at minute 49. Each caller is read-only, dispatches at most one repair, and delegates all privileged logic to the same sealed scheduler. +## Requirements-directory lock gate + +```mermaid +flowchart TD + Base["Authenticated base-commit blob"] + Path{"Direct child of a requirements/ directory, or conventional lock name?"} + Grammar{"Every non-directive line is an exact == SHA-256 pin or a bounded -r include?"} + Preflight{"Independent pip --require-hashes closure succeeds?"} + Accept["Materialize into the coverage image"] + Reject["Leave the file out of the networked build context"] + + Base --> Path + Path -->|"no"| Reject + Path -->|"yes"| Grammar + Grammar -->|"no"| Reject + Grammar -->|"yes"| Preflight + Preflight -->|"no"| Reject + Preflight -->|"yes"| Accept +``` + +A global `--require-hashes` directive is not integrity evidence. CWE-494 +forbids downloading code without verifying origin and digest. + ## Control-plane data flow ```mermaid @@ -123,4 +146,6 @@ trusted `uv` exporter is downloaded from the literal GitHub Releases URL for - [`docs/doctoring/hourly-nvidia-nim-autofix.md`](docs/doctoring/hourly-nvidia-nim-autofix.md) — current increment's repair-worker decision and APA 7th citations. - [`docs/doctoring/fast-mlsirm-hourly-review-caller.md`](docs/doctoring/fast-mlsirm-hourly-review-caller.md) - — product-specific psychometric repair heartbeat and scientific gates. \ No newline at end of file + — product-specific psychometric repair heartbeat and scientific gates. +- [`docs/doctoring/trusted-requirements-directory-lock-discovery.md`](docs/doctoring/trusted-requirements-directory-lock-discovery.md) + — current increment's lock-discovery decision and APA 7th citations. diff --git a/CHANGELOG.md b/CHANGELOG.md index fd1aebf43..132a108e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,11 +43,13 @@ Semantic Versioning where the repository publishes a release. - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. - Allowed commas and ASCII parentheses in the bounded Strix changed-file path policy so legal tracked Packrat fixtures can receive exact-head security analysis, while rejecting raw `..` components before normalization and keeping controls, backslashes, whitespace ambiguity, and shell punctuation fail-closed. - Bound each review-agent invocation key to the wrapper's complete canonical payload, including the base branch and requesting actor; altered fields with a valid-format key now fail before durable-leader election or forwarding, and wrapper write permission is job-scoped. +- Materialize complete hash-pinned `requirements/ci.txt` and other direct `requirements/*.txt` base-owned closures so isolated OpenCode coverage imports repository runtime dependencies without trusting pull-request metadata or broadening network access. - Bound both trusted-uv quality jobs to `github.event.pull_request.head.sha` and added a permanent two-checkout regression contract so exact-head compatibility, coverage, docstring, and compilation claims cannot silently measure GitHub's generated pull-request merge revision. - Made Strix treat only a single LiteLLM provider-error line containing NVIDIA NIM context and model-catalog 404 evidence as cross-model fallback evidence, rejecting cross-line signal assembly and provider-like target source literals; moved the public default to Nemotron 3 Super 120B and added a second NVIDIA hosted candidate before GitHub Models without neutralizing reported vulnerabilities. - Bind reusable scheduler implementation to the validated called-workflow repository, SHA, ref, and file path, and verify the checked-out commit before executing privileged scheduler logic. - Removed the ambiguous central-repository schedule fallback that could scan `.github` instead of Clearfolio when no external variable was configured; the active product caller now names Clearfolio explicitly while the reusable engine retains caller and dispatch overrides. - Corrected the conflict-ordering regression contract to select the conflict-specific snapshot and verification after the ordinary path adopted the same trusted helper. +- Hardened base-owned Python requirements materialization so candidate package lines require exact `==` pins with complete SHA-256 hashes and requirement includes use only normalized relative candidate lock paths; `./` and doubled-slash targets, range pins, malformed digests, pip option lines, URL/absolute/traversing/home/query/fragment/backslash/option-like includes, and include lines carrying extra inline options are rejected before the trusted build context. ### Security diff --git a/CLAUDE.md b/CLAUDE.md index d73a5c169..8cf0503a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,7 +68,8 @@ Details: `README.md` and `PR_GOVERNANCE_AUDIT.md`. - `docs/` — master context, Project protocol, `org-required-workflow-rollout.md`, `scorecard-governance.md`, SBOM inventory. Doctoring records live under `docs/doctoring/`. [`ARCHITECTURE.md`](ARCHITECTURE.md) is the control-plane - diagram for review, hourly NVIDIA NIM repair, and merge trust boundaries. + diagram for review, requirements-directory lock discovery, hourly NVIDIA NIM + repair, and merge trust boundaries. - `.jules/` — recorded performance (`bolt.md`) and security (`sentinel.md`) learnings from past work on `scripts/ci/`; worth scanning before optimizing or hardening those scripts. diff --git a/docs/doctoring/trusted-requirements-directory-lock-discovery.md b/docs/doctoring/trusted-requirements-directory-lock-discovery.md new file mode 100644 index 000000000..0be213f0d --- /dev/null +++ b/docs/doctoring/trusted-requirements-directory-lock-discovery.md @@ -0,0 +1,103 @@ +# Trusted requirements-directory lock discovery + +## Decision + +The central OpenCode coverage image materializes dependency closures only from +regular files in the authenticated pull-request base commit. In addition to the +conventional `requirements*.txt` and `requirements.lock` names, it recognizes a +`.txt` file that is a **direct child** of a directory named `requirements`, such +as `requirements/ci.txt` or `services/scoring_service/requirements/package.txt`. + +The path rule grants candidate status only. A global `--require-hashes` +directive is not trust evidence by itself. Every non-directive logical line must +be either: + +- an exact package `==` requirement with one or more complete 64-hex SHA-256 + `--hash=` values; or +- a two-token `-r` / `--requirement` include naming a bounded relative + candidate lock path (`requirements*.txt`, `requirements.lock`, or a + direct `.txt` child of a `requirements/` directory). + +The include grammar rejects absolute paths, `..` traversal, URL/scheme syntax, +query or fragment syntax, shell-home expansion, Windows-style separators, +option-like targets, and additional inline options or hashes. Package lines +using ranges such as `>=`, truncated/non-SHA-256-looking hash values, index or +other option lines, local/direct references, and other syntax that merely +contains a `--hash=` substring do not gain trusted candidate status. + +This parser is a **pre-materialization eligibility boundary**, not a dependency +solver. The exact trusted source path is recorded in the manifest and the +existing installer separately preflights every candidate as an independently +installable `pip --require-hashes` closure. That second proof remains mandatory: +pip's hash-checking mode intentionally fails when a requirement participating in +the installation is not fully hashed. Syntax qualification therefore cannot +substitute for dependency-closure proof. + +CWE-494 forbids downloading source or an executable from a remote location +without verifying origin and integrity (MITRE, 2026). A global +`--require-hashes` directive therefore cannot promote an unpinned or +range-pinned line into the networked coverage image. + +Unpinned notes, directive-only files, input files, deeper descendants, symbolic +links, pull-request-only files, malformed Git tree entries, and unsafe include +syntax remain excluded from the networked coverage image. + +## Operational reason + +Concrete environment locks are frequently organized below a `requirements` +directory and use role names such as `ci.txt` or `package.txt`. Ignoring those +safe base-owned locks leaves isolated coverage without runtime dependencies even +when the repository maintains a complete generated closure. The resulting import +failure measures the coverage image rather than the changed production code. + +Conversely, treating the presence of the substring `--hash=` as trust evidence +would let a range requirement, malformed digest, pip option, or path/URL include +cross the materialization boundary. The accepted design therefore combines +base-commit provenance, a narrow grammar, and an independent pip closure +preflight rather than relying on file names or hash-looking text alone. + +## Verification + +- A failing contract first proved that `requirements/ci.txt` was undiscoverable. +- A later RED security contract proved that range requirements, malformed + digests, pip option lines, absolute/traversing includes, and includes carrying + extra inline options could be materialized by the earlier substring test. +- Direct `requirements/*.txt` and nested-service equivalents remain eligible. +- A deeper `requirements/nested/ci.txt` path and unrelated `docs/ci.txt` remain + ineligible. +- Exact `==` package pins with complete SHA-256 hashes are accepted; `>=` and + malformed/truncated hash forms are rejected. +- Bounded relative includes such as `--requirement requirements-other.txt` and + `-r requirements/other.txt` are accepted. Current-directory prefixes + (`./requirements/other.txt`), empty path components (`requirements//other.txt`), + URL, absolute, traversal, home-expansion, query/fragment, backslash, and + option-like forms are rejected. +- A global `--require-hashes` directive combined with an unpinned requirement is + rejected rather than promoted into the networked coverage image. +- Only qualifying base-owned candidates are emitted from realistic temporary Git + bases; unpinned `.in`, note, and hostile direct-child files remain absent. +- Exact-head Python 3.14 quality requires the focused suite, complete central + suite, 100% production statement and branch coverage, 100% public docstrings, + compilation, and security/supply-chain workflows. Python 3.10 compatibility + remains a separate minimum-runtime contract. + +## References + +MITRE. (2026). *CWE-494: Download of code without integrity check*. +https://cwe.mitre.org/data/definitions/494.html + +Python Packaging Authority. (2026). *Install requires vs requirements files*. +Python Packaging User Guide. Retrieved August 10, 2026, from +https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/ + +Python Packaging Authority. (2026). *Repeatable installs*. pip documentation. +Retrieved August 10, 2026, from +https://pip.pypa.io/en/latest/topics/repeatable-installs/ + +Python Packaging Authority. (2026). *Requirements file format*. pip +26.1.2 documentation. Retrieved August 10, 2026, from +https://pip.pypa.io/en/stable/reference/requirements-file-format/ + +Python Packaging Authority. (2026). *Secure installs*. pip 26.1.2 +documentation. Retrieved August 10, 2026, from +https://pip.pypa.io/en/stable/topics/secure-installs/ diff --git a/scripts/ci/materialize_base_python_requirements.py b/scripts/ci/materialize_base_python_requirements.py old mode 100755 new mode 100644 index b16d4c745..5b6ecb653 --- a/scripts/ci/materialize_base_python_requirements.py +++ b/scripts/ci/materialize_base_python_requirements.py @@ -200,8 +200,6 @@ def _is_bounded_requirement_include(line: str) -> bool: and ".." not in include_path.parts and _is_candidate_lock_path(include_path) ) - - def _requirement_lines(content: bytes) -> list[str]: """Return logical requirement lines, joining backslash line-continuations. diff --git a/tests/test_requirements_directory_lock_materialization.py b/tests/test_requirements_directory_lock_materialization.py new file mode 100644 index 000000000..f80c4df60 --- /dev/null +++ b/tests/test_requirements_directory_lock_materialization.py @@ -0,0 +1,191 @@ +"""Regression contracts for trusted locks kept in a requirements directory.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path, PurePosixPath + +import pytest + +from scripts.ci import materialize_base_python_requirements as materializer + + +def _git(repo: Path, *args: str) -> str: + """Run one deterministic Git command in a temporary fixture repository.""" + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def test_requirements_directory_txt_is_a_candidate_lock_path() -> None: + """A direct ``requirements/*.txt`` lock is discoverable by its safe path.""" + assert materializer._is_candidate_lock_path(PurePosixPath("requirements/ci.txt")) + assert materializer._is_candidate_lock_path( + PurePosixPath("services/scoring_service/requirements/package.txt") + ) + assert not materializer._is_candidate_lock_path( + PurePosixPath("requirements/nested/ci.txt") + ) + assert not materializer._is_candidate_lock_path(PurePosixPath("docs/ci.txt")) + + +def test_materializes_hash_pinned_requirements_directory_lock( + tmp_path: Path, +) -> None: + """The exact base ``requirements/ci.txt`` closure reaches offline coverage.""" + repo = tmp_path / "repo" + requirements_dir = repo / "requirements" + requirements_dir.mkdir(parents=True) + _git(repo, "init") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + + (requirements_dir / "ci.txt").write_text( + "numpy==2.5.1 --hash=sha256:" + ("a" * 64) + "\n", + encoding="utf-8", + ) + (requirements_dir / "ci.in").write_text("numpy>=2\n", encoding="utf-8") + (requirements_dir / "notes.txt").write_text( + "human-readable notes only\n", encoding="utf-8" + ) + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base_sha = _git(repo, "rev-parse", "HEAD") + + output = tmp_path / "output" + manifest = materializer.materialize(repo, base_sha, output) + + assert manifest == [ + {"file": "requirements-000.txt", "source": "requirements/ci.txt"} + ] + assert (output / "requirements-000.txt").read_text(encoding="utf-8").startswith( + "numpy==2.5.1" + ) + + +@pytest.mark.parametrize( + ("content", "expected"), + ( + (b"--require-hashes\n", False), + (b"--require-hashes\ndemo==1\n", False), + ( + b"--require-hashes\ndemo==1 --hash=sha256:" + + (b"a" * 64) + + b"\n", + True, + ), + ( + b"pinned==1 --hash=sha256:" + + (b"b" * 64) + + b"\nunpinned==2\n", + False, + ), + (b"--index-url https://packages.example.invalid/simple\n", False), + (b"--requirement requirements-other.txt\n", True), + (b"-r requirements/other.txt\n", True), + (b"--requirement other.txt\n", False), + (b"-r ./locks/other.txt\n", False), + (b"-r ./requirements/other.txt\n", False), + (b"-r requirements/./other.txt\n", False), + (b"-r requirements//other.txt\n", False), + ), +) +def test_global_hash_directive_does_not_replace_per_requirement_trust( + content: bytes, + expected: bool, +) -> None: + """Only substantive hashed pins or bounded requirement includes qualify.""" + assert materializer._is_hash_pinned(content) is expected + + +@pytest.mark.parametrize( + "unsafe_content", + ( + b"demo>=1 --hash=sha256:" + (b"a" * 64) + b"\n", + b"demo==1 --hash=sha256:not-a-complete-digest\n", + b"--index-url https://packages.example.invalid/simple --hash=sha256:" + + (b"a" * 64) + + b"\n", + b"-r /tmp/absolute.txt\n", + b"--requirement ../parent.txt\n", + b"-r nested/../../escape.txt\n", + b"--requirement other.txt --hash=sha256:" + (b"a" * 64) + b"\n", + b"--requirement https://packages.example.invalid/lock.txt\n", + b"--requirement ~/private-lock.txt\n", + b"--requirement -option-like.txt\n", + b"--requirement locks\\windows.txt\n", + b"--requirement other.txt?variant=1\n", + b"--requirement other.txt#fragment\n", + ), +) +def test_unsafe_requirement_lines_are_rejected_before_materialization( + unsafe_content: bytes, +) -> None: + """Unsafe package and include syntax never gains trusted candidate status.""" + assert not materializer._is_hash_pinned(unsafe_content) + + +@pytest.mark.parametrize( + "unsafe_text", + ( + "demo>=1 --hash=sha256:" + ("a" * 64) + "\n", + "demo==1 --hash=sha256:not-a-complete-digest\n", + "--index-url https://packages.example.invalid/simple --hash=sha256:" + + ("a" * 64) + + "\n", + "-r /tmp/absolute.txt\n", + "--requirement ../parent.txt\n", + "--requirement https://packages.example.invalid/lock.txt\n", + ), +) +def test_unsafe_requirements_directory_candidate_is_excluded_from_manifest( + tmp_path: Path, + unsafe_text: str, +) -> None: + """Unsafe direct-child content is excluded before entering the build context.""" + repo = tmp_path / "repo" + requirements_dir = repo / "requirements" + requirements_dir.mkdir(parents=True) + _git(repo, "init") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + (requirements_dir / "ci.txt").write_text(unsafe_text, encoding="utf-8") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base_sha = _git(repo, "rev-parse", "HEAD") + + output = tmp_path / "output" + manifest = materializer.materialize(repo, base_sha, output) + + assert manifest == [] + assert (output / "manifest.json").read_text(encoding="utf-8") == "[]\n" + + +def test_rejects_global_hash_directive_with_unpinned_requirement( + tmp_path: Path, +) -> None: + """A global directive cannot make an unpinned direct-child lock trusted.""" + repo = tmp_path / "repo" + requirements_dir = repo / "requirements" + requirements_dir.mkdir(parents=True) + _git(repo, "init") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + + (requirements_dir / "ci.txt").write_text( + "--require-hashes\ndemo==1\n", + encoding="utf-8", + ) + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + base_sha = _git(repo, "rev-parse", "HEAD") + + output = tmp_path / "output" + manifest = materializer.materialize(repo, base_sha, output) + + assert manifest == [] + assert not materializer._is_hash_pinned(b"--require-hashes\ndemo==1\n") + assert (output / "manifest.json").read_text(encoding="utf-8") == "[]\n"