diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5928dd5..8fd61c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,15 @@ jobs: run: uv sync --locked --dev --extra docs --python "${{ steps.setup-python.outputs.python-path }}" - name: Build docs - run: uv run mkdocs build --strict + env: + ML4T_DOCS_COMMIT: ${{ github.sha }} + run: | + ML4T_DOCS_VERSION="$(uv run python -c 'from ml4t.models import __version__; print(__version__)')" + export ML4T_DOCS_VERSION + uv run mkdocs build --strict + ML4T_DOCS_SITE=site RELEASE_COMMIT="$ML4T_DOCS_COMMIT" \ + RELEASE_VERSION="$ML4T_DOCS_VERSION" \ + uv run python scripts/ci/verify_docs_deployment.py build-candidate: name: Build Candidate diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..b8c044c --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,63 @@ +name: Docs + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.12" + UV_NO_SOURCES: "1" + +jobs: + build: + name: Strict Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + with: + version: "0.10.9" + + - name: Set up Python + id: setup-python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install locked documentation environment + run: uv sync --locked --dev --extra docs --python "${{ steps.setup-python.outputs.python-path }}" + + - name: Record source identity + run: | + echo "ML4T_DOCS_VERSION=$(uv run python -c 'from ml4t.models import __version__; print(__version__)')" >> "$GITHUB_ENV" + echo "ML4T_DOCS_COMMIT=${GITHUB_SHA}" >> "$GITHUB_ENV" + + - name: Build and verify documentation + run: | + uv run mkdocs build --strict + ML4T_DOCS_SITE=site RELEASE_COMMIT="$ML4T_DOCS_COMMIT" \ + RELEASE_VERSION="$ML4T_DOCS_VERSION" \ + uv run python scripts/ci/verify_docs_deployment.py + + - name: Upload rendered documentation + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: docs-${{ github.sha }} + path: site/ + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbf9671..8db2ff6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -169,6 +169,9 @@ jobs: encoding="utf-8", ) PY + ML4T_DOCS_SITE=site RELEASE_COMMIT="$ML4T_DOCS_COMMIT" \ + RELEASE_VERSION="$ML4T_DOCS_VERSION" \ + uv run python scripts/ci/verify_docs_deployment.py - name: Require deploy credentials env: diff --git a/README.md b/README.md index 68b66ba..1fda9fd 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Finance-native model implementations for latent-factor estimation, stochastic discount factor learning, direct asset prediction, and end-to-end portfolio learning. -Documentation: https://ml4trading.io/docs/models/ +Documentation: [ml4trading.io/docs/models](https://www.ml4trading.io/docs/models/) ## Part of the ML4T Library Ecosystem @@ -213,3 +213,26 @@ Portfolio models learn allocations directly: - [Architecture](docs/reference/architecture.md) - [API Reference](docs/api/index.md) - [Book Guide](docs/book-guide/index.md) + +## Development + +Install the locked development environment, then run the repository gates before opening a pull +request: + +```bash +uv sync --locked --dev --extra docs +uv run ruff check src/ tests/ examples/ scripts/ +uv run ruff format --check src/ tests/ examples/ scripts/ +uv run ty check +uv run pytest tests/ -q --cov-report=json:coverage.json +uv run python scripts/ci/check_coverage.py coverage.json +uv run mkdocs build --strict +uv build +``` + +## Project Links + +- [Documentation](https://www.ml4trading.io/docs/models/) +- [Issues](https://github.com/ml4t/models/issues) +- [Releases](https://github.com/ml4t/models/releases) +- [License](LICENSE) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 1fd85f1..2cacd08 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -2,6 +2,9 @@ {% block extrahead %} {{ super() }} + + + None: + super().__init__() + self.values: dict[str, str] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag != "meta": + return + values = dict(attrs) + name = values.get("name") + content = values.get("content") + if name in REQUIRED_META and content is not None: + self.values[name] = content + + +def html_identity_failures(html: str, expected: dict[str, str], source: str) -> list[str]: + """Return missing or mismatched documentation identity fields.""" + parser = _MetadataParser() + parser.feed(html) + expected_meta = { + "ml4t-library": expected["library"], + "ml4t-version": expected["version"], + "ml4t-commit": expected["commit"], + } + return [ + f"{source}: {name} is {parser.values.get(name)!r}, expected {value!r}" + for name, value in expected_meta.items() + if parser.values.get(name) != value + ] + + +def verify_site(site: Path, expected: dict[str, str]) -> None: + """Verify that every rendered page exposes the expected release identity.""" + pages = sorted( + page for page in site.rglob("*.html") if "overrides" not in page.relative_to(site).parts + ) + if not pages: + raise RuntimeError(f"{site}: no rendered HTML pages found") + failures = [ + failure + for page in pages + for failure in html_identity_failures( + page.read_text(encoding="utf-8"), expected, str(page.relative_to(site)) + ) + ] + if failures: + raise RuntimeError("rendered documentation identity did not match:\n" + "\n".join(failures)) def _read_identity(url: str, commit: str, attempt: int) -> object: @@ -22,10 +74,18 @@ def _read_identity(url: str, commit: str, attempt: int) -> object: return json.load(response) +def _read_page(url: str, commit: str, attempt: int) -> str: + query = urlencode({"commit": commit, "attempt": attempt}) + request = Request(f"{url}?{query}", headers={"User-Agent": USER_AGENT}) + with urlopen(request, timeout=20) as response: + return response.read().decode("utf-8") + + def verify( - urls: tuple[str, ...], + identity_urls: tuple[str, ...], expected: dict[str, str], *, + page_urls: tuple[str, ...] = (), attempts: int = 24, retry_seconds: float = 10, ) -> None: @@ -36,9 +96,16 @@ def verify( last_error: Exception | None = None for attempt in range(attempts): try: - observed = [_read_identity(url, expected["commit"], attempt) for url in urls] + observed = [_read_identity(url, expected["commit"], attempt) for url in identity_urls] + page_failures = [ + failure + for url in page_urls + for failure in html_identity_failures( + _read_page(url, expected["commit"], attempt), expected, url + ) + ] last_error = None - if all(value == expected for value in observed): + if all(value == expected for value in observed) and not page_failures: return except (OSError, URLError, ValueError) as error: last_error = error @@ -58,12 +125,20 @@ def main() -> None: "library": "models", "version": os.environ["RELEASE_VERSION"], } + site = os.environ.get("ML4T_DOCS_SITE") + if site is not None: + verify_site(Path(site), expected) + return verify( ( "https://www.ml4trading.io/docs/models/release.json", f"https://www.ml4trading.io/docs/models/releases/{expected['version']}/release.json", ), expected, + page_urls=( + "https://www.ml4trading.io/docs/models/", + f"https://www.ml4trading.io/docs/models/releases/{expected['version']}/", + ), ) diff --git a/src/ml4t/models/_version.py b/src/ml4t/models/_version.py index b227e06..35e7d5b 100644 --- a/src/ml4t/models/_version.py +++ b/src/ml4t/models/_version.py @@ -1,3 +1,3 @@ """Package version shared by public and internal runtime metadata.""" -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index fe18db6..f7e97f4 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -168,12 +168,35 @@ def read_identity(_url: str, _commit: str, attempt: int) -> object: monkeypatch.setattr(verify_docs_deployment.time, "sleep", sleeps.append) with pytest.raises(RuntimeError, match="deployed documentation identity did not match"): - verify_docs_deployment.verify(("https://example.test/release.json",), {"commit": COMMIT}) + verify_docs_deployment.verify( + ("https://example.test/release.json",), + {"commit": COMMIT}, + ) assert attempts == list(range(24)) assert sleeps == [10] * 23 +def test_rendered_docs_verifier_requires_exact_release_identity(tmp_path: Path) -> None: + expected = {"commit": COMMIT, "library": "models", "version": __version__} + index = tmp_path / "index.html" + index.write_text( + '' + f'' + f'', + encoding="utf-8", + ) + + verify_docs_deployment.verify_site(tmp_path, expected) + html = index.read_text(encoding="utf-8").replace( + f'', + '', + ) + index.write_text(html, encoding="utf-8") + with pytest.raises(RuntimeError, match="ml4t-version is 'wrong'"): + verify_docs_deployment.verify_site(tmp_path, expected) + + def test_release_workflow_reuses_one_commit_bound_candidate() -> None: ci = _workflow("ci.yml") release_workflow = _workflow("release.yml") @@ -218,12 +241,24 @@ def test_release_workflow_reuses_one_commit_bound_candidate() -> None: def test_only_release_workflow_can_deploy_documentation() -> None: - for name in ("ci.yml", "ecosystem.yml"): + for name in ("ci.yml", "docs.yml", "ecosystem.yml"): assert "push-to-another-repository" not in ( ROOT / ".github" / "workflows" / name ).read_text(encoding="utf-8") +def test_standalone_docs_workflow_is_read_only_and_verifies_strict_build() -> None: + workflow = _workflow("docs.yml") + build = workflow["jobs"]["build"] + commands = "\n".join(step.get("run", "") for step in build["steps"]) + + assert workflow["permissions"] == {"contents": "read"} + assert "permissions" not in build + assert "uv run mkdocs build --strict" in commands + assert "ML4T_DOCS_SITE=site" in commands + assert "scripts/ci/verify_docs_deployment.py" in commands + + def test_ci_uses_locked_dependencies_and_requires_cuda_for_releases() -> None: workflows = [ (ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8")