diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 7a4d61b..dd3912d 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -43,6 +43,60 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule `test_auto_mixed_pass_stops_ignoring_undecided_distractors` + `test_mixed_static_arming_pass_stops_ignoring_fail_armed`. +## From 2026-07-24 publish-github-releases review + +- [ ] **sdist-contents assertion** — build the sdist and assert it contains only + intended paths. `pyproject.toml` declares no `[tool.hatch.build.targets.sdist]` + section, so hatchling's default selection honors only the **root** `.gitignore` + and sweeps in everything else sitting in the tree at build time. Two distinct + consequences, worth keeping apart: + - **What actually reaches PyPI today: nothing unintended.** A local `uv build` + in a developed worktree produces a 135 MB sdist carrying + `evalboard/node_modules/**` (8520 files) and `evalboard/.next/**` (190), + because `evalboard/.gitignore` is nested and therefore not honored. CI is + spared only incidentally — `release.yml` never runs `npm`/`pnpm install`, so + those paths do not exist on the runner at `uv build` time. Verified against + the published artifacts: the 0.8.9 and 0.8.2 sdists on PyPI are ~7.5 MB / + ~550 files with **zero** `node_modules` entries. (A dirty-tree release would + not silently ship JS either — 135 MB exceeds PyPI's 100 MB per-file limit, so + it fails at upload. The real exposure is a broken release, not a stealth one.) + - **The live hazard is untracked files a workflow leaves in the tree**, which + hatchling *does* package: a `release-notes.md` written at the repo root by a + CI step landed in `coder_eval-X.Y.Z/release-notes.md` (verified by building + it). This is why the "Publish GitHub Release" step writes to + `${RUNNER_TEMP}` — a convention no check enforces. + + Not cheap: needs a real `uv build` inside the test suite (slow) plus a decision + on whether to add an explicit sdist include/exclude allowlist, which changes + published artifacts. Worth pairing with the allowlist so the contract is + declared rather than inferred from hatchling's defaults — caught in the + 2026-07-24 ci/publish-github-releases review. +- [ ] **CE032 — run the existing AST lint rules over Python embedded in + `.github/workflows/*.yml`.** CE008/CE009/CE010 already forbid unencoded + `read_text`/`open`/`subprocess.run`, but `tests/lint/runner.py::check_paths` + walks only `*.py` under `src/`, so Python inside a `run:` heredoc is invisible + to ruff, pyright, pytest, coverage *and* the CE runner. Would have caught the + four unencoded `read_text`/`write_text` calls fixed by hand in this review + (`release.yml` ×2, `publish-testpypi.yml` ×2). Needs a heredoc extractor + (`python3 - <<'PY' … PY` → dedent → `ast.parse`) with line-number mapping back + to the YAML; wire as a `tests/test_custom_lint.py` class like CE027–CE031 + rather than a `BaseRule`. Also consider extending CE008 to `write_text` (it + matches only `read_text` today, though `src/` happens to be clean). +- [ ] **CE033 — interpreter heredocs in `.github/workflows/**` must use a quoted + delimiter** (`<<'PY'`, not `< + +Writes the section body (heading line excluded, stripped) to ````, +or an empty file when no matching section exists — the caller treats an empty +file as "fall back to GitHub's generated notes". +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +_REPO_ROOT = Path(__file__).resolve().parents[2] + +# semantic-release's default changelog template renders headings as +# "## v0.8.9 (2026-07-23)". Capture everything after that heading line up to the +# next "## v" heading, or EOF for the newest entry (the common case on a release +# run, since the just-generated section is at the top of the file). +# +# The trailing \b on the version is load-bearing: without it "0.8.1" would match +# the "## v0.8.10" heading and publish the wrong section. +_SECTION_TEMPLATE = r"(?m)^## v{version}\b.*?$(.*?)(?=^## v|\Z)" + + +def extract_section(changelog_text: str, version: str) -> str: + """Return the changelog body for ``version``, or ``""`` when absent. + + ``version`` is regex-escaped, so a PEP 440 local/prerelease version + containing metacharacters (``+``, ``!``) is matched literally. + """ + pattern = _SECTION_TEMPLATE.format(version=re.escape(version)) + match = re.search(pattern, changelog_text, re.S) + return match.group(1).strip() if match else "" + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print("usage: release_notes.py ", file=sys.stderr) + return 2 + version, out_path = argv[1], argv[2] + # Encoding is pinned rather than left to the ambient locale: CHANGELOG.md + # verifiably carries non-ASCII (arrows, em dashes, check marks), so a + # non-UTF-8 locale would raise UnicodeDecodeError here and — under the + # caller's `continue-on-error: true` — silently publish no Release at all. + text = (_REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + body = extract_section(text, version) + Path(out_path).write_text(body, encoding="utf-8") + if not body: + # Surfaced as a workflow annotation; the caller falls back to + # --generate-notes when the file is empty. + print(f"::warning::no CHANGELOG section found for v{version}; using GitHub's generated notes") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index 0a86425..3bf1dbf 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -64,8 +64,14 @@ jobs: CURRENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") # PEP 440 dev release: sorts before the base release, unique per run. DEV_VERSION="${CURRENT_VERSION}.dev${RUN_NUMBER}" - python3 - <rc` version, then build and # publish the wheel to public PyPI + a `:` GHCR image. It does NOT move -# `:latest`, tag, commit, or push to main. This lets a branch be dry-run on the +# `:latest`, tag, commit, push to main, or create a GitHub Release (there is no +# tag for a Release to point at). This lets a branch be dry-run on the # ADO nightly infra before merge (pinned via the pipeline's `coderEvalVersion`). # The `bump` input is ignored off main. On `main` the behavior is unchanged. @@ -160,10 +161,12 @@ jobs: version = os.environ["PRE_VERSION"] for path, key in (("pyproject.toml", "version"), ("src/coder_eval/__init__.py", "__version__")): p = pathlib.Path(path) - new, n = re.subn(rf'(?m)^{key}\s*=\s*".+?"$', f'{key} = "{version}"', p.read_text(), count=1) + # encoding pinned, not left to the ambient locale: a non-UTF-8 default + # would raise UnicodeDecodeError mid-release on a non-ASCII source file. + new, n = re.subn(rf'(?m)^{key}\s*=\s*".+?"$', f'{key} = "{version}"', p.read_text(encoding="utf-8"), count=1) if n != 1: raise SystemExit(f"version pattern did not match {path} (matched {n})") - p.write_text(new) + p.write_text(new, encoding="utf-8") print(f"Stamped prerelease version: {version}") PY uv lock @@ -241,6 +244,106 @@ jobs: path: dist/ if-no-files-found: error + # Publish the GitHub Release for the tag pushed above. semantic-release runs + # with --no-vcs-release because it also runs --no-push (the commit is amended + # and the tag re-pointed first), so it cannot create the release itself -- it + # happens here, once the tag is actually on the remote. A published Release is + # what GitHub Marketplace listings are cut from, so every release needs one. + # (`gh release create` cannot tick the "Publish this Action to the + # Marketplace" checkbox -- that stays a one-time manual step in the GitHub UI + # on the first Release; every subsequent release then lists automatically.) + # + # Deliberately placed AFTER "Build wheel + sdist" and "Upload dist for PyPI + # publish" rather than at the earliest legal point after the tag push: those + # two steps are the last ones that can still fail for an already-tagged + # version, and a Release announcing a version whose artifacts never built is + # worse than a missing Release. This narrows the window rather than closing + # it -- publish-pypi is a separate job, so the actual upload to PyPI still + # happens after this. Running here also keeps a slow/hung `gh` API call from + # eating the 15-minute job budget BEFORE the artifacts are safe, which would + # produce exactly the stranded-tag state the note below warns about. + # + # Notes are the CHANGELOG section semantic-release just generated for this + # version, sliced by .github/scripts/release_notes.py (a real module, so the + # regex is unit-tested -- see tests/test_release_notes.py); an empty result + # falls back to GitHub's generated notes. + # + # ACCEPTED RISK: those notes render commit subjects, i.e. squashed PR titles. + # The Release body is a first-party surface that GitHub also fans out in + # notification emails, so it carries text that was reviewed as *code*, not as + # markdown -- a PR title can land an arbitrary link in it. Bounded to + # content/link spoofing (GitHub strips raw HTML from release bodies) and + # gated by this repo's mandatory PR review. Revisit with `--draft` plus a + # human glance, or link-stripping in release_notes.py, if the repo ever takes + # drive-by contributions. + - name: Publish GitHub Release + id: gh_release + if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' + # Best-effort, mirroring the GHCR steps below. main, the version tag, the + # moving major tag, and the dist artifact are all in place by the time this + # runs, so a transient GitHub API failure here must not fail the job: the + # publish-pypi job is `needs: release`, so a failure would SKIP the PyPI + # publish of an already-tagged version and strand `@vN` on an action.yml pin + # whose version was never published. The next step turns the swallowed + # failure into a loud annotation instead of a collapsed step marker. + continue-on-error: true + env: + # The app token, not GITHUB_TOKEN: the workflow's `permissions:` are + # contents: read, and `gh release create` needs contents: write. Granting + # the job contents: write to use GITHUB_TOKEN here would ADD a second + # write credential rather than remove one -- `actions/checkout` above + # already persists this same app token in .git/config for every step in + # the job, so scoping it out of this one step's env buys no isolation. + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # Passed via env (not interpolated into the script) per GitHub's + # injection guidance. + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + # Written under RUNNER_TEMP, never the repo root: hatchling's default sdist + # file selection sweeps in untracked files at the root (verified -- it ships + # even git-ignored paths), so a notes file left in the tree would leak into + # the sdist that the "Build wheel + sdist" step above produced and the + # publish-pypi job uploads. + NOTES_FILE="${RUNNER_TEMP}/release-notes.md" + python3 .github/scripts/release_notes.py "$VERSION" "$NOTES_FILE" + # Empty notes file => no CHANGELOG section was found (the script already + # emitted the ::warning::); let GitHub generate the body instead. + if [ -s "$NOTES_FILE" ]; then + NOTES=(--notes-file "$NOTES_FILE") + else + NOTES=(--generate-notes) + fi + gh release create "v${VERSION}" \ + --title "v${VERSION}" \ + --verify-tag \ + --latest \ + "${NOTES[@]}" + + # `continue-on-error` above hides a failure in a collapsed step marker that + # nobody expands on an otherwise-green release run -- the same silence that + # let "no GitHub Releases at all" go unnoticed until this PR. Re-raise it as + # an ::error annotation plus a run-summary block, WITHOUT failing the job + # (that would skip publish-pypi, see above). + - name: Flag missing GitHub Release + if: always() && steps.gh_release.outcome == 'failure' + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + MAJOR="v${VERSION%%.*}" + echo "::error title=GitHub Release not published::v${VERSION} was tagged and its artifacts built, but 'gh release create' failed. Create the Release by hand so ${MAJOR} and the Marketplace listing resolve." + { + echo "### :x: GitHub Release for \`v${VERSION}\` was NOT created" + echo + echo "The version tag, the moving \`${MAJOR}\` tag, and the PyPI artifacts are unaffected —" + echo "only \`gh release create\` failed. Create it by hand:" + echo + echo '```sh' + echo "gh release create v${VERSION} --title v${VERSION} --verify-tag --latest --generate-notes" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + # Build + push the agent image HERE, in the same job that produced the # version, so the `:` tag is built from the correct pyproject (bumped # by semantic-release on main, or the stamped rc on a prerelease). diff --git a/Makefile b/Makefile index dcc075c..974cc6c 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,17 @@ install: ## Install project with dev + uipath dependencies (hash-verified from uv sync --frozen --extra dev --extra uipath uv run pre-commit install +# `.github/scripts/` is in scope on purpose: release tooling that lives in a real +# module (rather than inline in a workflow `run:` heredoc) is exactly what ruff, +# pyright and pytest can see — leaving it unlinted would forfeit the reason it was +# extracted. +LINT_PATHS := src/ tests/ .github/scripts/ + format: ## Auto-format code with ruff - uv run ruff format src/ tests/ + uv run ruff format $(LINT_PATHS) check: ## Run linting checks - uv run ruff check src/ tests/ + uv run ruff check $(LINT_PATHS) lint: ## Run custom architectural lint rules (CE001+) uv run pytest tests/test_custom_lint.py -v --tb=short --no-header -p no:warnings @@ -42,8 +48,8 @@ test-cov: ## Run tests with coverage report verify: ## Run all verification steps (CI equivalent) - uv run ruff format --check src/ tests/ - uv run ruff check src/ tests/ + uv run ruff format --check $(LINT_PATHS) + uv run ruff check $(LINT_PATHS) uv run pyright uv run pytest tests/test_custom_lint.py -v --tb=short --no-header -p no:warnings # uv run pip-audit --desc --skip-editable diff --git a/tests/test_action_version_pin.py b/tests/test_action_version_pin.py new file mode 100644 index 0000000..4b14431 --- /dev/null +++ b/tests/test_action_version_pin.py @@ -0,0 +1,59 @@ +"""``action.yml``'s ``version:`` default must equal ``pyproject.toml``'s version. + +The published composite action installs ``coder-eval==``, so a +consumer pinning ``UiPath/coder_eval@vX.Y.Z`` (or the moving ``@v0``) must get +X.Y.Z and not some other release. ``release.yml``'s "Regenerate uv.lock, bump +action.yml pin, and amend release commit" step sed-bumps the default inside the +release commit, which makes the invariant mechanically true *at rest* — every +commit on main has the two in agreement. + +Nothing asserted it, which is how ``action.yml`` shipped pinned to 0.8.6 while +main was already 0.8.9: the sed lives on the release path only, so a hand-edit +(or a release whose amend step was skipped) drifts silently and ``@v0`` +consumers install a version other than the tag they pinned. + +This also guards the ``# <-- kept in sync`` anchor itself: ``release.yml``'s sed +is keyed on that exact trailing comment, so a reformat that detaches it turns +the release-time bump into a no-op (caught there by a ``grep -q`` guard, but +only after the tag exists). +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +ACTION_YML = REPO_ROOT / "action.yml" +PYPROJECT = REPO_ROOT / "pyproject.toml" + +# Mirrors the anchor release.yml's sed matches: indentation-tolerant, keyed on the +# unique trailing comment. +_PIN_PATTERN = re.compile(r'^[ \t]*default: "(?P\d+\.\d+\.\d+)"[ \t]+# <-- kept in sync', re.MULTILINE) + + +def _project_version() -> str: + return tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["project"]["version"] + + +def test_action_version_pin_anchor_is_present_and_unique(): + """release.yml's sed is keyed on this anchor; a detached/duplicated one breaks it.""" + matches = _PIN_PATTERN.findall(ACTION_YML.read_text(encoding="utf-8")) + assert len(matches) == 1, ( + f"expected exactly one '# <-- kept in sync' version pin in action.yml, found {len(matches)}; " + "release.yml's sed anchor is keyed on it" + ) + + +def test_action_version_pin_matches_pyproject_version(): + match = _PIN_PATTERN.search(ACTION_YML.read_text(encoding="utf-8")) + assert match is not None, "version pin anchor missing from action.yml" + pinned = match.group("version") + expected = _project_version() + assert pinned == expected, ( + f"action.yml pins coder-eval=={pinned} but pyproject.toml is {expected}. " + f"Consumers of UiPath/coder_eval@v{expected} would install {pinned}. " + "Update the `default:` in action.yml (release.yml bumps it automatically on release)." + ) diff --git a/tests/test_release_notes.py b/tests/test_release_notes.py new file mode 100644 index 0000000..c1fefde --- /dev/null +++ b/tests/test_release_notes.py @@ -0,0 +1,145 @@ +"""Unit tests for ``.github/scripts/release_notes.py``. + +The notes slicer runs exactly once per release, on ``main``, after the version +tag has already been pushed — and the prerelease dispatch skips the step, so +there is no rehearsal path. These tests are the only thing that exercises the +regex and its three failure modes before a real release depends on it. + +The final test couples the regex to semantic-release's *actual* rendering of the +real ``CHANGELOG.md``: a changelog-template or heading-format change upstream +would otherwise degrade every Release body to ``--generate-notes`` with only a +``::warning::`` inside a ``continue-on-error: true`` step as the signal. +""" + +from __future__ import annotations + +import importlib.util +import tomllib +from pathlib import Path +from types import ModuleType + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / ".github" / "scripts" / "release_notes.py" + + +def _load_script() -> ModuleType: + """Import the script by path — ``.github/scripts`` is not an importable package.""" + spec = importlib.util.spec_from_file_location("release_notes", SCRIPT_PATH) + assert spec is not None and spec.loader is not None, f"cannot load {SCRIPT_PATH}" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +release_notes = _load_script() + + +CHANGELOG = """# CHANGELOG + + + +## v0.8.10 (2026-07-24) + +### Bug Fixes + +- Newest entry + +## v0.8.9 (2026-07-23) + +### Features + +- Middle entry with a link ([#38](https://example.invalid/pull/38)) + +## v0.8.1 (2026-07-01) + +### Bug Fixes + +- Oldest entry +""" + + +class TestExtractSection: + def test_newest_section_stops_at_next_heading(self): + """The \\Z branch is not taken when a later section exists.""" + body = release_notes.extract_section(CHANGELOG, "0.8.10") + assert body == "### Bug Fixes\n\n- Newest entry" + assert "v0.8.9" not in body + + def test_middle_section_is_bounded_on_both_sides(self): + body = release_notes.extract_section(CHANGELOG, "0.8.9") + assert body.startswith("### Features") + assert body.endswith("([#38](https://example.invalid/pull/38))") + assert "Newest entry" not in body + assert "Oldest entry" not in body + + def test_last_section_in_file_uses_eof_branch(self): + """The (?=^## v|\\Z) alternation's \\Z arm — no trailing heading to stop at.""" + assert release_notes.extract_section(CHANGELOG, "0.8.1") == "### Bug Fixes\n\n- Oldest entry" + + def test_absent_version_returns_empty_string(self): + """Empty string is the contract that makes release.yml fall back to --generate-notes.""" + assert release_notes.extract_section(CHANGELOG, "9.9.9") == "" + + def test_version_prefix_does_not_match_longer_version(self): + """Guards the trailing \\b: "0.8.1" must not slice the "v0.8.10" section. + + Without the word boundary this returns the 0.8.10 notes, i.e. a release + published with a *different version's* changelog as its body. + """ + body = release_notes.extract_section(CHANGELOG, "0.8.1") + assert "Newest entry" not in body + assert "Oldest entry" in body + + def test_regex_metacharacters_in_version_are_literal(self): + """re.escape() — a PEP 440 local version must not be read as a pattern.""" + text = "## v1.0.0+local.1 (2026-01-01)\n\n- Local build\n" + assert release_notes.extract_section(text, "1.0.0+local.1") == "- Local build" + # The unescaped form would let "." match any character. + assert release_notes.extract_section(text, "1.0.0+localX1") == "" + + def test_heading_only_section_yields_empty_body(self): + """A section with no content is indistinguishable from absent — both fall back.""" + assert release_notes.extract_section("## v1.2.3 (2026-01-01)\n", "1.2.3") == "" + + +class TestMain: + def test_writes_utf8_body_to_output_path(self, tmp_path: Path): + """Round-trip the non-ASCII bytes the real CHANGELOG.md contains.""" + out = tmp_path / "release-notes.md" + # Uses the repo's own CHANGELOG.md, which carries → — ✓. + version = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] + assert release_notes.main(["release_notes.py", version, str(out)]) == 0 + assert out.read_text(encoding="utf-8").strip() != "" + + def test_absent_version_writes_empty_file(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + out = tmp_path / "release-notes.md" + assert release_notes.main(["release_notes.py", "0.0.0", str(out)]) == 0 + assert out.exists(), "an empty file must still be created; release.yml tests it with [ -s ]" + assert out.read_text(encoding="utf-8") == "" + assert "::warning::" in capsys.readouterr().out + + @pytest.mark.parametrize("argv", [["release_notes.py"], ["release_notes.py", "1.2.3"]]) + def test_bad_argv_returns_usage_error(self, argv: list[str], capsys: pytest.CaptureFixture[str]): + assert release_notes.main(argv) == 2 + assert "usage:" in capsys.readouterr().err + + +def test_current_version_has_a_slicable_changelog_section(): + """Couples the regex to semantic-release's real rendering, not a fixture of it. + + ``pyproject.toml``'s version is always the most recently released one at rest, + so its section must exist in ``CHANGELOG.md`` and be non-empty. If a + ``[tool.semantic_release.changelog]`` template change drops the ``v`` prefix or + reshapes the heading, this fails here instead of silently degrading the next + release's notes. + """ + version = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] + changelog = (REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + body = release_notes.extract_section(changelog, version) + assert body, f"no CHANGELOG.md section for the current version v{version}" + # The slice must not bleed into the previous release's section. + assert not body.lstrip().startswith("## v") + assert "\n## v" not in body