diff --git a/autotest/test_programs.py b/autotest/test_programs.py index 6ebb6815..37141660 100644 --- a/autotest/test_programs.py +++ b/autotest/test_programs.py @@ -55,10 +55,16 @@ def isolated_cache(tmp_path, monkeypatch): return cache -def _make_zip(path: Path, files: dict[str, bytes]) -> Path: +def _make_zip(path: Path, files: dict[str, bytes], modes: dict[str, int] | None = None) -> Path: + """Write a zip like the real release archives: built on unix, with each + member's permissions recorded (0o755 unless overridden in `modes`).""" + modes = modes or {} with zipfile.ZipFile(path, "w") as zf: for name, content in files.items(): - zf.writestr(name, content) + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = (0o100000 | modes.get(name, 0o755)) << 16 + zf.writestr(info, content) return path @@ -200,6 +206,110 @@ def test_no_match_raises(self, tmp_path): archive, tmp_path / "out", release_tag="1.0", ostag="linux", subset={"nope"} ) + def test_non_program_files_skipped(self, tmp_path): + """mt3d-usgs archives bundle a LICENSE next to the binaries.""" + archive = _make_zip( + tmp_path / "linux.zip", + {"mt3dusgs": b"bin", "mt3dusgsdbl": b"bin", "LICENSE": b"text", "notes.txt": b"x"}, + modes={"LICENSE": 0o644, "notes.txt": 0o644}, + ) + dest = tmp_path / "out" + extracted = extract_release_archive(archive, dest, release_tag="v1.1.1", ostag="linux") + assert {p.name for p in extracted} == {"mt3dusgs", "mt3dusgsdbl"} + assert not (dest / "LICENSE").exists() + + def test_non_program_files_skipped_without_unix_modes(self, tmp_path): + """Archives built on Windows record no unix permissions.""" + archive = tmp_path / "win64.zip" + with zipfile.ZipFile(archive, "w") as zf: + for name in ("LICENSE", "mt3dusgs.exe", "README.md"): + info = zipfile.ZipInfo(name) + info.create_system = 0 + zf.writestr(info, b"x") + extracted = extract_release_archive(archive, tmp_path / "w", release_tag="1", ostag="win64") + assert [p.name for p in extracted] == ["mt3dusgs"] + + archive = tmp_path / "linux.zip" + with zipfile.ZipFile(archive, "w") as zf: + for name in ("LICENSE", "mt3dusgs", "README.md"): + info = zipfile.ZipInfo(name) + info.create_system = 0 + zf.writestr(info, b"x") + extracted = extract_release_archive(archive, tmp_path / "l", release_tag="1", ostag="linux") + assert [p.name for p in extracted] == ["mt3dusgs"] + + def test_release_tag_version_normalized(self, tmp_path): + archive = _make_zip(tmp_path / "linux.zip", {"mfusg": b"bin"}) + extracted = extract_release_archive( + archive, tmp_path / "out", release_tag="v1.5.1", ostag="linux" + ) + assert extracted[0].version == "1.5.1" + + def test_code_json_dbl_variants_and_undescribed_binaries(self, tmp_path): + archive = _make_zip( + tmp_path / "linux.zip", + { + "code.json": json.dumps( + { + "mf2005": {"version": "1.12.00", "double_switch": True}, + "swtv4": {"version": "4.00.05", "double_switch": True}, + } + ).encode(), + "mf2005": b"bin", + "mf2005dbl": b"bin", + "swtv4": b"bin", # double_switch, but no swtv4dbl shipped + "mf5to6": b"bin", # not described by code.json + }, + modes={"code.json": 0o644}, + ) + dest = tmp_path / "out" + extracted = extract_release_archive(archive, dest, release_tag="29.0", ostag="linux") + versions = {p.name: p.version for p in extracted} + assert versions == { + "mf2005": "1.12.00", + "mf2005dbl": "1.12.00", + "swtv4": "4.00.05", + "mf5to6": "29.0", + } + + dest2 = tmp_path / "out2" + extracted = extract_release_archive( + archive, dest2, release_tag="29.0", ostag="linux", subset={"mf2005dbl"} + ) + assert [p.name for p in extracted] == ["mf2005dbl"] + assert (dest2 / "mf2005dbl").exists() + assert not (dest2 / "mf2005").exists() + + def test_partially_unknown_subset_raises(self, tmp_path): + archive = _make_zip(tmp_path / "linux.zip", {"mf6": b"bin", "LICENSE": b"t"}) + with pytest.raises(ProgramInstallationError, match=r"\['nope'\].*available: mf6$"): + extract_release_archive( + archive, tmp_path / "out", release_tag="1.0", ostag="linux", subset={"mf6", "nope"} + ) + assert not (tmp_path / "out" / "mf6").exists() + + def test_reextract_overwrites_existing(self, tmp_path): + dest = tmp_path / "out" + dest.mkdir() + (dest / "mf6").write_bytes(b"old") + archive = _make_zip(tmp_path / "linux.zip", {"bin/mf6": b"new"}) + extract_release_archive(archive, dest, release_tag="1.0", ostag="linux") + assert (dest / "mf6").read_bytes() == b"new" + assert sorted(p.name for p in dest.iterdir()) == ["mf6"] + + def test_failed_write_leaves_no_part_file(self, tmp_path, monkeypatch): + archive = _make_zip(tmp_path / "linux.zip", {"mf6": b"bin"}) + dest = tmp_path / "out" + + def _boom(src, dst): + dst.write(b"partial") + raise OSError("disk full") + + monkeypatch.setattr("modflow_devtools.programs.shutil.copyfileobj", _boom) + with pytest.raises(OSError, match="disk full"): + extract_release_archive(archive, dest, release_tag="1.0", ostag="linux") + assert list(dest.iterdir()) == [] + class TestInstallationMetadata: def test_add_list_remove_roundtrip(self, isolated_cache): @@ -279,6 +389,48 @@ def test_get_executable_filters_by_version(self, isolated_cache, tmp_path): assert get_executable("mf6", version="6.7.0") == tmp_path / "6.7.0" / "mf6" assert get_executable("mf6", version="6.8.0") == tmp_path / "6.8.0" / "mf6" + def test_versions_compared_without_leading_v(self, isolated_cache, tmp_path): + (tmp_path / "mfusg").write_bytes(b"bin") + inst = register_installation("mfusg", "v1.5.1", tmp_path, ["mfusg"]) + assert inst.version == "1.5.1" + assert get_executable("mfusg", version="1.5.1") == tmp_path / "mfusg" + assert get_executable("mfusg", version="v1.5.1") == tmp_path / "mfusg" + assert get_executable("mfusg", version="1.5") is None + + def test_versions_compared_semantically(self, isolated_cache, tmp_path): + """pymake's code.json records mfusg's v1.5.00 release as '1.5'.""" + (tmp_path / "mfusg").write_bytes(b"bin") + register_installation("mfusg", "1.5", tmp_path, ["mfusg"]) + assert get_executable("mfusg", version="v1.5.00") == tmp_path / "mfusg" + assert get_executable("mfusg", version="1.5.0") == tmp_path / "mfusg" + assert get_executable("mfusg", version="1.5.1") is None + register_installation("mfusg", "v1.5.00", tmp_path, ["mfusg"]) + assert len(list_installed("mfusg")["mfusg"]) == 1 + + def test_non_pep440_versions_compared_as_strings(self, isolated_cache, tmp_path): + (tmp_path / "x").write_bytes(b"bin") + register_installation("x", "nightly-abc", tmp_path, ["x"]) + assert get_executable("x", version="nightly-abc") == tmp_path / "x" + assert get_executable("x", version="nightly-abd") is None + + def test_legacy_v_prefixed_ledger_entries_still_match(self, isolated_cache, tmp_path): + """Ledgers written before normalization recorded raw tags like 'v1.5.1'.""" + (tmp_path / "mfusg").write_bytes(b"bin") + meta = InstallationMetadata("mfusg") + meta.installations = [ + ProgramInstallation( + "v1.5.1", "linux", tmp_path, datetime.now(UTC), {"origin": "x"}, ["mfusg"] + ) + ] + meta.save() + assert get_executable("mfusg", version="1.5.1") == tmp_path / "mfusg" + # re-installing the same version replaces rather than duplicates + register_installation("mfusg", "1.5.1", tmp_path, ["mfusg"]) + assert len(list_installed("mfusg")["mfusg"]) == 1 + uninstall_program("mfusg", version="1.5.1") + assert not (tmp_path / "mfusg").exists() + assert list_installed("mfusg") == {} + def test_uninstall_removes_file_and_forgets(self, isolated_cache, tmp_path): exe = tmp_path / "mf6" exe.write_bytes(b"fake") @@ -724,4 +876,51 @@ def _fake_get(url, **kwargs): monkeypatch.setattr("modflow_devtools.programs.requests.get", _fake_get) with pytest.raises(ProgramInstallationError, match=r"1\.0\.0"): - get_release(repo="mfnwt", tag="not-a-real-tag") + get_release(repo="mfusg", tag="not-a-real-tag") + + @staticmethod + def _fake_releases(monkeypatch, releases): + class _FakeResponse: + def __init__(self, status_code, payload=None): + self.status_code = status_code + self._payload = payload + + def raise_for_status(self): + import requests + + if self.status_code >= 400: + raise requests.exceptions.HTTPError(response=self) + + def json(self): + return self._payload + + def _fake_get(url, **kwargs): + if "/releases/" in url: # /releases/latest or /releases/tags/{tag} + return _FakeResponse(404) + return _FakeResponse(200, releases) + + monkeypatch.setattr("modflow_devtools.programs.requests.get", _fake_get) + + def test_repo_without_releases_says_so(self, monkeypatch): + from modflow_devtools.programs import get_release, get_releases + + self._fake_releases(monkeypatch, []) + assert get_releases(repo="mfnwt") == [] + with pytest.raises(ProgramInstallationError, match="has no published releases"): + get_release(repo="mfnwt") + + def test_drafts_excluded(self, monkeypatch): + from modflow_devtools.programs import get_release, get_releases + + self._fake_releases(monkeypatch, [{"tag_name": "v3.3", "draft": True}]) + assert get_releases(repo="vs2dt") == [] + with pytest.raises(ProgramInstallationError, match="has no published releases"): + get_release(repo="vs2dt") + + def test_latest_only_offered_with_a_full_release(self, monkeypatch): + from modflow_devtools.programs import get_release, get_releases + + self._fake_releases(monkeypatch, [{"tag_name": "2.0rc1", "prerelease": True}]) + assert get_releases(repo="x") == ["2.0rc1"] + with pytest.raises(ProgramInstallationError, match=r"'latest' not found.*: 2\.0rc1$"): + get_release(repo="x") diff --git a/docs/md/dev/programs.md b/docs/md/dev/programs.md index b719a0b6..46ceed7c 100644 --- a/docs/md/dev/programs.md +++ b/docs/md/dev/programs.md @@ -57,9 +57,9 @@ No registry, no bootstrap file, and - unlike the first iteration - no fixed allo | `modflow6` | One archive per ostag containing `mf6`/`zbud6`/`mf5to6`/`libmf6`, no `code.json` | Shared release tag | | `modflow6-nightly-build` | Same shape as `modflow6`, nightly tags | Shared nightly tag | -**Why open rather than allowlisted:** a growing number of individual program repos already publish releases in the same single-program shape as `modflow6` (one archive per ostag, no `code.json`) - confirmed live: `mfnwt`, `mt3d-usgs`, `vs2dt`, `gridgen`, `triangle`, `zonbud`, `zonbudusg` all do, alongside `mf6`/`mf6-nightly`. That's already 8 of the ~17 programs `executables` bundles, installable directly from their own repos with zero code changes here - `extract_release_archive`'s shape-autodetection (see below) doesn't care whether the repo is one of the three well-known ones. The remaining programs (`mf2005`, `mt3dms`, `mfusg`, and others with no independent release yet) haven't made that jump; as they do, they work automatically too. Restricting `repo` to a fixed list would have meant re-adding entries by hand as each program repo catches up, for a check that only prevents a typo from reaching the GitHub API - and the GitHub API already reports a typo clearly on its own once `get_release`'s 404 handling is solid (which it has to be regardless, for a real `repo` value with a bad `tag`). +**Why open rather than allowlisted:** a growing number of individual program repos already publish releases in the same single-program shape as `modflow6` (one archive per ostag, no `code.json`) - confirmed live (Nov 2026): `mfusg`, `mt3d-usgs`, `gridgen`, `triangle`, `zonbud`, `zonbudusg` all do, alongside `mf6`/`mf6-nightly`. That's already 7 of the ~20 programs `executables` bundles, installable directly from their own repos with zero code changes here - `extract_release_archive`'s shape-autodetection (see below) doesn't care whether the repo is one of the three well-known ones. The remaining programs (`mfnwt` and `vs2dt`, which have no published release yet - `vs2dt` has only a draft; `mf2005` and `mt3dms`, whose releases are source/legacy archives with no ostag-named assets; and others with no repo releases at all) haven't made that jump; as they do, they work automatically too. Restricting `repo` to a fixed list would have meant re-adding entries by hand as each program repo catches up, for a check that only prevents a typo from reaching the GitHub API - and the GitHub API already reports a typo clearly on its own once `get_release`'s 404 handling is solid (which it has to be regardless, for a real `repo` value with a bad `tag`). -`get_release`/`get_releases` hit the GitHub API directly (`GET /repos/{owner}/{repo}/releases[/tags/{tag}]`), with retry/backoff on transient failures, mirroring `get_modflow.py`'s own retry logic. `owner` defaults to `MODFLOW-ORG` but is overridable (e.g. to test a fork). A repo that doesn't exist (or has no releases) 404s at *both* the `/releases/tags/{tag}` and `/releases` endpoints, so `get_release`'s 404 handler - which calls `get_releases` to list available tags for a friendlier error message - catches `get_releases`'s own `ProgramInstallationError` in turn and reports "repo not found" instead of a confusing "tag not found, choose from: []" or an unwrapped `requests.exceptions.HTTPError` leaking out of the library. +`get_release`/`get_releases` hit the GitHub API directly (`GET /repos/{owner}/{repo}/releases[/tags/{tag}]`), with retry/backoff on transient failures, mirroring `get_modflow.py`'s own retry logic. `owner` defaults to `MODFLOW-ORG` but is overridable (e.g. to test a fork). A repo that doesn't exist 404s at *both* the `/releases/tags/{tag}` and `/releases` endpoints, so `get_release`'s 404 handler - which calls `get_releases` to list available tags for a friendlier error message - catches `get_releases`'s own `ProgramInstallationError` in turn and reports "repo not found" instead of an unwrapped `requests.exceptions.HTTPError` leaking out of the library. A repo that exists but has no published releases 404s only at the tag/`latest` endpoint; `get_releases` then returns an empty list and `get_release` reports "has no published releases". `get_releases` leaves out drafts (only visible to authenticated users with push access, and not fetchable by tag), and only offers `latest` when there's a full (non-prerelease) release for GitHub's `/releases/latest` to resolve to. ### Asset selection @@ -70,12 +70,12 @@ No registry, no bootstrap file, and - unlike the first iteration - no fixed allo `extract_release_archive` generalizes `get_modflow.py`'s extraction logic across both archive shapes it needs to handle: - Gathers files nested under a top-level `bin/` directory if present, falling back to the archive root. -- If a `code.json` manifest is present (the `executables` shape), each program's version comes from it, and `shared_object`/exe-suffix handling follows `code.json`. -- Otherwise (the `modflow6`/`modflow6-nightly-build` shape), every extracted file is versioned as the release tag, keyed by filename stem. +- Only executables and shared libraries are candidates: Windows `.exe`/`.dll`, unix shared libraries by suffix, and unix executables by their executable bit (when the archive recorded unix permissions; otherwise, suffix-less files). Known documentation names (`LICENSE`, `README`, ...) are always skipped - e.g. `mt3d-usgs` archives bundle a `LICENSE`, which must not land in `bindir` with a ledger entry. +- If a `code.json` manifest is present (the `executables` shape), each program's version comes from it, and `shared_object`/exe-suffix handling follows `code.json`. Entries with `double_switch` also make `dbl` installable, versioned like ``. Binaries the manifest doesn't describe (e.g. `mf5to6`) are versioned as the release tag. +- Otherwise (the `modflow6`/`modflow6-nightly-build` shape), every program is versioned as the release tag. +- Versions are normalized by stripping a leading `v`/`v.` (single-program repos tag `v1.5.1`; `code.json` says `1.5.1`), and the ledger compares versions semantically (PEP 440 where both parse, normalized strings otherwise), so entries recorded before normalization still match, and precision differences don't matter - pymake's `code.json` records mfusg's `v1.5.00` release as `1.5`. - Detection is based on the presence of `code.json` in the archive, not on which `repo` was requested - so a future distribution that adopts either shape needs no code change here. -- `subset` (a set of program names or filenames) filters what gets extracted from either shape. - -Not ported: `code.json`'s legacy `*dbl` double-precision variant handling (a few old MODFLOW-2005-era programs ship both single- and double-precision builds). Scoped out as a niche build variant, not central to installing modern ecosystem programs; a real gap if someone needs those specific programs' double-precision builds through this API. +- `subset` (a set of program names or filenames) filters what gets extracted from either shape; naming anything not in the archive is an error, listing the installable programs. ### Installation @@ -173,7 +173,7 @@ Programs are expected to publish pre-built binaries for all supported platforms; `pymake` today plays two roles for `executables`: building each program from source, and knowing the combined list of what to build. This API's job is neither of those - it installs from releases that already exist. But an open `repo` (see "Program sources" above) is what makes a path to retiring pymake possible, in two independent steps: -1. **Already true today, no further work needed:** as individual program repos adopt their own build/release CI (meson + GitHub Actions publishing per-platform zips - the pattern `mfnwt`, `mt3d-usgs`, `vs2dt`, `gridgen`, `triangle`, `zonbud`, and `zonbudusg` already follow), each becomes installable directly via `install_program(repo=)`, bypassing both `executables` and pymake for that program entirely. This is a per-program-repo migration, not a devtools change. +1. **Already true today, no further work needed:** as individual program repos adopt their own build/release CI (meson + GitHub Actions publishing per-platform zips - the pattern `mfusg`, `mt3d-usgs`, `gridgen`, `triangle`, `zonbud`, and `zonbudusg` already follow), each becomes installable directly via `install_program(repo=)`, bypassing both `executables` and pymake for that program entirely. This is a per-program-repo migration, not a devtools change. 2. **Not yet built, and deliberately not this module's job:** `executables` itself could stop invoking pymake to build everything from source, and instead have its release CI *compose* a combined bundle by fetching each participating program's latest release asset per platform (via this module's `get_release`/`download_archive`), extracting it, and re-packing everything into one archive plus a freshly generated `code.json`. This is the mirror image of `extract_release_archive`, and could reuse most of its primitives, but the composition policy (which programs to bundle, how to name/version the result) is `executables`-repo-specific business logic - it belongs in that repo's own CI script, not in this library, for the same reason the registry contract in the first iteration didn't belong here either. @@ -184,5 +184,4 @@ Once (1) covers enough programs, `executables` may not need to exist as a combin 1. **Building programs from source.** Programs must publish pre-built binaries. 2. **Cross-platform installs.** No installing Windows binaries on Linux, etc. 3. **A registry contract for program repositories.** See "First iteration" above. -4. **Double-precision (`*dbl`) build variants.** See "Extraction" above. -5. **Semantic version ranges / aliases** (e.g. `mf6@^6.6`, `mf6@latest` as a named alias distinct from the literal release tag `latest`). Could be added later if there's real demand. +4. **Semantic version ranges / aliases** (e.g. `mf6@^6.6`, `mf6@latest` as a named alias distinct from the literal release tag `latest`). Could be added later if there's real demand. diff --git a/docs/md/programs.md b/docs/md/programs.md index 330bf416..0b374a19 100644 --- a/docs/md/programs.md +++ b/docs/md/programs.md @@ -59,14 +59,14 @@ mf programs install --repo executables --subset mfnwt,mf2005 --bindir /usr/local `repo` is **not** restricted to a fixed list - any repo under `owner` (default `MODFLOW-ORG`) with a GitHub release and a platform-matching asset works, including installing a single program directly from its own repo: ```python -install_program(repo="mfnwt", bindir="/usr/local/bin") +install_program(repo="mfusg", bindir="/usr/local/bin") install_program(repo="gridgen", bindir="/usr/local/bin") ``` Or via CLI: ```bash -mf programs install --repo mfnwt --bindir /usr/local/bin +mf programs install --repo mfusg --bindir /usr/local/bin mf programs install --repo gridgen --bindir /usr/local/bin ``` @@ -78,7 +78,11 @@ mf programs install --repo gridgen --bindir /usr/local/bin | `modflow6` | `mf6`, `zbud6`, `mf5to6`, `libmf6` | All share the release tag as their version | | `modflow6-nightly-build` | Same programs as `modflow6`, nightly builds | All share the nightly build tag | -A growing number of individual program repos (`mfnwt`, `mt3d-usgs`, `vs2dt`, `gridgen`, `triangle`, `zonbud`, `zonbudusg`, ...) already publish releases in the same single-program shape as `modflow6` - one archive per platform, no `code.json` needed since there's only one program in it. Anything shaped like that just works by passing its repo name. +A growing number of individual program repos (`mfusg`, `mt3d-usgs`, `gridgen`, `triangle`, `zonbud`, `zonbudusg`, ...) already publish releases in the same single-program shape as `modflow6` - one archive per platform, no `code.json` needed since there's only one program in it. Anything shaped like that just works by passing its repo name. Others (e.g. `mfnwt`, `vs2dt`) don't publish installable releases yet; install those from `executables`. + +Only executables and shared libraries are installed from an archive; other files bundled with them (e.g. a `LICENSE`) are skipped. Double-precision variants (e.g. `mf2005dbl`) are separate programs: name them in `program`/`subset` to install them. + +Versions are recorded without a leading `v`, so a program installed from a release tagged `v1.5.1` is found by `get_executable(..., version="1.5.1")` (or `"v1.5.1"`). Versions are compared semantically, so `"1.5"` and `"1.5.00"` are the same version. `owner` defaults to `MODFLOW-ORG`; override it to test against a fork. diff --git a/modflow_devtools/programs/__init__.py b/modflow_devtools/programs/__init__.py index c5a94b3c..9645813f 100644 --- a/modflow_devtools/programs/__init__.py +++ b/modflow_devtools/programs/__init__.py @@ -27,6 +27,7 @@ import pooch import requests # type: ignore[import-untyped] +from packaging.version import InvalidVersion, Version from modflow_devtools.ostags import get_binary_suffixes, get_modflow_ostag @@ -49,8 +50,8 @@ """The three repos get_modflow.py supports out of the box - not an allowlist. Any owner/repo with a GitHub release and an ostag-matching asset works with `install_program`/`get_release`; a growing number of individual program repos -(mfnwt, mt3d-usgs, vs2dt, gridgen, triangle, zonbud, zonbudusg, ...) already -publish releases in the same shape `executables` and `modflow6` do.""" +(mfusg, mt3d-usgs, gridgen, triangle, zonbud, zonbudusg, ...) already publish +releases in the same shape `executables` and `modflow6` do.""" GITHUB_API = "https://api.github.com" _RETRYABLE_STATUS = {429, 500, 502, 503, 504} @@ -90,6 +91,29 @@ def _verify_hash(file_path: Path, expected_hash: str) -> bool: return actual_digest.lower() == expected_digest.lower() +def _normalize_version(version: str) -> str: + """ + Strip a leading 'v' (or 'v.') from a version string, so a release tag like + 'v1.5.1' (single-program repos) and a `code.json` version like '1.5.1' + (`executables`) are recorded, and compared, the same way. + """ + return re.sub(r"^[vV]\.?(?=\d)", "", version.strip()) + + +def _same_version(a: str, b: str) -> bool: + """ + Whether two version strings name the same version. Compared semantically + where both parse as PEP 440 versions, so precision differences don't + matter (pymake's `code.json` records mfusg's `v1.5.00` release as '1.5'); + otherwise (e.g. arbitrary tag names) compared as normalized strings. + """ + a, b = _normalize_version(a), _normalize_version(b) + try: + return Version(a) == Version(b) + except InvalidVersion: + return a == b + + def _github_headers() -> dict[str, str]: token = os.environ.get("GITHUB_TOKEN") return {"Authorization": f"Bearer {token}"} if token else {} @@ -124,14 +148,22 @@ def get_releases( tries: int | None = None, delay: float | None = None, ) -> list[str]: - """List available release tags for owner/repo, plus 'latest'.""" + """ + List installable release tags for owner/repo, preceded by 'latest' if the + repo has at least one full (non-prerelease) release for it to resolve to. + Draft releases are excluded: they're only visible to authenticated users + with push access, and can't be fetched by tag. Empty if the repo has no + published releases. + """ try: data = _request_json( - f"{GITHUB_API}/repos/{owner}/{repo}/releases", tries=tries, delay=delay + f"{GITHUB_API}/repos/{owner}/{repo}/releases?per_page=100", tries=tries, delay=delay ) except requests.exceptions.HTTPError as err: raise ProgramInstallationError(f"repo {owner}/{repo} not found: {err}") from err - return ["latest", *(r["tag_name"] for r in data)] + published = [r for r in data if not r.get("draft")] + has_latest = any(not r.get("prerelease") for r in published) + return (["latest"] if has_latest else []) + [r["tag_name"] for r in published] def get_release( @@ -156,6 +188,10 @@ def get_release( available = get_releases(owner, repo, tries=tries, delay=delay) except ProgramInstallationError: raise ProgramInstallationError(f"repo {owner}/{repo} not found") from err + if not available: + raise ProgramInstallationError( + f"repo {owner}/{repo} has no published releases" + ) from err raise ProgramInstallationError( f"release {tag!r} not found for {owner}/{repo}; choose from: {', '.join(available)}" ) from err @@ -262,6 +298,34 @@ def _normalize_subset(program: str | None, subset) -> set[str] | None: return items or None +_NON_PROGRAM_STEMS = frozenset( + {"license", "licence", "readme", "copying", "notice", "changelog", "changes", "authors"} +) + + +def _member_kind(info: zipfile.ZipInfo, exe_suffix: str, lib_suffix: str) -> bool | None: + """ + Classify an archive member: True for a shared library, False for an + executable, None for anything else (license, readme, docs, ...). + + Windows executables are recognized by their `.exe` suffix. Unix ones have + no suffix, so they're recognized by their executable bit when the archive + recorded unix permissions, falling back to "no suffix and not an obvious + documentation file" when it didn't. + """ + name = Path(info.filename).name + if Path(name).stem.lower() in _NON_PROGRAM_STEMS: + return None + if lib_suffix and name.endswith(lib_suffix): + return True + if exe_suffix: + return False if name.endswith(exe_suffix) else None + mode = info.external_attr >> 16 + if info.create_system == 3 and mode: + return False if mode & 0o111 else None + return None if Path(name).suffix else False + + def extract_release_archive( archive: Path, dest_dir: Path, @@ -278,79 +342,89 @@ def extract_release_archive( Looks for files nested under a top-level `bin/` directory in the archive, falling back to the archive root if none is found (mirrors the actual layouts of `executables`, `modflow6`, and `modflow6-nightly-build` - releases). If the archive also contains a `code.json` manifest (used by - the combined `executables` distribution to bundle many independently - versioned programs into one archive), each program's own version comes - from there; otherwise every extracted executable is versioned as - `release_tag`, since a `modflow6`/`modflow6-nightly-build` release ships - every program at the same version. - - Double-precision program variants (`code.json`'s legacy `*dbl` entries, - used by a few old MODFLOW-2005-era programs) are not handled - a niche - build variant, not central to installing modern ecosystem programs. + releases). Only executables and shared libraries are installed; other + files (e.g. a `LICENSE` bundled alongside the binaries) are skipped. + + If the archive contains a `code.json` manifest (used by the combined + `executables` distribution to bundle many independently versioned + programs into one archive), each program's own version comes from there. + Double-precision variants (`dbl`, for entries with `double_switch`) + are installable programs in their own right, versioned like their base + program. Any other binary the manifest doesn't describe (e.g. `mf5to6`) + is versioned as `release_tag`, as is every program in an archive without + a manifest, since a single-program or `modflow6` release ships everything + at the same version. Versions are normalized (see `_normalize_version`). + + `subset` items may be program names or file names. Raises + `ProgramInstallationError` if any is not in the archive. """ dest_dir.mkdir(parents=True, exist_ok=True) exe_suffix, lib_suffix = get_binary_suffixes(ostag) + tag_version = _normalize_version(release_tag) with zipfile.ZipFile(archive) as zf: - names = zf.namelist() - full_path = {Path(n).name: n for n in names if Path(n).parent.name == "bin"} - files = set(full_path) if full_path else set(names) + infos = [i for i in zf.infolist() if not i.is_dir()] + in_bin = [i for i in infos if Path(i.filename).parent.name == "bin"] + members = {Path(i.filename).name: i for i in (in_bin or infos)} code = None - if "code.json" in files: - code = json.loads(zf.read(full_path.get("code.json", "code.json"))) - files.discard("code.json") - - extracted: list[_ExtractedProgram] = [] - to_extract: set[str] = set() - - if code: - for key in sorted(code): - meta = code[key] - is_so = bool(meta.get("shared_object")) - fname = f"{key}{lib_suffix if is_so else exe_suffix}" - if fname not in files: + if "code.json" in members: + code = json.loads(zf.read(members.pop("code.json"))) + + # program name -> (member, version, is_shared_object) + available: dict[str, tuple[zipfile.ZipInfo, str, bool]] = {} + manifest: dict[str, tuple[str, bool]] = {} + for key, meta in (code or {}).items(): + version = _normalize_version(str(meta["version"])) + is_so = bool(meta.get("shared_object")) + manifest[f"{key}{lib_suffix if is_so else exe_suffix}"] = (version, is_so) + if meta.get("double_switch") and not is_so: + manifest[f"{key}dbl{exe_suffix}"] = (version, False) + for fname, info in members.items(): + if fname in manifest: + version, is_so = manifest[fname] + else: + kind = _member_kind(info, exe_suffix, lib_suffix) + if kind is None: continue - if subset and key not in subset and fname not in subset: - continue - to_extract.add(full_path.get(fname, fname)) - extracted.append(_ExtractedProgram(key, meta["version"], dest_dir / fname, is_so)) + version, is_so = tag_version, kind + name = fname.removesuffix(lib_suffix if is_so else exe_suffix) + available[name] = (info, version, is_so) + + if not available: + raise ProgramInstallationError(f"no executables found in archive {archive.name}") + + if subset: + by_file = {Path(info.filename).name: name for name, (info, _, _) in available.items()} + missing = sorted(s for s in subset if s not in available and s not in by_file) + if missing: + raise ProgramInstallationError( + f"no matching executables found for {missing}; " + f"available: {', '.join(sorted(available))}" + ) + selected = {by_file.get(s, s) for s in subset} else: - for fname in sorted(files): - stem = Path(fname).stem - if subset and fname not in subset and stem not in subset: - continue - to_extract.add(full_path.get(fname, fname)) - is_so = bool(lib_suffix) and fname.endswith(lib_suffix) - extracted.append(_ExtractedProgram(stem, release_tag, dest_dir / fname, is_so)) - - if not to_extract: - raise ProgramInstallationError( - "no matching executables found" - + (f" for subset {sorted(subset)}" if subset else "") - + f"; available: {', '.join(sorted(files))}" - ) + selected = set(available) if verbose: - print(f"extracting {len(to_extract)} file(s) to {dest_dir}") - - zf.extractall(dest_dir, members=to_extract) - - if full_path: - rmdirs: set[Path] = set() - for member in to_extract: - member_path = Path(member) - (dest_dir / member_path).replace(dest_dir / member_path.name) - rmdirs.add(member_path.parent) - for rmdir in sorted(rmdirs, key=lambda p: len(p.parts), reverse=True): - for ancestor in (rmdir, *rmdir.parents): - if ancestor == Path(): - break - try: - (dest_dir / ancestor).rmdir() - except OSError: - break + print(f"extracting {len(selected)} file(s) to {dest_dir}") + + extracted: list[_ExtractedProgram] = [] + for name in sorted(selected): + info, version, is_so = available[name] + target = dest_dir / Path(info.filename).name + # write beside the target and swap it in, so an existing (possibly + # running, or code-signed and cached by the OS) binary is replaced + # rather than modified in place + temp = target.with_name(target.name + ".part") + try: + with zf.open(info) as src, temp.open("wb") as dst: + shutil.copyfileobj(src, dst) + temp.replace(target) + except BaseException: + temp.unlink(missing_ok=True) + raise + extracted.append(_ExtractedProgram(name, version, target, is_so)) if os.name != "nt": for prog in extracted: @@ -630,7 +704,10 @@ def add_installation(self, installation: ProgramInstallation) -> None: self.installations = [ inst for inst in self.installations - if not (inst.version == installation.version and inst.bindir == installation.bindir) + if not ( + _same_version(inst.version, installation.version) + and inst.bindir == installation.bindir + ) ] self.installations.append(installation) self.save() @@ -641,7 +718,7 @@ def remove_installation(self, version: str, bindir: Path) -> None: self.installations = [ inst for inst in self.installations - if not (inst.version == version and inst.bindir == bindir) + if not (_same_version(inst.version, version) and inst.bindir == bindir) ] self.save() @@ -677,11 +754,13 @@ def install_program( Any repo under `owner` with a GitHub release and an ostag-matching asset - not restricted to `KNOWN_REPOS` (default: "executables", the combined legacy distribution). Individual program repos work directly, - e.g. repo="mfnwt" or repo="gridgen". + e.g. repo="mfusg" or repo="gridgen". owner : str GitHub repository owner; override to test against a fork. version : str - Release tag, or "latest". + Release tag, or "latest". Installed programs are recorded under a + normalized version (no leading 'v'), e.g. tag "v1.5.1" -> "1.5.1". + Ledger lookups compare versions semantically, so "1.5" matches "1.5.00". bindir : str | Path, optional Installation directory. Auto-selected (see `get_bindir_options`) if omitted. @@ -770,7 +849,7 @@ def register_installation( hook other installers can call - `install_program` uses it internally too. """ installation = ProgramInstallation( - version=version, + version=_normalize_version(version), platform=platform or get_platform(), bindir=Path(bindir), installed_at=installed_at or datetime.now(UTC), @@ -804,7 +883,8 @@ def uninstall_program( to_remove = [ inst for inst in metadata.list_installations() - if inst.version == version and (bindir is None or inst.bindir == Path(bindir)) + if _same_version(inst.version, version) + and (bindir is None or inst.bindir == Path(bindir)) ] if not to_remove: @@ -838,7 +918,7 @@ def get_executable( installations = metadata.list_installations() if version is not None: - installations = [i for i in installations if i.version == version] + installations = [i for i in installations if _same_version(i.version, version)] if bindir is not None: bindir = Path(bindir) installations = [i for i in installations if i.bindir == bindir] diff --git a/modflow_devtools/programs/__main__.py b/modflow_devtools/programs/__main__.py index aa23c743..93ceceb5 100644 --- a/modflow_devtools/programs/__main__.py +++ b/modflow_devtools/programs/__main__.py @@ -125,7 +125,7 @@ def main(): default=DEFAULT_REPO, help=f"Source repo under --owner; default is '{DEFAULT_REPO}'. Not restricted to " f"{KNOWN_REPOS} - any repo with a GitHub release and a matching platform asset " - "works, e.g. 'mfnwt', 'gridgen'.", + "works, e.g. 'mfusg', 'gridgen'.", ) install_parser.add_argument( "--owner", diff --git a/pyproject.toml b/pyproject.toml index f632574a..e23de0f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,11 @@ dynamic = ["version"] ecosystem = [ "boltons", "filelock", + "packaging", "pooch", "pyaml", "pydantic", + "requests", "tomli", "tomli-w" ]