From ed7c010c0d95aeb7561119ef940a951bbf737690 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 10 Nov 2026 03:52:52 -0800 Subject: [PATCH 1/3] ci(release): regenerate code.md after patching code.json make-code-json writes both code.json and code.md from pymake's internal program catalog. patch_code_json.py corrected code.json to the versions actually fetched via releases.json, but code.md was left stale, and it is what feeds the release body and the README table. Regenerate code.md from the patched code.json in the same format pymake uses. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 2 +- scripts/patch_code_json.py | 41 ++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86515b8..499357b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,7 +114,7 @@ jobs: pixi run --manifest-path modflow6/pixi.toml make-program mf2005,mfusg,mfnwt,mflgr --appdir $ostag --double --keep --zip $ostag.zip --verbose pixi run --manifest-path modflow6/pixi.toml make-code-json --appdir $ostag --zip $ostag.zip --verbose - - name: Patch code.json with fetched program versions + - name: Patch code.json and code.md with fetched program versions run: | ostag="${{ steps.ostag.outputs.ostag }}" pixi run --manifest-path modflow6/pixi.toml python scripts/patch_code_json.py \ diff --git a/scripts/patch_code_json.py b/scripts/patch_code_json.py index b5a21c3..6bbb0fc 100644 --- a/scripts/patch_code_json.py +++ b/scripts/patch_code_json.py @@ -3,7 +3,8 @@ make-code-json reads versions from pymake's internal program catalog, which may be out of date relative to the versions actually fetched. This script overwrites the version (and url) for each program listed in releases.json with the values -from the releases.json manifest. +from the releases.json manifest, then regenerates code.md (also written by +make-code-json from the same stale data) from the patched code.json. Usage: python patch_code_json.py --manifest releases.json --ostag linux --zip linux.zip @@ -36,7 +37,23 @@ def _get_url_date(url): return None -def patch(manifest_path, code_json_path, ostag, zip_path=None): +def write_code_md(code, code_md_path): + """Write the code.md table in the same format as pymake's make-code-json.""" + lines = [ + "| Program | Version | UTC Date |", + "| ------- | ------- | ---- |", + ] + for target in sorted(code): + target_dict = code[target] + date = target_dict.get("url_download_asset_date") + line = f"| {target} | {target_dict['version']} |" + line += f" {date} |" if date else " |" + lines.append(line) + Path(code_md_path).write_text("\n".join(lines) + "\n") + print(f"wrote {code_md_path}") + + +def patch(manifest_path, code_json_path, ostag, zip_path=None, code_md_path=None): with open(manifest_path) as f: manifest = json.load(f) @@ -72,6 +89,8 @@ def patch(manifest_path, code_json_path, ostag, zip_path=None): if not changed: print("code.json already up to date") + if code_md_path: + write_code_md(code, code_md_path) return for line in changed: @@ -80,6 +99,9 @@ def patch(manifest_path, code_json_path, ostag, zip_path=None): code_json_path.write_text(json.dumps(code, indent=4) + "\n") print(f"wrote {code_json_path}") + if code_md_path: + write_code_md(code, code_md_path) + if zip_path and Path(zip_path).exists(): with zipfile.ZipFile(zip_path, "a", zipfile.ZIP_DEFLATED) as zf: zf.write(code_json_path, "code.json") @@ -109,8 +131,21 @@ def main(): dest="zip_path", help="zip file to update with the patched code.json", ) + parser.add_argument( + "--code-md", + dest="code_md_path", + default="code.md", + help="path to code.md to regenerate from the patched code.json " + "(default: code.md; pass an empty string to skip)", + ) args = parser.parse_args() - patch(args.manifest, args.code_json, args.ostag, args.zip_path) + patch( + args.manifest, + args.code_json, + args.ostag, + args.zip_path, + args.code_md_path or None, + ) if __name__ == "__main__": From 5b840b8a37bda450ed4b3ecc0163d197656a8fbc Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 10 Nov 2026 03:56:10 -0800 Subject: [PATCH 2/3] ci(release): replace code.json in zip and appdir when patching patch_code_json.py appended the patched code.json to a zip that already contained make-code-json's copy, producing a duplicate entry whose resolution depends on the extracting tool, and left the stale copy in the app directory untouched. Rebuild the archive without the old entry, and overwrite the appdir copy via a new --appdir option. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 1 + scripts/patch_code_json.py | 59 +++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 499357b..fc5ed8a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,6 +120,7 @@ jobs: pixi run --manifest-path modflow6/pixi.toml python scripts/patch_code_json.py \ --manifest releases.json \ --ostag $ostag \ + --appdir $ostag \ --zip $ostag.zip - name: Show programs diff --git a/scripts/patch_code_json.py b/scripts/patch_code_json.py index 6bbb0fc..7cdab78 100644 --- a/scripts/patch_code_json.py +++ b/scripts/patch_code_json.py @@ -4,17 +4,22 @@ be out of date relative to the versions actually fetched. This script overwrites the version (and url) for each program listed in releases.json with the values from the releases.json manifest, then regenerates code.md (also written by -make-code-json from the same stale data) from the patched code.json. +make-code-json from the same stale data) from the patched code.json. If +--appdir is given, the copy of code.json there is also overwritten, and if +--zip is given, the code.json entry in the archive is replaced (not appended). Usage: - python patch_code_json.py --manifest releases.json --ostag linux --zip linux.zip + python patch_code_json.py --manifest releases.json --ostag linux --appdir linux --zip linux.zip python patch_code_json.py --manifest releases.json --ostag linux code.json """ import argparse import datetime import json +import os +import shutil import sys +import tempfile import urllib.request import zipfile from pathlib import Path @@ -53,7 +58,39 @@ def write_code_md(code, code_md_path): print(f"wrote {code_md_path}") -def patch(manifest_path, code_json_path, ostag, zip_path=None, code_md_path=None): +def replace_in_zip(zip_path, src_path, arcname): + """Replace (or add) a single entry in a zip archive. + + zipfile cannot remove entries, and appending a file with an existing name + creates a duplicate entry, so rebuild the archive without the old entry. + """ + zip_path = Path(zip_path) + fd, tmp = tempfile.mkstemp(suffix=".zip", dir=zip_path.parent) + os.close(fd) + try: + with zipfile.ZipFile(zip_path) as zin, zipfile.ZipFile( + tmp, "w", zipfile.ZIP_DEFLATED + ) as zout: + for item in zin.infolist(): + if item.filename == arcname: + continue + with zin.open(item) as src, zout.open(item, "w") as dst: + shutil.copyfileobj(src, dst) + zout.write(src_path, arcname) + os.replace(tmp, zip_path) + finally: + if os.path.exists(tmp): + os.remove(tmp) + + +def patch( + manifest_path, + code_json_path, + ostag, + zip_path=None, + code_md_path=None, + appdir=None, +): with open(manifest_path) as f: manifest = json.load(f) @@ -99,13 +136,18 @@ def patch(manifest_path, code_json_path, ostag, zip_path=None, code_md_path=None code_json_path.write_text(json.dumps(code, indent=4) + "\n") print(f"wrote {code_json_path}") + if appdir: + appdir_json = Path(appdir) / "code.json" + if appdir_json.resolve() != code_json_path.resolve(): + shutil.copyfile(code_json_path, appdir_json) + print(f"wrote {appdir_json}") + if code_md_path: write_code_md(code, code_md_path) if zip_path and Path(zip_path).exists(): - with zipfile.ZipFile(zip_path, "a", zipfile.ZIP_DEFLATED) as zf: - zf.write(code_json_path, "code.json") - print(f"updated code.json in {zip_path}") + replace_in_zip(zip_path, code_json_path, "code.json") + print(f"replaced code.json in {zip_path}") def main(): @@ -131,6 +173,10 @@ def main(): dest="zip_path", help="zip file to update with the patched code.json", ) + parser.add_argument( + "--appdir", + help="program directory whose copy of code.json should also be patched", + ) parser.add_argument( "--code-md", dest="code_md_path", @@ -145,6 +191,7 @@ def main(): args.ostag, args.zip_path, args.code_md_path or None, + args.appdir, ) From 8af8944c5578cb8071063ee153da67e153e7cca3 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 10 Nov 2026 04:01:59 -0800 Subject: [PATCH 3/3] ci(release): make patch_code_json.py the only writer of code.json to the zip Rather than letting make-code-json zip a stale code.json and then rebuilding the archive to replace it, run make-code-json without --zip and have patch_code_json.py add the corrected code.json once (every run, including when nothing changed). It fails if the zip already holds a code.json instead of writing a duplicate entry. Drop --appdir: the program directory is not distributed, so its stale copy does not matter. Document the metadata steps in DEVELOPER.md. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 5 +-- DEVELOPER.md | 9 +++- scripts/patch_code_json.py | 83 +++++++++++------------------------ 3 files changed, 34 insertions(+), 63 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc5ed8a..c0c24f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,15 +112,14 @@ jobs: fetched=$(pixi run --manifest-path modflow6/pixi.toml python scripts/fetch_releases.py --manifest releases.json --list) pixi run --manifest-path modflow6/pixi.toml make-program : --appdir $ostag --exclude "$fetched" --keep --zip $ostag.zip --verbose pixi run --manifest-path modflow6/pixi.toml make-program mf2005,mfusg,mfnwt,mflgr --appdir $ostag --double --keep --zip $ostag.zip --verbose - pixi run --manifest-path modflow6/pixi.toml make-code-json --appdir $ostag --zip $ostag.zip --verbose + pixi run --manifest-path modflow6/pixi.toml make-code-json --appdir $ostag --verbose - - name: Patch code.json and code.md with fetched program versions + - name: Patch code.json and code.md with fetched program versions, add code.json to zip run: | ostag="${{ steps.ostag.outputs.ostag }}" pixi run --manifest-path modflow6/pixi.toml python scripts/patch_code_json.py \ --manifest releases.json \ --ostag $ostag \ - --appdir $ostag \ --zip $ostag.zip - name: Show programs diff --git a/DEVELOPER.md b/DEVELOPER.md index ddec837..e7be825 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -91,8 +91,13 @@ The file `releases.json` in the repository root is a manifest listing programs t During a release build, the workflow: 1. Runs `scripts/fetch_releases.py` to download pre-built binaries for the current platform from each repository listed in `releases.json`. -2. Runs pymake to compile the remaining programs, excluding those already fetched. -3. Combines everything into the platform zip alongside pymake-generated metadata. +2. Runs pymake to compile the remaining programs, excluding those already fetched, and adds them to the platform zip. +3. Runs pymake's `make-code-json` to generate `code.json` and `code.md` metadata. These come from pymake's internal program catalog, which may list different versions than were actually fetched. +4. Runs `scripts/patch_code_json.py` to correct the version, URL, and date in `code.json` for each program in `releases.json`, regenerate `code.md` from the corrected `code.json`, and add `code.json` to the platform zip. + +`make-code-json` is deliberately run without `--zip`, so the patch script is the only thing that writes `code.json` into the zip. If the zip already contains a `code.json`, the patch script fails rather than adding a second, conflicting entry. The `code.json` that `make-code-json` also writes into the program directory is not corrected, but that directory is not distributed, only the zip is. + +The top-level `code.json` and `code.md` are uploaded with the build artifacts. The release job uses `code.md` for the release notes and for the program table in `README.md`. The fetch script handles platform-specific file extensions (`.exe`, `.dll`, `.dylib`, `.so`) and supports renaming programs when the archive filename differs from the distribution name (e.g., `mfusgt` in the archive becomes `mfusg_gsi` in the distribution). diff --git a/scripts/patch_code_json.py b/scripts/patch_code_json.py index 7cdab78..581ce31 100644 --- a/scripts/patch_code_json.py +++ b/scripts/patch_code_json.py @@ -4,22 +4,22 @@ be out of date relative to the versions actually fetched. This script overwrites the version (and url) for each program listed in releases.json with the values from the releases.json manifest, then regenerates code.md (also written by -make-code-json from the same stale data) from the patched code.json. If ---appdir is given, the copy of code.json there is also overwritten, and if ---zip is given, the code.json entry in the archive is replaced (not appended). +make-code-json from the same stale data) from the patched code.json. + +If --zip is given, the patched code.json is added to the archive. This script +is meant to be the only thing that puts code.json in the zip, so make-code-json +should be run *without* --zip; if the archive already contains a code.json the +script fails rather than writing a duplicate entry. Usage: - python patch_code_json.py --manifest releases.json --ostag linux --appdir linux --zip linux.zip + python patch_code_json.py --manifest releases.json --ostag linux --zip linux.zip python patch_code_json.py --manifest releases.json --ostag linux code.json """ import argparse import datetime import json -import os -import shutil import sys -import tempfile import urllib.request import zipfile from pathlib import Path @@ -58,38 +58,12 @@ def write_code_md(code, code_md_path): print(f"wrote {code_md_path}") -def replace_in_zip(zip_path, src_path, arcname): - """Replace (or add) a single entry in a zip archive. - - zipfile cannot remove entries, and appending a file with an existing name - creates a duplicate entry, so rebuild the archive without the old entry. - """ - zip_path = Path(zip_path) - fd, tmp = tempfile.mkstemp(suffix=".zip", dir=zip_path.parent) - os.close(fd) - try: - with zipfile.ZipFile(zip_path) as zin, zipfile.ZipFile( - tmp, "w", zipfile.ZIP_DEFLATED - ) as zout: - for item in zin.infolist(): - if item.filename == arcname: - continue - with zin.open(item) as src, zout.open(item, "w") as dst: - shutil.copyfileobj(src, dst) - zout.write(src_path, arcname) - os.replace(tmp, zip_path) - finally: - if os.path.exists(tmp): - os.remove(tmp) - - def patch( manifest_path, code_json_path, ostag, zip_path=None, code_md_path=None, - appdir=None, ): with open(manifest_path) as f: manifest = json.load(f) @@ -124,30 +98,28 @@ def patch( code[prog_name]["url_download_asset_date"] = date changed.append(f" {prog_name}: {prev_version!r} -> {version!r}") - if not changed: + if changed: + for line in changed: + print(line) + code_json_path.write_text(json.dumps(code, indent=4) + "\n") + print(f"wrote {code_json_path}") + else: print("code.json already up to date") - if code_md_path: - write_code_md(code, code_md_path) - return - - for line in changed: - print(line) - - code_json_path.write_text(json.dumps(code, indent=4) + "\n") - print(f"wrote {code_json_path}") - - if appdir: - appdir_json = Path(appdir) / "code.json" - if appdir_json.resolve() != code_json_path.resolve(): - shutil.copyfile(code_json_path, appdir_json) - print(f"wrote {appdir_json}") if code_md_path: write_code_md(code, code_md_path) - if zip_path and Path(zip_path).exists(): - replace_in_zip(zip_path, code_json_path, "code.json") - print(f"replaced code.json in {zip_path}") + if zip_path: + with zipfile.ZipFile(zip_path, "a", zipfile.ZIP_DEFLATED) as zf: + if "code.json" in zf.namelist(): + print( + f"{zip_path} already contains code.json; run make-code-json " + "without --zip so this script is the only writer", + file=sys.stderr, + ) + sys.exit(1) + zf.write(code_json_path, "code.json") + print(f"added code.json to {zip_path}") def main(): @@ -171,11 +143,7 @@ def main(): parser.add_argument( "--zip", dest="zip_path", - help="zip file to update with the patched code.json", - ) - parser.add_argument( - "--appdir", - help="program directory whose copy of code.json should also be patched", + help="zip file to add the patched code.json to", ) parser.add_argument( "--code-md", @@ -191,7 +159,6 @@ def main(): args.ostag, args.zip_path, args.code_md_path or None, - args.appdir, )