Repository navigation
CI: prepare Windows wheel dependencies once and back off on upstream rate limits #91
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -198,154 +198,32 @@ jobs: | |||||
| AL_BACKEND_UDA: AL_BACKEND_UDA=ON | ||||||
| UDA_REF: "2.9.3" | ||||||
|
|
||||||
| # Windows wheels are built by the dedicated build_wheels_windows job | ||||||
| # below: they share a vcpkg dependency tree prepared once by the | ||||||
| # windows_deps job (issue #44), and keeping them in this matrix would | ||||||
| # force `needs: windows_deps` onto the Linux/macOS legs too. | ||||||
|
|
||||||
| steps: | ||||||
| - uses: actions/checkout@v7 | ||||||
| with: | ||||||
| fetch-depth: 0 # unshallow for setuptools-scm.get_version to discover tags | ||||||
|
|
||||||
| - uses: actions/setup-python@v7 | ||||||
| with: | ||||||
| # python installation used when to start cibuildwheel | ||||||
| python-version: "3.12" | ||||||
| cache: 'pip' | ||||||
|
|
||||||
| - name: Restore cibuildwheel cache | ||||||
| id: cache | ||||||
| uses: actions/cache@v6 | ||||||
| with: | ||||||
| path: | | ||||||
| ~/Library/Caches/cibuildwheel | ||||||
| ~/.cache/cibuildwheel | ||||||
| # cf. CIBW_CACHE_PATH | ||||||
| key: "cibuildwheel-${{ runner.os }}-${{ runner.arch }}-cp${{ matrix.python }}-${{ matrix.platform_id }}" | ||||||
|
|
||||||
| - uses: pypa/[email protected] | ||||||
| if: startsWith(matrix.os, 'ubuntu-') | ||||||
| env: | ||||||
|
|
@@ -454,10 +332,152 @@ jobs: | |||||
| path: wheelhouse/*.whl | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| # Issue #44: prepare the pinned x64-windows vcpkg dependency closure ONCE, | ||||||
| # then let every Windows wheel job restore it read-only. Previously all five | ||||||
| # cold matrix jobs independently ran vcpkg against the same upstream source | ||||||
| # hosts and the same mutable cache key: a rate-limited host (HTTP 429 from | ||||||
| # gitlab.dkrz.de for libaec) failed the whole blocking workflow, and the five | ||||||
| # always-save steps raced on one key. Now a cold workflow downloads each | ||||||
| # source archive at most once (both C:\vcpkg\installed and the | ||||||
| # content-addressed C:\vcpkg\downloads are cached), and only this job ever | ||||||
| # saves the cache. | ||||||
| windows_deps: | ||||||
| name: Prepare Windows vcpkg dependencies | ||||||
| runs-on: windows-2022 | ||||||
| env: | ||||||
| # The before-build script's default (matrix.UDA_REF was never exported to | ||||||
| # the Windows build environment, so Windows has always built this | ||||||
| # version); pinned explicitly here because it is part of the cache key. | ||||||
| UDA_REF: "2.8.1" | ||||||
| VCPKG_ROOT: C:/vcpkg | ||||||
| VCPKG_TARGET_TRIPLET: x64-windows | ||||||
| outputs: | ||||||
| cache-key: ${{ steps.key.outputs.key }} | ||||||
| defaults: | ||||||
| run: | ||||||
| shell: bash | ||||||
| steps: | ||||||
| - uses: actions/checkout@v7 | ||||||
|
|
||||||
| - name: Compute content-addressed cache key | ||||||
| id: key | ||||||
| # Key covers the manifest (vcpkg.json), the dependency-install script, | ||||||
| # the triplet, the UDA ref, and the runner's vcpkg baseline commit — | ||||||
| # any of these changing invalidates the prepared tree. restore-keys | ||||||
| # allow warm partial reuse; the exact key is what wheel jobs consume. | ||||||
| run: | | ||||||
| vcpkg_commit=$(git -C /c/vcpkg rev-parse HEAD 2>/dev/null || echo unknown) | ||||||
| echo "key=win-deps-${VCPKG_TARGET_TRIPLET}-uda${UDA_REF}-${{ hashFiles('vcpkg.json', 'ci/wheels/cibw_before_build_win.sh') }}-vcpkg-${vcpkg_commit}" >> "$GITHUB_OUTPUT" | ||||||
|
|
||||||
| - name: Cache vcpkg installed tree and downloads | ||||||
| id: cache | ||||||
| uses: actions/cache@v6 | ||||||
| with: | ||||||
| path: | | ||||||
| C:\vcpkg\installed | ||||||
| C:\vcpkg\downloads | ||||||
| key: ${{ steps.key.outputs.key }} | ||||||
| restore-keys: | | ||||||
| win-deps-${{ env.VCPKG_TARGET_TRIPLET }}-uda${{ env.UDA_REF }}- | ||||||
|
|
||||||
| - name: Install the pinned dependency closure (cold cache only) | ||||||
| if: steps.cache.outputs.cache-hit != 'true' | ||||||
| # The same script every wheel job runs as CIBW before-build; running it | ||||||
| # here keeps a single source of truth for the closure. It is idempotent | ||||||
| # against a partially restored tree: vcpkg skips installed packages and | ||||||
| # the UDA build is skipped when uda-cpp.pc is already installed — which | ||||||
| # also makes the retry loop below resume where the last attempt left | ||||||
| # off. The backoff exists because vcpkg's own retries are seconds | ||||||
| # apart: a rate-limiting upstream (HTTP 429 from gitlab.dkrz.de for | ||||||
| # libaec, issue #44) needs minutes for its window to reset. | ||||||
| run: | | ||||||
| for attempt in 1 2 3 4 5; do | ||||||
| if bash ./ci/wheels/cibw_before_build_win.sh; then | ||||||
| exit 0 | ||||||
| fi | ||||||
| echo "Dependency install attempt ${attempt} failed (upstream rate limit?); backing off $((attempt * 90))s" | ||||||
| sleep $((attempt * 90)) | ||||||
| done | ||||||
| echo "FATAL: dependency closure install failed after 5 attempts" | ||||||
| exit 1 | ||||||
|
|
||||||
| build_wheels_windows: | ||||||
| name: build_wheels (Python ${{ matrix.python }}, win_amd64) | ||||||
| runs-on: windows-2022 | ||||||
| needs: [windows_deps] | ||||||
| strategy: | ||||||
| # Ensure that a wheel builder finishes even if another fails | ||||||
| fail-fast: false | ||||||
| matrix: | ||||||
| python: [310, 311, 312, 313, 314] | ||||||
|
|
||||||
| # Recent changes in GitHub Actions Cache's API have rendered the x-gha | ||||||
| # binary caching provider useless. | ||||||
| # See https://github.com/microsoft/vcpkg-tool/pull/1662 — vcpkg reuse is | ||||||
| # instead provided by the windows_deps job's prepared-tree cache above. | ||||||
| steps: | ||||||
| - uses: actions/checkout@v7 | ||||||
| with: | ||||||
| fetch-depth: 0 # unshallow for setuptools-scm.get_version to discover tags | ||||||
|
|
||||||
| - uses: actions/setup-python@v7 | ||||||
| with: | ||||||
| # python installation used when to start cibuildwheel | ||||||
| python-version: "3.12" | ||||||
| cache: 'pip' | ||||||
|
|
||||||
| - name: Install cibuildwheel for windows | ||||||
| run: | | ||||||
| python -m pip install cibuildwheel==3.1.0 | ||||||
|
|
||||||
| - name: Restore cibuildwheel cache | ||||||
| id: cache | ||||||
| uses: actions/cache@v6 | ||||||
| with: | ||||||
| path: | | ||||||
| ~\AppData\Local\pypa\cibuildwheel\Cache | ||||||
| # cf. CIBW_CACHE_PATH | ||||||
| key: "cibuildwheel-${{ runner.os }}-${{ runner.arch }}-cp${{ matrix.python }}-win_amd64" | ||||||
|
|
||||||
| - name: Restore prepared windows deps (read-only) | ||||||
| uses: actions/cache/restore@v6 | ||||||
| with: | ||||||
| path: | | ||||||
| C:\vcpkg\installed | ||||||
| C:\vcpkg\downloads | ||||||
| key: ${{ needs.windows_deps.outputs.cache-key }} | ||||||
| # The key was saved (or verified) by windows_deps in this same run; a | ||||||
| # miss means the prepared tree is unavailable and this job would fall | ||||||
| # back to a cold upstream fetch — the stampede issue #44 removes. | ||||||
| fail-on-cache-miss: true | ||||||
|
|
||||||
| - name: Build wheels for windows | ||||||
| run: | | ||||||
| python -c "import sys, pprint, cibuildwheel; pprint.pp(sys.path); pprint.pp(cibuildwheel)" | ||||||
| python -m cibuildwheel --output-dir wheelhouse --config-file pyproject.toml | ||||||
| env: | ||||||
| UDA_REF: "2.8.1" # keep in sync with windows_deps.env.UDA_REF | ||||||
|
Contributor
There was a problem hiding this comment. Should this also be bumped to
Suggested change
|
||||||
| CIBW_BUILD: cp${{ matrix.python }}-win_amd64 | ||||||
| CIBW_PLATFORM: windows | ||||||
| CIBW_ARCHS: AMD64 | ||||||
| CIBW_BUILD_VERBOSITY: 1 | ||||||
| CIBW_SKIP: "cp31?t-*" # Skip free-threading builds (not supported yet) | ||||||
| CIBW_CONFIG_SETTINGS: > | ||||||
| cmake.define.AL_BACKEND_HDF5=ON | ||||||
| cmake.define.AL_BACKEND_MDSPLUS=OFF | ||||||
| cmake.define.AL_BACKEND_UDA=OFF | ||||||
|
Contributor
There was a problem hiding this comment. I think we can enable the UDA backend with Windows as done on Conda forge:
Suggested change
Contributor
There was a problem hiding this comment. There may be path problems but we can fix them in follow-up commits once the CI build has been attempted. |
||||||
| # cmake.args="--debug-find;--debug-output" | ||||||
|
|
||||||
| - uses: actions/upload-artifact@v7 | ||||||
| with: | ||||||
| name: cibw-wheels-cp${{ matrix.python }}-win_amd64 | ||||||
| path: wheelhouse/*.whl | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| publish_wheels_testpypi: | ||||||
| # only publish to TestPyPI on develop pushes and not from forks | ||||||
| if: github.ref=='refs/heads/develop' && github.repository_owner == 'iterorganization' | ||||||
| needs: [build_wheels, build_wheels_windows] | ||||||
| runs-on: ubuntu-24.04 | ||||||
| permissions: | ||||||
| # IMPORTANT: this permission is mandatory for Trusted Publishing | ||||||
|
|
@@ -484,7 +504,7 @@ jobs: | |||||
| publish_wheels_pypi: | ||||||
| # only publish to PyPI on tag pushes and not from forks | ||||||
| if: startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'iterorganization' | ||||||
| needs: [build_wheels, build_wheels_windows] | ||||||
| runs-on: ubuntu-24.04 | ||||||
| permissions: | ||||||
| # IMPORTANT: this permission is mandatory for Trusted Publishing | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Any particular reason for downgrading?