From cdff20974dae214c9cd3ac331fd1d1a3e3e34571 Mon Sep 17 00:00:00 2001 From: Xenolphthalein <21215869+Xenolphthalein@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:27:55 +0100 Subject: [PATCH] chore: a lint job (ruff, hadolint, actionlint), one gate job, ci, and release notes with the commits and the tool versions that changed - image/Dockerfile: bash with pipefail and arrays, as hadolint asks (DL3008 ignored: apt versions follow the dated snapshot, not pins) - README: the consumers of a pin, which tests need what, the source packages lists start with the release after 2026.11.12, ci-* tags are CI's cache, no channels --- .github/workflows/image.yml | 84 ++++++++++++++++++++++++++++++++++++- README.md | 29 +++++++++---- image/Dockerfile | 16 +++---- pyproject.toml | 38 +++++++++++++++++ tests/__init__.py | 2 + tests/test_image.py | 1 - 6 files changed, 151 insertions(+), 19 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index d1c198f..530084b 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -4,7 +4,8 @@ # ghcr.io/overpressure/builder:ci--, and a later run (main after the merge, the tag) # pulls it instead of building it again. A tag (YYYY.MM.DD, or YYYY.MM.DD.N for a second one that day) # publishes the tested variants as one multi-arch image, ghcr.io/overpressure/builder:TAG, and a release -# naming its digest, with each variant's packages and versions. +# naming its digest, with each variant's packages and versions and what changed since the previous release. +# The last job, ci, passes only when the lint and both variants passed: the one check main's ruleset requires. name: Image on: @@ -25,6 +26,27 @@ env: IMAGE: ghcr.io/overpressure/builder jobs: + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 + with: + version: "0.9.26" + - name: The Python code, overpressure's ruff (pyproject.toml) + run: | + uvx ruff@0.17.4 format --check . + uvx ruff@0.17.4 check . + # DL3008 (pin apt packages' versions) is off on purpose: the packages are Ubuntu's as they are when the + # image is built, and the published image, pinned by digest, is what pins a build's tools (Dockerfile). + - name: The Dockerfile + run: docker run --rm -i hadolint/hadolint:v2.15.1@sha256:32dac94127fd60b7b7e3fbfc65e1383b9b5e25c9bfd7b8536de7a539fe68a12d hadolint --ignore DL3008 - < image/Dockerfile + - name: The workflows + run: docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 -color + image: strategy: fail-fast: false @@ -108,13 +130,16 @@ jobs: if-no-files-found: error publish: - needs: image + needs: [lint, image] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-24.04 permissions: contents: write packages: write steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 # the history and the tags: what changed since the previous release - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: packages-* @@ -134,6 +159,38 @@ jobs: GH_TOKEN: ${{ github.token }} DIGEST: ${{ steps.image.outputs.digest }} run: | + set -euo pipefail + # The previous release: the one whose tag sorts last before this one by version; none for the first. + gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 --json tagName --jq '.[].tagName' > releases + previous=$({ cat releases; echo "$GITHUB_REF_NAME"; } | sort -uV | grep -xF -B1 "$GITHUB_REF_NAME" | head -n -1) + if [ -n "$previous" ]; then + gh release download "$previous" --repo "$GITHUB_REPOSITORY" --pattern 'packages-*.txt' --dir previous + { + echo "## Changes" + echo + git log --no-merges --format='- %s' "$previous..$GITHUB_REF_NAME" -- image/ | grep . || echo "- none in image/" + echo + echo "## Packages since $previous" + for arch in amd64 arm64; do + echo + echo "### $arch" + echo + # Each line: package, version, architecture (and, from the release after 2026.11.12 on, source + # package and version); a package is its name and architecture. + awk 'NR == FNR { old[$1 " " $3] = $2; next } + { new[$1 " " $3] = $2 } + END { + for (k in new) { + split(k, p, " ") + if (!(k in old)) print "- added: " p[1] " " new[k] + else if (old[k] != new[k]) print "- changed: " p[1] " " old[k] " -> " new[k] + } + for (k in old) if (!(k in new)) { split(k, p, " "); print "- removed: " p[1] " " old[k] } + }' "previous/packages-$arch.txt" "packages-$arch.txt" | sort -k3,3 > "changes-$arch" + if [ -s "changes-$arch" ]; then cat "changes-$arch"; else echo "- none changed"; fi + done + } > changes.md + fi { echo "Pin it as:" echo @@ -143,7 +200,30 @@ jobs: echo echo "Built from image/ as of ${GITHUB_SHA}, tested and published as it was built; the packages of each variant are attached." echo + if [ -f changes.md ]; then cat changes.md; echo; fi echo "The image holds Ubuntu 24.04's packages under their own licenses; each line of the package lists names a package, its version, architecture, source package and source version, whose sources are at https://launchpad.net/ubuntu/+source/SOURCE/VERSION (README, License)." } > notes.md gh release create "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" --verify-tag \ --title "$GITHUB_REF_NAME" --notes-file notes.md packages-amd64.txt packages-arm64.txt + + # The one check main's ruleset requires: it passes only when the lint and both variants passed. A variant + # built before is pulled, not skipped, so no job here is skipped by design: a skipped one means a job it + # needs failed or the run was cancelled. The publication, on tags only, is not part of it. + ci: + if: always() + needs: [lint, image] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Every job passed + env: + NEEDS: ${{ toJSON(needs) }} + run: | + set -euo pipefail + failed=$(jq -r 'to_entries[] | select(.value.result != "success") | "\(.key): \(.value.result)"' <<< "$NEEDS") + if [ -n "$failed" ]; then + echo "$failed" >&2 + exit 1 + fi + echo "every job passed" diff --git a/README.md b/README.md index 257a3c5..b27bade 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,11 @@ are built with the amd64 variant only: on arm64 the compiler is the native one, ## The contract -What overpressure's tooling (`overpressure/linux.py`) relies on. A change to it is a new image that the -consumers pin on purpose. +What the consumers rely on: overpressure's tooling (`IMAGE` in +[overpressure/linux.py](https://github.com/overpressure/overpressure/blob/main/overpressure/linux.py)) and the +packages' CI (`BUILDER` in overpressure/packages' +[packages.yml](https://github.com/overpressure/packages/blob/main/.github/workflows/packages.yml)). A change to it +is a new image that the consumers pin on purpose. - **How it runs.** Privileged (`--privileged`): the builds mount `/proc`, `/sys` and overlayfs and chroot. The checkout is bound at `/work`, the working directory; build roots live in a volume at `/var/tmp`. @@ -56,11 +59,18 @@ Every pull request, push to `main` and tag checks both variants' tools and runs ([image.yml](.github/workflows/image.yml)). Each variant is built once per content of `image/` and kept as `ghcr.io/overpressure/builder:ci--`; later runs pull it instead of building it again. A tag `YYYY.MM.DD` (or `YYYY.MM.DD.N`) publishes the tested variants as one multi-arch image under that tag, with a -release that names its digest and lists each variant's packages. `main` changes only by pull request. -Published images are kept: an old pin keeps working. +release that names its digest and lists each variant's packages; its notes list the commits to `image/` and +each variant's packages added, removed or at another version since the previous release. `main` changes only +by pull request; the run's last job, `ci`, passes only when the lint and both variants passed, and is the +check `main` requires. Published images are kept: an old pin keeps working. + +The `ci-*` tags in `ghcr.io/overpressure/builder` are CI's cache of tested variants: never pin them. A release is +an exact pin by digest, so there are no channels. ```sh -python3 -m unittest -v # the recipe test; the script tests need x86-64 Linux, root and bash +python3 -m unittest -v # the recipe test; the binfmt tests need bash on x86-64 Linux, the hand-back tests root +uvx ruff@0.17.4 format --check . && uvx ruff@0.17.4 check . # the Python code (pyproject.toml) +docker run --rm -i hadolint/hadolint:v2.15.1 hadolint --ignore DL3008 - < image/Dockerfile # the Dockerfile docker build -t builder:local image ``` @@ -69,7 +79,8 @@ docker build -t builder:local image [The Unlicense](LICENSE): the Dockerfile and the scripts. The image itself contains Ubuntu 24.04's packages, unmodified, each under its own licenses (many GPL or LGPL), stated in the image's `/usr/share/doc/PACKAGE/copyright`. Their sources are Ubuntu's: each release's `packages-ARCH.txt` (in the -image `/usr/share/overpressure-packages.txt`) lists every package with its version and its source package and -version, and every published version of a source package, with its `.dsc`, upstream and Debian archives, is -kept at `https://launchpad.net/ubuntu/+source/SOURCE/VERSION` (also by date on snapshot.ubuntu.com). The -image is a build tool for developers and CI; nothing of it goes onto a headset. +image `/usr/share/overpressure-packages.txt`) lists every package with its version and architecture, and from +the release after 2026.11.12 on also its source package and source version (2026.11.12's lists carry name, +version and architecture only). Every published version of a source package, with its `.dsc`, upstream and +Debian archives, is kept at `https://launchpad.net/ubuntu/+source/SOURCE/VERSION` (also by date on +snapshot.ubuntu.com). The image is a build tool for developers and CI; nothing of it goes onto a headset. diff --git a/image/Dockerfile b/image/Dockerfile index cc45194..74e10d7 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -10,17 +10,19 @@ # (README.md): consumers pull it, nobody needs to rebuild it to get the same tools. FROM ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c ARG DEBIAN_FRONTEND=noninteractive +# A failing command in a pipe fails the build (the package list's dpkg-query). +SHELL ["/bin/bash", "-o", "pipefail", "-c"] # The kernel's cross compiler only where the host is no arm64 (there the native gcc has its name). RUN set -eu; \ - arch=$(dpkg --print-architecture); \ - apt="apt-get -o Acquire::Retries=5"; \ - cross=$([ "$arch" = arm64 ] || echo gcc-aarch64-linux-gnu); \ - $apt update; \ - $apt install -y --no-install-recommends \ + apt=(apt-get -o Acquire::Retries=5); \ + cross=(); \ + [ "$(dpkg --print-architecture)" = arm64 ] || cross=(gcc-aarch64-linux-gnu); \ + "${apt[@]}" update; \ + "${apt[@]}" install -y --no-install-recommends \ bash ca-certificates coreutils curl diffutils findutils tar zstd xz-utils python3 e2fsprogs util-linux \ clang-20 lld-20 qemu-user-static \ - make gcc libc6-dev $cross flex bison bc git perl cpio libssl-dev libelf-dev dwarves device-tree-compiler \ - shellcheck; \ + make gcc libc6-dev "${cross[@]}" flex bison bc git perl cpio libssl-dev libelf-dev dwarves \ + device-tree-compiler shellcheck; \ rm -rf /var/lib/apt/lists/*; \ dpkg-query -W -f '${Package} ${Version} ${Architecture} ${source:Package} ${source:Version}\n' | sort \ > /usr/share/overpressure-packages.txt; \ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9973d1d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Unlicense +# The linter's settings, overpressure's (https://github.com/overpressure/overpressure/blob/main/pyproject.toml); +# nothing here is a Python package. CI runs ruff pinned through uvx (.github/workflows/image.yml). + +[tool.ruff] +line-length = 120 +target-version = "py39" + +[tool.ruff.lint] +# Every rule. What is off below conflicts with another rule, with the formatter or with how the +# project is built, and says so; everything else is fixed, or carries a noqa with its reason. +select = ["ALL"] +ignore = [ + # Incompatible with D211 and D212, which ruff keeps: no blank line before a class docstring, + # and a multi-line docstring's summary on its first line. + "D203", + "D213", + # The formatter places trailing commas (ruff's documented conflict). + "COM812", + # pytest's style: the tests are unittest (standard library only), whose assertion methods + # report what differed; a bare assert does not, and S101 forbids it. + "PT009", + "PT027", + # One with statement for several contexts needs parentheses from Python 3.10 on to wrap. + "SIM117", +] + +[tool.ruff.lint.per-file-ignores] +# Tests: their names say what they check, as docstrings would; they return nothing; their expected +# values are literals by nature. +"tests/**" = ["D101", "D102", "D105", "D107", "ANN201", "ANN204", "PLR2004", "S101"] + +[tool.ruff.lint.flake8-copyright] +# The project's files name their license by SPDX identifier; the Unlicense has no copyright holder. +notice-rgx = "SPDX-License-Identifier: " + +[tool.ruff.lint.pydocstyle] +convention = "pep257" diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..706fec3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Unlicense +"""The image's checks: its scripts and its recipe.""" diff --git a/tests/test_image.py b/tests/test_image.py index a825a9b..305d3b9 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -16,7 +16,6 @@ import unittest from pathlib import Path - IMAGE = Path(__file__).resolve().parents[1] / "image" LINUX_BASH = sys.platform.startswith("linux") and shutil.which("bash") is not None #: What binfmt.sh writes to register: the kernel decodes the escapes itself, so they arrive as text.