diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0139f8f..dbb1ab7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,10 +36,9 @@ jobs: runs-on: ubuntu-latest # The gate starts a Volume Agent, and the Agent creates and inspects every qcow2 # chain by running the pinned qemu-img — v6 §7 forbids a parser of our own — so it - # refuses to start without one. `task qemu:tools` pulls the image qemu.yml publishes - # and extracts the binary; building QEMU here would be tens of minutes inside a - # sixty-minute job. This is the preflight the Taskfile's own comment predicted the - # qcow2 lane would need. + # refuses to start without one. `task machine` fetches the pinned spin-machine release, + # which is minutes of download against the tens of minutes compiling QEMU here would + # cost, and this repository no longer has a QEMU to compile. permissions: contents: read packages: read @@ -94,14 +93,10 @@ jobs: # Named as its own step so a missing or unpublished image fails here, under a name # that says what is wrong, rather than sixteen minutes into the gate as a lane that # cannot start an Agent. - - name: The pinned qemu-img the Agent needs - env: - QEMU_IMAGE: ghcr.io/${{ github.repository }}/qemu - run: task qemu:tools + - name: The pinned machine (QEMU, qemu-img, the kernel) + run: task machine - name: The merge gate (task ci:full) - env: - QEMU_IMAGE: ghcr.io/${{ github.repository }}/qemu run: task ci:full # The guest lane: the four demos, each booting a real Linux kernel against a qcow2 this @@ -153,26 +148,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # QEMU *and* qemu-img, with the loader and library closure they were built against. - # A bare runner has none of those, which is what ADR-0025 originally answered by - # running the lane inside the runtime image; that is not available here, because the - # demos start the dev Postgres through Docker and a container job has no daemon. - # The wrapper qemu:tools writes solves the same problem from the other side and is - # already proven — it is how every lane gets qemu-img today. - - name: The pinned QEMU and qemu-img - env: - QEMU_IMAGE: ghcr.io/${{ github.repository }}/qemu - run: task qemu:tools - - # storage does not build a kernel (ADR-0021). The mirror is the only source on a - # machine with no sibling spinbox checkout, and it is resolved by content tag so the - # pin is spelled out in exactly one place — the Taskfile — and a moved tag cannot - # swap the kernel under a lane that already certified one. - - name: The pinned guest kernel - run: | - set -euo pipefail - tag=$(task guest:kernel:tag) - task guest:kernel:fetch GUEST_KERNEL_IMAGE=ghcr.io/${{ github.repository }}/guest-kernel:"$tag" + # Both QEMU binaries, qemu-img and the kernel, in one step — they are one artefact. + # + # This used to be two steps and a paragraph about loaders: the binaries were + # dynamically linked against their build image, so a bare runner could not run them + # and a wrapper had to carry the loader and the library closure beside them. They are + # statically linked now, so an extracted tree runs wherever the kernel does and all + # of that is gone. + - name: The pinned machine (QEMU, qemu-img, the kernel) + run: task machine # Asserts the lane's inputs before anything boots: the kernel is the one that was # pinned, is a PVH ELF, carries the four config options a boot depends on, and the diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml deleted file mode 100644 index 9ffe64d..0000000 --- a/.github/workflows/qemu.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: QEMU - -# The pinned QEMU (CLAUDE.md, and QEMU_VERSION in Taskfile.yml) takes tens of minutes to -# build from source, far too slow for the per-push gate. So it is its own workflow, -# triggered only when something that can change the artefact changes, and it publishes a -# runtime image for the lanes plus the extracted binaries as an artefact. -on: - push: - branches: [main] - paths: - - Dockerfile.qemu - - .github/workflows/qemu.yml - - Taskfile.yml # the version and the build flags live here - pull_request: - paths: - - Dockerfile.qemu - - .github/workflows/qemu.yml - - Taskfile.yml - workflow_dispatch: - -permissions: - contents: read - packages: write - -concurrency: - group: qemu-${{ github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - # A cold QEMU build is tens of minutes and a warm one is a cache restore, so the - # spread is wide and the ceiling has to sit above the cold case. What it is really - # for is the other end: the default is six hours, and a `configure` that stops on a - # prompt or a registry that never answers would hold a runner for all of them - # without ever printing a failure. - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Set up Task - uses: go-task/setup-task@v2 - - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - # The pinned version comes from the Taskfile, never from a literal here: two - # places to bump is how CI and production end up on different QEMUs. - - name: Read the pinned QEMU version - id: qemu - run: echo "version=$(task qemu:version)" >> "$GITHUB_OUTPUT" - - # `task qemu:build` runs the same buildx command a developer runs, with the GitHub - # Actions cache backend. It ends in `task qemu:verify`, so a build that produced the - # wrong version or lost a device we depend on fails here, not in a lane weeks later. - - name: Build QEMU and verify what came out - run: task qemu:build QEMU_CACHE_FROM=type=gha QEMU_CACHE_TO=type=gha,mode=max QEMU_JOBS=4 - - - name: Upload the extracted binaries - uses: actions/upload-artifact@v4 - with: - name: qemu-${{ steps.qemu.outputs.version }}-linux-amd64 - path: _output/ - retention-days: 30 - if-no-files-found: error - - # Publishing is skipped for pull requests: a fork's token cannot write packages, - # and an unreviewed branch has no business claiming the version tag. - - name: Log in to GitHub Packages - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Two tags: the QEMU version, which is what everything pins, and the commit, - # which is what makes a specific build reproducible when the version tag has - # moved because a flag changed. - - name: Publish the runtime image - if: github.event_name != 'pull_request' - run: | - task qemu:push \ - QEMU_IMAGE=ghcr.io/${{ github.repository }}/qemu \ - QEMU_IMAGE_EXTRA_TAG=${{ github.sha }} \ - QEMU_CACHE_FROM=type=gha \ - QEMU_CACHE_TO=type=gha,mode=max \ - QEMU_JOBS=4 diff --git a/Dockerfile.guest-kernel b/Dockerfile.guest-kernel deleted file mode 100644 index 6e7cfa6..0000000 --- a/Dockerfile.guest-kernel +++ /dev/null @@ -1,36 +0,0 @@ -# syntax=docker/dockerfile:1 - -# The guest kernel, mirrored (ADR-0022). -# -# storage does not build a kernel; it consumes spinbox's (ADR-0021). This file does not -# build one either — it moves an artefact that already exists into a registry and back -# out again, so the lane's kernel can be named by digest instead of by a relative path -# into someone's home directory. -# -# Two directions, one file: -# -# --target package context holds `vmlinux` → an image carrying just that file. -# `task guest:kernel:push` publishes it. -# --target extract pull GUEST_KERNEL_REF and write /vmlinux onto the host. -# `task guest:kernel:fetch` uses it, the same way `task qemu:build` -# gets binaries out of an image with `--output type=local`. -# -# The image is a single file on `scratch` on purpose: nothing in it can be executed, -# nothing needs updating for CVEs, and its digest is a hash of the kernel and nothing -# else. - -# Declared before the first FROM so it is a *global* arg — one below a FROM belongs to -# that stage only, and the `FROM ${GUEST_KERNEL_REF}` line then resolves to blank and -# fails the parse even when the target that uses it was not requested. The default is -# kept resolvable for the same reason: every stage is parsed, whichever one is built. -# The extract path always passes a real reference, and it should be a digest — a tag can -# be moved, and then the pin in Taskfile.yml is the only thing that would notice. -ARG GUEST_KERNEL_REF=scratch - -FROM scratch AS package -COPY vmlinux /vmlinux - -FROM ${GUEST_KERNEL_REF} AS published - -FROM scratch AS extract -COPY --from=published /vmlinux /vmlinux diff --git a/Dockerfile.qemu b/Dockerfile.qemu deleted file mode 100644 index 1c373ad..0000000 --- a/Dockerfile.qemu +++ /dev/null @@ -1,330 +0,0 @@ -# QEMU build for the remote-volumes work. -# -# Modelled on spin-stack/spinbox's Dockerfile.qemu (same BuildKit cache-mount -# strategy, same debloat flags) with two differences that matter here: -# -# 1. QEMU is pinned to 11.1.1 — the version CLAUDE.md pins for CI and prod — -# not spinbox's 11.0.1. -# 2. --enable-tools keeps qemu-img, which is not a convenience: v6 forbids a -# qcow2 parser of our own, so every chain this system creates and inspects -# is that binary being run. -# -# What is *not* built: --enable-vhost-user-blk-server and the storage daemon it -# served. The guest gets an ordinary virtio-blk-pci disk over a local qcow2 chain, -# and the vhost-user data path went with the local block engine — a flag that -# lengthens every build for a device nothing asks for is not a spare part. -# -# The `extract` stage writes the binary + firmware to _output/ (task qemu:build). - -FROM ubuntu:26.04 AS builder - -ARG QEMU_VERSION=11.1.1 -ARG JOBS=8 -# Bump when the configure flags or hack/qemu-devices.mak change: the build directories -# live in cache mounts and configure only runs when one holds no config.status, so -# without this a flag change would silently reuse the previous configuration. -ARG QEMU_CONFIG_REV=6 -ARG DEBIAN_FRONTEND=noninteractive - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# Keep .debs so the cache mounts below are useful. -RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ - echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache - -RUN --mount=type=cache,sharing=locked,id=qemu-aptlib,target=/var/lib/apt \ - --mount=type=cache,sharing=locked,id=qemu-aptcache,target=/var/cache/apt \ - apt-get update && apt-get upgrade -y && \ - apt-get --no-install-recommends install -y \ - build-essential \ - meson \ - ninja-build \ - pkg-config \ - python3 \ - python3-venv \ - python3-tomli \ - bash \ - coreutils \ - curl \ - ca-certificates \ - xz-utils \ - libglib2.0-dev \ - libglib2.0-0 \ - zlib1g-dev \ - libpixman-1-dev \ - linux-libc-dev \ - libcap-ng-dev \ - libseccomp-dev \ - libseccomp2 \ - libaio-dev \ - liburing-dev \ - acpica-tools \ - git \ - flex \ - bison \ - patch && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Source tarball + extracted tree live in a cache mount, shared across builds. -RUN --mount=type=cache,sharing=locked,id=qemu-src-${QEMU_VERSION},target=/var/cache/qemu \ - if [ ! -f "/var/cache/qemu/qemu-${QEMU_VERSION}.tar.xz" ]; then \ - curl -fsSL "https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz" \ - -o "/var/cache/qemu/qemu-${QEMU_VERSION}.tar.xz"; \ - fi && \ - curl -fsSL "https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz.sig" \ - -o "/var/cache/qemu/qemu-${QEMU_VERSION}.tar.xz.sig" || true && \ - if [ ! -d "/var/cache/qemu/qemu-${QEMU_VERSION}" ]; then \ - tar -xf "/var/cache/qemu/qemu-${QEMU_VERSION}.tar.xz" -C /var/cache/qemu; \ - fi - -# The device set (see hack/qemu-devices.mak) and the flags both builds share. -# -# ENV rather than a list repeated twice: the two builds below differ in exactly one thing, -# and a copied flag list is a place for them to drift into differing in two. -COPY hack/qemu-devices.mak /build/spin-devices.mak - -ENV QEMU_FLAGS="\ - --target-list=x86_64-softmmu \ - --with-devices-x86_64=spin \ - --enable-system \ - --disable-user \ - --enable-tools \ - --enable-kvm \ - --enable-linux-aio \ - --enable-linux-io-uring \ - --enable-cap-ng \ - --enable-seccomp \ - --enable-lto \ - --enable-strip \ - --disable-containers \ - --disable-docs \ - --disable-debug-info \ - --disable-werror \ - --disable-plugins \ - --disable-xen \ - --disable-xen-pci-passthrough \ - --disable-whpx \ - --disable-hvf \ - --disable-nvmm \ - --disable-vhost-user-blk-server \ - --disable-sdl \ - --disable-gtk \ - --disable-curses \ - --disable-opengl \ - --disable-virglrenderer \ - --disable-spice \ - --disable-vnc \ - --disable-png \ - --disable-vte \ - --disable-xkbcommon \ - --disable-brlapi \ - --disable-alsa \ - --disable-coreaudio \ - --disable-oss \ - --disable-pa \ - --disable-jack \ - --disable-sndio \ - --disable-pipewire \ - --disable-slirp \ - --disable-passt \ - --disable-vde \ - --disable-netmap \ - --disable-gio \ - --disable-curl \ - --disable-libssh \ - --disable-libnfs \ - --disable-libiscsi \ - --disable-rbd \ - --disable-vvfat \ - --disable-vdi \ - --disable-vmdk \ - --disable-qed \ - --disable-qcow1 \ - --disable-bochs \ - --disable-cloop \ - --disable-dmg \ - --disable-parallels \ - --disable-vhdx \ - --disable-vpc \ - --disable-tpm \ - --disable-libusb \ - --disable-usb-redir \ - --disable-smartcard \ - --disable-u2f \ - --disable-canokey \ - --disable-guest-agent \ - --disable-guest-agent-msi \ - --disable-capstone \ - --disable-fuse \ - --disable-fuse-lseek \ - --disable-multiprocess \ - --disable-replication \ - --disable-rdma \ - --disable-libdw \ - --disable-gettext \ - --disable-virtfs \ - --disable-hv-balloon" - -# What is *not* in that list, and why, because the tempting ones fail late: -# pixman, glib, gnutls/nettle — q35's display and qcow2's crypto still want them. -# fdt — MICROVM depends on it (hw/i386/Kconfig). -# CONFIG_CXL — not offered as optional by upstream; turning it off -# broke the 11.0.2 link (pxb_cxl_realize, acpi_build). -# --without-default-features — then every device q35 needs is ours to enumerate, and -# a missing one is a link error or an unusable machine. - -# The guard is config.status and not Makefile, and the directory is emptied before a -# configure runs. configure writes the Makefile early and config.status only once it has -# finished, so a configure that failed leaves a build directory that looks configured and -# a `make` that dies at `./config.status: No such file or directory` — in a cache mount, -# on every later build, until somebody thinks to prune it. -# -# Two builds, differing in one flag: whether TCG is compiled in. -# -# /opt/qemu is what a host serving tenants runs, and it cannot emulate. A machine with no -# /dev/kvm gets an error at start-up instead of a guest at a tenth of the speed producing -# timings that read like measurements — which is a failure mode this repository has already -# paid for once, and the reason hack/demo-lib.sh prints the accelerator it chose. -# -# /opt/qemu-tcg is for CI, where the runner has no /dev/kvm and the guest boots a kernel, -# an initramfs and one disk. Only its system binary is kept: qemu-img and qemu-nbd link no -# accelerator at all, so there is one of each and they come from the build above. -RUN --mount=type=cache,sharing=locked,id=qemu-src-${QEMU_VERSION},target=/var/cache/qemu \ - --mount=type=cache,sharing=locked,id=qemu-build-kvm-${QEMU_VERSION}-${QEMU_CONFIG_REV},target=/build/qemu-build-kvm \ - cp /build/spin-devices.mak \ - "/var/cache/qemu/qemu-${QEMU_VERSION}/configs/devices/x86_64-softmmu/spin.mak" && \ - mkdir -p qemu-build-kvm && cd qemu-build-kvm && \ - if [ ! -f config.status ]; then \ - rm -rf ./* && \ - /var/cache/qemu/qemu-${QEMU_VERSION}/configure \ - --prefix=/opt/qemu \ - --disable-tcg \ - ${QEMU_FLAGS}; \ - fi && \ - make -j${JOBS} && \ - make install - -RUN --mount=type=cache,sharing=locked,id=qemu-src-${QEMU_VERSION},target=/var/cache/qemu \ - --mount=type=cache,sharing=locked,id=qemu-build-tcg-${QEMU_VERSION}-${QEMU_CONFIG_REV},target=/build/qemu-build-tcg \ - cp /build/spin-devices.mak \ - "/var/cache/qemu/qemu-${QEMU_VERSION}/configs/devices/x86_64-softmmu/spin.mak" && \ - mkdir -p qemu-build-tcg && cd qemu-build-tcg && \ - if [ ! -f config.status ]; then \ - rm -rf ./* && \ - /var/cache/qemu/qemu-${QEMU_VERSION}/configure \ - --prefix=/opt/qemu-tcg \ - --enable-tcg \ - ${QEMU_FLAGS}; \ - fi && \ - make -j${JOBS} && \ - make install - -# Firmware the microVM needs to boot, next to the binary. -RUN --mount=type=cache,sharing=locked,id=qemu-src-${QEMU_VERSION},target=/var/cache/qemu \ - cp -a /var/cache/qemu/qemu-${QEMU_VERSION}/pc-bios/* /opt/qemu/share/qemu/ && \ - test -f /opt/qemu/share/qemu/bios-256k.bin && \ - mkdir -p /build/pc-bios && \ - cp /var/cache/qemu/qemu-${QEMU_VERSION}/pc-bios/kvmvapic.bin /build/pc-bios/ && \ - cp /var/cache/qemu/qemu-${QEMU_VERSION}/pc-bios/vgabios-stdvga.bin /build/pc-bios/ - -# Fail the build here, not in a lane three phases later, if this configuration did not -# produce the devices, the tools and the two accelerators the image ships. -# -# The last two lines are the split itself: the binary a tenant's host runs must refuse to -# emulate, and CI's must be able to. Neither is visible in a file listing, and getting them -# backwards is a fleet that quietly runs at a tenth of the speed or a CI lane that cannot -# boot at all. -RUN /opt/qemu/bin/qemu-system-x86_64 -device help 2>&1 | grep -q 'virtio-blk-pci' && \ - /opt/qemu/bin/qemu-system-x86_64 -device help 2>&1 | grep -q 'virtio-net-pci' && \ - ! /opt/qemu/bin/qemu-system-x86_64 -device help 2>&1 | grep -qE '^name "(e1000e?|rtl8139|pcnet|vmxnet3|igb|ne2k_pci|tulip)"' && \ - /opt/qemu/bin/qemu-system-x86_64 -machine help | grep -q '^q35 ' && \ - ! /opt/qemu/bin/qemu-system-x86_64 -machine help | grep -q '^pc-i440fx' && \ - test -x /opt/qemu/bin/qemu-img && \ - /opt/qemu/bin/qemu-system-x86_64 --version && \ - /opt/qemu-tcg/bin/qemu-system-x86_64 --version && \ - ! /opt/qemu/bin/qemu-system-x86_64 -accel help | grep -qx tcg && \ - /opt/qemu-tcg/bin/qemu-system-x86_64 -accel help | grep -qx tcg - -# ============================================ -# Runtime image (used by the integration lane) -# ============================================ -FROM ubuntu:26.04 AS runtime - -ARG DEBIAN_FRONTEND=noninteractive - -RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ - echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache - -RUN --mount=type=cache,sharing=locked,id=qemu-runtime-aptlib,target=/var/lib/apt \ - --mount=type=cache,sharing=locked,id=qemu-runtime-aptcache,target=/var/cache/apt \ - apt-get update && \ - # libaio1t64 and not libaio1: Ubuntu's 64-bit time_t transition renamed the runtime - # package, so the old name resolves to nothing and `apt-get install` exits 100 — - # which is what this stage did until the first CI run went looking for it. The - # builder stage takes the -dev packages, whose names did not change, which is why - # the build kept succeeding while the runtime image could not be assembled at all. - apt-get --no-install-recommends install -y \ - libglib2.0-0 \ - libpixman-1-0 \ - libcap-ng0 \ - libseccomp2 \ - libaio1t64 \ - liburing2 \ - zlib1g && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /opt/qemu/bin/qemu-system-x86_64 /usr/local/bin/qemu-system-x86_64 -COPY --from=builder /opt/qemu-tcg/bin/qemu-system-x86_64 /usr/local/bin/qemu-system-x86_64-tcg -COPY --from=builder /opt/qemu/bin/qemu-img /usr/local/bin/qemu-img -COPY --from=builder /opt/qemu/bin/qemu-nbd /usr/local/bin/qemu-nbd -# The firmware this project boots with, and not `share/qemu/` — that directory is 331 MB, -# of which 317 MB is edk2 for aarch64, arm, riscv and loongarch, in a build whose target -# list is x86_64-softmmu. `task qemu:tools` copies this tree out of the image on every -# developer machine and every CI run, so the whole of it was moving on every run. -# -# The same five files the extract stage below takes, and that set is proven by execution: -# every demo boots a real guest against it. -COPY --from=builder /opt/qemu/share/qemu/bios.bin /usr/share/spin-stack/qemu/bios.bin -COPY --from=builder /opt/qemu/share/qemu/bios-256k.bin /usr/share/spin-stack/qemu/bios-256k.bin -COPY --from=builder /opt/qemu/share/qemu/pvh.bin /usr/share/spin-stack/qemu/pvh.bin -COPY --from=builder /opt/qemu/share/qemu/efi-virtio.rom /usr/share/spin-stack/qemu/efi-virtio.rom -COPY --from=builder /build/pc-bios/kvmvapic.bin /usr/share/spin-stack/qemu/kvmvapic.bin -COPY --from=builder /build/pc-bios/vgabios-stdvga.bin /usr/share/spin-stack/qemu/vgabios-stdvga.bin - -ENTRYPOINT ["/usr/local/bin/qemu-system-x86_64"] - -# ============================================ -# Extract stage: binaries onto the host (_output/) -# ============================================ -FROM scratch AS extract - -COPY --from=builder /opt/qemu/bin/qemu-system-x86_64 /bin/qemu-system-x86_64 -COPY --from=builder /opt/qemu-tcg/bin/qemu-system-x86_64 /bin/qemu-system-x86_64-tcg -COPY --from=builder /opt/qemu/bin/qemu-img /bin/qemu-img -COPY --from=builder /opt/qemu/bin/qemu-nbd /bin/qemu-nbd -COPY --from=builder /opt/qemu/share/qemu/bios.bin /share/spin-stack/qemu/bios.bin -COPY --from=builder /opt/qemu/share/qemu/bios-256k.bin /share/spin-stack/qemu/bios-256k.bin -COPY --from=builder /opt/qemu/share/qemu/pvh.bin /share/spin-stack/qemu/pvh.bin -COPY --from=builder /opt/qemu/share/qemu/efi-virtio.rom /share/spin-stack/qemu/efi-virtio.rom -COPY --from=builder /build/pc-bios/kvmvapic.bin /share/spin-stack/qemu/kvmvapic.bin -COPY --from=builder /build/pc-bios/vgabios-stdvga.bin /share/spin-stack/qemu/vgabios-stdvga.bin -# pvh.bin above is the one that matters for the guest lane: the kernel we boot is an ELF -# with Xen PVH notes (CONFIG_PVH=y), and QEMU enters it through this ROM — no real mode, -# no bzImage boot protocol. linuxboot_dma.bin is deliberately NOT extracted: it is the -# option ROM for a *bzImage* boot, which nothing here does. Verified by execution: the -# guest boots with this exact set and boots identically with linuxboot_dma.bin removed. -# -# efi-virtio.rom is the sixth file and the one that is not about booting: a NIC whose -# option ROM is missing does not slow a VM down, it stops it from starting at all — -# `failed to find romfile`, at realize, before any guest code runs. That is how the first -# cut of this list was found to be wrong, and virtio-net is the one card this build keeps -# (hack/qemu-devices.mak). Every other card's model is gone, so no other ROM can be asked -# for; a caller that adds one back has to add its ROM here too. -# -# `-net none` is on every demo's machine line for the same argument one level down: a card -# nobody attaches is still a device the guest enumerates. -# -# Shipping the whole pc-bios tree instead is how 317 MB of firmware for aarch64, riscv and -# loongarch ended up in an image whose target list is x86_64-softmmu. diff --git a/Taskfile.yml b/Taskfile.yml index a580a34..2137cab 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -19,18 +19,22 @@ vars: OUTPUT_DIR: "{{.ROOT_DIR}}/_output" # Project-local by default so no task needs root; override for a shared CI cache. BUILDKIT_CACHE_DIR: '{{.BUILDKIT_CACHE_DIR | default (printf "%s/.cache/buildkit" .ROOT_DIR)}}' - BUILDKIT_CACHE_QEMU: '{{.BUILDKIT_CACHE_DIR}}/storage-qemu' - # QEMU is pinned here and in CLAUDE.md; the same version runs in CI and prod. - QEMU_VERSION: 11.1.1 - QEMU_JOBS: '{{.QEMU_JOBS | default 8}}' - # BuildKit cache backends. Local by default (a laptop); CI overrides them with - # `type=gha` so the build is not duplicated in a workflow file — the Dockerfile and - # the flags stay in one place, which is the rule the whole Taskfile exists for. - QEMU_CACHE_FROM: '{{.QEMU_CACHE_FROM | default (printf "type=local,src=%s" .BUILDKIT_CACHE_QEMU)}}' - QEMU_CACHE_TO: '{{.QEMU_CACHE_TO | default (printf "type=local,dest=%s,mode=max,compression=zstd" .BUILDKIT_CACHE_QEMU)}}' - # Where the runtime image is published. Overridden in CI with the GitHub Packages - # path; the default is a local tag so `task qemu:image` needs no registry. - QEMU_IMAGE: '{{.QEMU_IMAGE | default "spin-stack/storage-qemu"}}' + # --- the machine ------------------------------------------------------------------- + # QEMU, the guest kernel and the firmware are one versioned artefact built in + # spin-stack/spin-machine. This repository builds none of them: it used to build a QEMU + # from a recipe copied from another project, against a device list that was + # byte-for-byte that project's, and it mirrored that project's kernel through a registry + # so a machine with no sibling checkout could find one. All of that is what having the + # machine in one place replaced. + # + # Pinned by version and by content, for the reason the kernel used to be pinned by hash: + # "which machine did this lane certify?" needs an answer that is not "whatever was + # published that day". Bump both together and say in the commit why the machine changed. + SPIN_MACHINE_VERSION: '{{.SPIN_MACHINE_VERSION | default "v20260907.01"}}' + SPIN_MACHINE_SHA256: '{{.SPIN_MACHINE_SHA256 | default "72db5f9789a37370acaac25bf0aab2490f9b0fa0ee846a65f80651408bb0da47"}}' + # A sibling checkout that has already built one wins over the published release. + SPIN_MACHINE_DIR: '{{.SPIN_MACHINE_DIR | default "../spin-machine"}}' + # Object-store backend under test (§6.1 conformance suite). Pinned by digest so a # re-tagged image cannot silently change what the suite certified. RUSTFS_IMAGE: "rustfs/rustfs:latest@sha256:84ce557a0245a06a9aae5516f55ee0f007fca78d41df356f419306fdc0cb168c" @@ -47,22 +51,8 @@ vars: POSTGRES_IMAGE: postgres:18-alpine DEVDB_CONTAINER: spin-storage-devdb DEVDB_PORT: '{{.DEVDB_PORT | default "55432"}}' - # --- the guest kernel (ADR-0021, ADR-0022) --- - # storage does not build a kernel: it consumes spinbox's artefact, never its code. - # The lane reads it from ONE canonical path, and `task guest:kernel:fetch` puts it there - # from whichever source is available. Nothing else resolves a kernel path. + # The guest kernel, at the one path the lane reads. Put there by `task machine`. GUEST_KERNEL: '{{.OUTPUT_DIR}}/guest/vmlinux' - # Pinned by content, the same rule as RUSTFS_IMAGE: "which kernel did the lane - # certify" has to have an answer that is not "whatever was in _output that day". - # Bump deliberately with `task guest:kernel:pin` and say in the commit why it moved. - GUEST_KERNEL_VERSION: 7.2.1 - GUEST_KERNEL_SHA256: bd0ff2ed66f1837d1f7d26251c61b300d43810c8a320cfc02fd62179821a5c29 - # Source 1: a sibling spinbox checkout that has already built one (no network). - SPINBOX_KERNEL: '{{.SPINBOX_KERNEL | default "../spinbox/_output/spinbox-kernel-x86_64"}}' - # Source 2: the mirror. Unset by default — nothing is published yet — so the failure - # of `task guest:kernel:fetch` on a machine with no spinbox checkout names it as missing - # rather than pointing at a registry path that answers 404. - GUEST_KERNEL_IMAGE: '{{.GUEST_KERNEL_IMAGE | default ""}}' # Target database for db:plan / db:apply — the local development database by # default. Point DB_* at a real one to plan or apply against it; the password # comes from the environment (PGPASSWORD), never from a var in this file. @@ -100,7 +90,7 @@ includes: tools: {taskfile: ./taskfiles/tools.yml, flatten: true} generate: {taskfile: ./taskfiles/generate.yml, flatten: true} db: {taskfile: ./taskfiles/db.yml, flatten: true} - qemu: {taskfile: ./taskfiles/qemu.yml, flatten: true} + machine: {taskfile: ./taskfiles/machine.yml, flatten: true} guest: {taskfile: ./taskfiles/guest.yml, flatten: true} demo: {taskfile: ./taskfiles/demo.yml, flatten: true} backend: {taskfile: ./taskfiles/backend.yml, flatten: true} @@ -144,10 +134,9 @@ tasks: as processes, against a real Postgres 18 and the pinned RustFS. It is the only lane that can see the two binaries disagree — about a key file, about a flag nobody passes — because everything else drives Go types in-process. - # qemu:tools and not qemu:build: the Agent creates every chain by running the pinned - # qemu-img, so this lane cannot start one without it, and compiling QEMU here would - # put tens of minutes into the merge gate. - deps: [build:cmd, qemu:tools] + # The Agent creates and inspects every qcow2 chain by running the pinned qemu-img — v6 + # forbids a parser of our own — so this lane cannot start one without a machine. + deps: [build:cmd, machine] cmds: - RUSTFS_IMAGE={{.RUSTFS_IMAGE}} go test -tags e2e -count=1 -timeout 15m ./integration/e2e/... @@ -157,7 +146,7 @@ tasks: to S3 rather than to a container. Needs credentials, and creates one bucket per test that it purges on the way out. Not in ci:full — the merge gate must not need a cloud account. - deps: [build:cmd, qemu:tools] + deps: [build:cmd, machine] cmds: - SPIN_OBJECT_STORE=aws go test -tags e2e -count=1 -timeout 30m ./integration/e2e/... diff --git a/cmd/volume-agent/main.go b/cmd/volume-agent/main.go index b5dc4ac..5128777 100644 --- a/cmd/volume-agent/main.go +++ b/cmd/volume-agent/main.go @@ -101,7 +101,7 @@ func run() (err error) { // Required rather than defaulted to PATH: v6 pins QEMU to one version for CI // and production, and a chain created by whichever qemu-img a login shell found // is a chain nobody pinned. - return errors.New("-qemu-img is required: name the pinned binary (task qemu:build puts it in _output/bin)") + return errors.New("-qemu-img is required: name the pinned binary (task machine puts it in _output/bin)") } cfg := agent.Config{ diff --git a/hack/demo-lib.sh b/hack/demo-lib.sh index ffe9e2e..574e01f 100755 --- a/hack/demo-lib.sh +++ b/hack/demo-lib.sh @@ -29,6 +29,11 @@ OUT=$ROOT/_output # Two binaries, and which one runs is the same decision as which accelerator: the # production build has no TCG compiled in at all, so it is not a binary a machine without # KVM can run slowly — it is one that will not start. See ACCEL below. +# Every demo passes `-vga none` as well as `-display none`. The second only says not to +# open a window; the device is still created, and this machine's QEMU ships no +# vgabios-stdvga.bin for it — it has no display adapter, on purpose. Without the flag a +# guest dies at start-up with `failed to find romfile "vgabios-stdvga.bin"`, which reads +# like a missing firmware file rather than a device nobody wanted. QEMU=$OUT/bin/qemu-system-x86_64 QEMU_TCG=$OUT/bin/qemu-system-x86_64-tcg QEMU_IMG=$OUT/bin/qemu-img @@ -71,9 +76,9 @@ say() { printf '\n=== %s\n' "$*"; } die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } need() { test -x "$1" || test -f "$1" || die "missing $1 — run: $2"; } -need "$QEMU" "task qemu:build" -need "$QEMU_IMG" "task qemu:build" -need "$KERNEL" "task guest:kernel:fetch" +need "$QEMU" "task machine" +need "$QEMU_IMG" "task machine" +need "$KERNEL" "task machine" need "$INITRAMFS" "task guest:build" need "$CP" "task build:cmd" need "$AGENT" "task build:cmd" diff --git a/hack/guest-kernel.sh b/hack/guest-kernel.sh deleted file mode 100755 index d85762a..0000000 --- a/hack/guest-kernel.sh +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env bash -# The kernel the QEMU guest lane boots: acquire it, and prove it is the one we pinned. -# -# storage does not build a kernel — it consumes spinbox's artefact (ADR-0021). What this -# script adds is that the artefact stops being *a path on one developer's machine*: it is -# pinned by content hash and can come from a registry, so the lane is runnable somewhere -# other than a laptop with a sibling checkout (ADR-0022). -# -# Everything lands at one canonical path ($KERNEL, default _output/guest/vmlinux). No task -# and no test resolves `../spinbox/...` any more; that path is only ever a *source* to -# copy from, and one of several. -# -# fetch put the pinned kernel at $KERNEL, from the first source that has it -# verify assert what has to hold about it before a guest is booted with it -# pin print the sha256 of a file, for bumping GUEST_KERNEL_SHA256 -set -euo pipefail - -KERNEL=${KERNEL:-_output/guest/vmlinux} -# The pin. Empty lets `fetch` accept whatever it finds — for bisecting a kernel change, -# never for CI — and is refused outright by `verify`, which has nothing to verify without -# it. -KERNEL_SHA256=${KERNEL_SHA256:-} -KERNEL_VERSION=${KERNEL_VERSION:-} -# Source 1: a sibling spinbox checkout that has already built one. -SPINBOX_KERNEL=${SPINBOX_KERNEL:-} -# Source 2: the mirrored image (ADR-0022). Pin it by digest, not by tag. -KERNEL_IMAGE=${KERNEL_IMAGE:-} - -sha256() { sha256sum "$1" | cut -d' ' -f1; } - -# matches_pin succeeds when the file is there and is the artefact we pinned. With no pin -# declared, presence is all there is to check. -matches_pin() { - local f=$1 - [ -f "$f" ] || return 1 - [ -n "$KERNEL_SHA256" ] || return 0 - [ "$(sha256 "$f")" = "$KERNEL_SHA256" ] -} - -# --- fetch ------------------------------------------------------------------------ - -# from_local copies a kernel an adjacent spinbox checkout has already built. This is the -# zero-network path and the one that works today; the registry below is what makes the -# lane runnable where no such checkout exists. -from_local() { - [ -n "$SPINBOX_KERNEL" ] && [ -f "$SPINBOX_KERNEL" ] || return 1 - if ! matches_pin "$SPINBOX_KERNEL"; then - echo " $SPINBOX_KERNEL is $(sha256 "$SPINBOX_KERNEL"), not the pinned $KERNEL_SHA256" - return 1 - fi - cp -f "$SPINBOX_KERNEL" "$KERNEL" - echo " copied from $SPINBOX_KERNEL" -} - -# from_image pulls the mirror. buildx with a local output is the same mechanism -# `task qemu:build` uses to get binaries out of an image, so there is one way in this -# repository to turn a published artefact into a file in _output/. -from_image() { - [ -n "$KERNEL_IMAGE" ] || return 1 - command -v docker >/dev/null || { echo " no docker, cannot pull $KERNEL_IMAGE"; return 1; } - local tmp - tmp=$(mktemp -d) - # shellcheck disable=SC2064 # $tmp must expand now, not at trap time - trap "rm -rf '$tmp'" RETURN - docker buildx build \ - --file Dockerfile.guest-kernel \ - --target extract \ - --platform linux/amd64 \ - --build-arg "GUEST_KERNEL_REF=$KERNEL_IMAGE" \ - --output "type=local,dest=$tmp" \ - "$tmp" >/dev/null || { echo " pulling $KERNEL_IMAGE failed"; return 1; } - [ -f "$tmp/vmlinux" ] || { echo " $KERNEL_IMAGE has no /vmlinux"; return 1; } - if ! matches_pin "$tmp/vmlinux"; then - echo " $KERNEL_IMAGE carries $(sha256 "$tmp/vmlinux"), not the pinned $KERNEL_SHA256" - return 1 - fi - mv "$tmp/vmlinux" "$KERNEL" - echo " pulled from $KERNEL_IMAGE" -} - -cmd_fetch() { - mkdir -p "$(dirname "$KERNEL")" - - if matches_pin "$KERNEL"; then - echo "kernel already at $KERNEL (${KERNEL_VERSION:-unpinned version})" - return 0 - fi - # A file that is there but wrong is the case worth being loud about: it is how a lane - # certifies one kernel and reports on another. - if [ -f "$KERNEL" ]; then - echo "$KERNEL does not match the pin — replacing it" - fi - - echo "fetching the guest kernel ${KERNEL_VERSION:+$KERNEL_VERSION }(pin ${KERNEL_SHA256:-none})..." - if from_local || from_image; then - echo "✓ $KERNEL" - return 0 - fi - - cat >&2 <} - 2. the mirrored image — GUEST_KERNEL_IMAGE=${KERNEL_IMAGE:-} - -Any one of these fixes it: - cd ../spinbox && task build:kernel # then re-run; note it may exit non-zero and - # still emit the artefact (cache permissions) - task guest:kernel:fetch GUEST_KERNEL_IMAGE=ghcr.io///guest-kernel: - task guest:kernel:push GUEST_KERNEL_IMAGE=... # publish one you already have - -If the kernel legitimately changed, re-pin it rather than clearing the pin: - task guest:kernel:pin SPINBOX_KERNEL= -EOF - return 1 -} - -# --- verify ----------------------------------------------------------------------- - -# ikconfig prints the .config the kernel carries (CONFIG_IKCONFIG=y embeds it gzipped -# after the IKCFG_ST marker). Reading it from the vmlinux rather than from a config file -# next to it is the point: the artefact travels alone, and this is the only statement -# about it that cannot go stale. -ikconfig() { - local off - off=$(grep -abo -m1 'IKCFG_ST' "$KERNEL" 2>/dev/null | cut -d: -f1) || return 1 - [ -n "$off" ] || return 1 - # The gzip stream starts right after the 8-byte marker and is followed by the rest of - # the kernel image, so gzip decompresses the config and *then* exits non-zero on the - # trailing bytes. Under `set -o pipefail` that failure is the whole pipeline's, which - # is why the status is discarded here and the caller judges by the output instead. - tail -c "+$((off + 9))" "$KERNEL" | { gzip -dc 2>/dev/null || true; } -} - -# Each of these turns a boot-time symptom into a named failure here. Without VIRTIO_BLK -# there is no /dev/vda and the guest reports a missing device, which reads as a backend -# bug; without PVH, QEMU has no entry point and fails opening a ROM; without -# BLK_DEV_INITRD the initramfs is ignored and PID 1 never runs; without the 8250 console -# the verdict the host greps for is never printed. -readonly REQUIRED_CONFIG=( - CONFIG_VIRTIO_BLK - CONFIG_PVH - CONFIG_BLK_DEV_INITRD - CONFIG_SERIAL_8250_CONSOLE -) - -# cmd_verify refuses every state that is not "checked and correct", including a check it -# was not asked to perform (an empty pin) and one it could not perform (no embedded config, -# no readelf). Each refusal names the input and the task that produces it. -cmd_verify() { - local fail=0 - - test -f "$KERNEL" || { echo "no kernel at $KERNEL — run: task guest:kernel:fetch" >&2; return 1; } - - # `fetch` honours an empty pin — bisecting a kernel change needs that — but verifying - # against no pin is how a lane certifies one kernel and reports on another. - test -n "$KERNEL_SHA256" || { - echo "GUEST_KERNEL_SHA256 is empty: there is no pin to verify $KERNEL against" >&2 - echo "print its hash with 'task guest:kernel:pin -- $KERNEL' and set GUEST_KERNEL_SHA256 in Taskfile.yml" >&2 - return 1; } - local got - got=$(sha256 "$KERNEL") - if [ "$got" != "$KERNEL_SHA256" ]; then - echo "$KERNEL is not the pinned kernel:" >&2 - echo " pinned $KERNEL_SHA256" >&2 - echo " found $got" >&2 - echo "re-fetch it (task guest:kernel:fetch) or re-pin deliberately (task guest:kernel:pin)" >&2 - return 1 - fi - - # The lane boots a PVH ELF, not a bzImage. QEMU's failure for the wrong one is a - # rom-open error that reads like a backend bug, which is why it is checked here. - head -c4 "$KERNEL" | grep -q 'ELF' || { - echo "$KERNEL is not an ELF: the lane boots a PVH kernel, not a bzImage" >&2; return 1; } - # readelf absent used to skip the PVH-note check silently, on the theory that the tool is - # optional. It is not: without the note QEMU has no entry point, and a machine that - # cannot look is a machine that must not say OK. - command -v readelf >/dev/null || { - echo "readelf is missing, so $KERNEL's Xen PVH note cannot be checked" >&2 - echo "install binutils (apt-get install binutils) — QEMU has no entry point for a kernel without that note" >&2 - return 1; } - readelf -n "$KERNEL" 2>/dev/null | grep -q 'Xen' || { - echo "$KERNEL has no Xen PVH note: QEMU has no entry point for it (needs CONFIG_PVH=y)" >&2 - return 1; } - - local config - config=$(ikconfig || true) - # A kernel with no embedded config is legal in general and impossible here: the hash - # above established this is the pinned artefact, and that one embeds its config. So - # "unverifiable" means the pinned artefact changed shape. - [ -n "$config" ] || { - echo "$KERNEL has no embedded config (CONFIG_IKCONFIG=n), so ${REQUIRED_CONFIG[*]} cannot be checked" >&2 - echo "it matches GUEST_KERNEL_SHA256, so the pinned artefact itself changed: re-pin a kernel built with" >&2 - echo "CONFIG_IKCONFIG=y (cd ../spinbox && task build:kernel; task guest:kernel:pin -- )" >&2 - return 1; } - local opt - for opt in "${REQUIRED_CONFIG[@]}"; do - if ! grep -q "^${opt}=y$" <<<"$config"; then - echo "$KERNEL was built without ${opt}=y" >&2 - fail=1 - fi - done - [ "$fail" -eq 0 ] || return 1 - - # Just the version, not the full banner: the banner runs to the build host and - # timestamp, and the first copy of it in the image is a truncated format string. - local version - version=$(strings -a "$KERNEL" 2>/dev/null | grep -m1 -o 'Linux version [^ ]*' || true) - echo "OK: ${version:-$KERNEL} — pinned, PVH ELF, ${REQUIRED_CONFIG[*]} present" -} - -# --- pin -------------------------------------------------------------------------- - -cmd_pin() { - local f=${1:-${SPINBOX_KERNEL:-$KERNEL}} - test -f "$f" || { echo "no such file: $f" >&2; return 1; } - echo "$f" - echo " sha256: $(sha256 "$f")" - local version - version=$(strings -a "$f" 2>/dev/null | grep -m1 '^Linux version ' || true) - [ -n "$version" ] && echo " $version" - echo - echo "put that hash in Taskfile.yml as GUEST_KERNEL_SHA256, and record why it moved." -} - -case "${1:-}" in - fetch) cmd_fetch ;; - verify) cmd_verify ;; - pin) shift; cmd_pin "$@" ;; - *) echo "usage: $0 {fetch|verify|pin [file]}" >&2; exit 2 ;; -esac diff --git a/hack/publish-guest-inputs.sh b/hack/publish-guest-inputs.sh deleted file mode 100755 index e0bcf75..0000000 --- a/hack/publish-guest-inputs.sh +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/env bash -# The two artefacts CI's guest-backed jobs need and this repository does not build: the -# pinned QEMU runtime image and the pinned guest kernel. One command publishes both. -# -# WHY THIS EXISTS. Publishing was documented rather than automated, across four files, and -# most of the ways to get it wrong are silent: a path CI does not resolve, the version tag -# instead of the content hash, a package the repository's Actions token cannot read. All -# three end as "not published" — the message printed when nobody published anything. -# -# check the preconditions, each with its remedy. No side effects. -# publish check, publish both, then prove it by resolving them the way ci.yml does. -# Idempotent: re-run after the QEMU build finishes and it verifies. -# -# It does not build QEMU locally: qemu.yml builds the same artefact on a runner with the -# shared BuildKit cache and has workflow_dispatch. The kernel is the opposite case — -# storage never builds one (ADR-0021), so having it is a precondition, not a step. -# -# Nothing below has been run against a real registry; every precondition has been -# exercised in both states. -set -euo pipefail - -cd "$(dirname "$0")/.." - -# The registry is ghcr.io and only ghcr.io: ci.yml resolves both artefacts under -# `ghcr.io/${{ github.repository }}`, and a second registry here would be configuration for -# a need nobody has, whose first symptom is a lane resolving a path nothing published to. -readonly REGISTRY=ghcr.io - -# Overridable so a caller can point at a different binary — the same reason DEADCODE and -# TASK_EXE are overridable elsewhere in hack/. It is also how the precondition checks were -# exercised in the "everything present" state on a machine whose gh token lacks the scope. -GH=${GH:-gh} -DOCKER=${DOCKER:-docker} -TASK_EXE=${TASK_EXE:-task} - -# Supplied by the Taskfile (the single place each is declared); defaulted so the script is -# runnable by hand while still failing loudly on a value it cannot derive. -REPO=${REPO:-} -KERNEL=${KERNEL:-_output/guest/vmlinux} -KERNEL_SHA256=${KERNEL_SHA256:-} -KERNEL_VERSION=${KERNEL_VERSION:-} -SPINBOX_KERNEL=${SPINBOX_KERNEL:-} -QEMU_VERSION=${QEMU_VERSION:-} - -# Resolved by check_repository, used by everything after it. -qemu_image="" -kernel_image="" -kernel_source="" # the file publish will push: $KERNEL or $SPINBOX_KERNEL - -failures=0 -ok() { printf ' ok %s\n' "$1"; } -# A check that could not run says so, rather than printing nothing. Silence is how a -# reader concludes that everything above the failure was fine: the preflight this replaced -# had four states in which it reported OK having proven nothing, and this is the same trap -# one file over. It is not counted as a failure — the input it depends on already was. -skipped() { printf ' SKIPPED %s\n' "$1"; } -missing() { - printf ' MISSING %s\n' "$1" - shift - local line - for line in "$@"; do printf ' %s\n' "$line"; done - failures=$((failures + 1)) -} - -sha256() { sha256sum "$1" | cut -d' ' -f1; } - -# --- preconditions ------------------------------------------------------------------ -# -# Each one is a thing that has to be true before publishing can work, and each failure -# names the input and the command that fixes it. The order is the order a human would hit -# them, so the first MISSING is usually the only one that matters. - -check_repository() { - if [ -z "$REPO" ]; then - # `gh repo view` reads the git remote. This repository's remote is not always a - # GitHub URL (a local bare repo is a perfectly good origin), so the fallback is - # not an error condition — it is the normal case for a checkout that was never - # cloned from GitHub, and the remedy is one variable. - REPO=$($GH repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null || true) - fi - if [ -z "$REPO" ]; then - missing "the repository (owner/name)" \ - "nothing to name the packages after: 'gh repo view' could not resolve one from" \ - "this checkout's git remote, and REPO was not set." \ - "remedy: task guest:inputs:publish REPO=/" - return - fi - if [[ ! $REPO =~ ^[^/]+/[^/]+$ ]]; then - missing "the repository (owner/name)" \ - "REPO=$REPO is not /" \ - "remedy: task guest:inputs:publish REPO=/" - return - fi - # Both refs are computed here, from the same sources ci.yml computes them from — the - # Taskfile's QEMU_VERSION and `task guest:kernel:tag`. Nothing in this script spells a - # version or a tag format out a second time: a path published here that the workflow - # does not resolve is indistinguishable, from the workflow's side, from nothing having - # been published. - qemu_image="$REGISTRY/$REPO/qemu:$QEMU_VERSION" - kernel_image="$REGISTRY/$REPO/guest-kernel:$($TASK_EXE guest:kernel:tag)" - ok "repository $REPO" - ok "target $qemu_image" - ok "target $kernel_image" -} - -check_gh() { - command -v "$GH" >/dev/null 2>&1 || { - missing "the gh CLI" \ - "needed to dispatch the QEMU workflow and to authenticate the registry push" \ - "remedy: install GitHub CLI (https://cli.github.com)" - return - } - # One request answers three questions: is the token valid, whose is it, and what may it - # do. The scopes come from the response header rather than from `gh auth status`'s - # prose, because the prose is for a human and has changed shape between releases. - local head - head=$($GH api -i user 2>/dev/null) || { - missing "a working GitHub login" \ - "gh could not authenticate (no token, expired token, or no network)" \ - "remedy: gh auth login" - return - } - local scopes - scopes=$(grep -i '^x-oauth-scopes:' <<<"$head" | tr -d '\r' | cut -d' ' -f2- || true) - ok "gh authenticated as $($GH api user -q .login 2>/dev/null || echo '?')" - # write:packages is what publishing a container to ghcr.io needs, and this script logs - # Docker in with gh's token so the human never types a `docker login` — which means the - # scope is this script's precondition and not a note in a document. A token without it - # fails at the push, after the kernel has been built into an image, with a 403 that - # reads like a permissions problem with the repository. - case ",${scopes// /}," in - *,write:packages,*) ok "token carries write:packages" ;; - *) missing "write:packages on the gh token" \ - "scopes are: ${scopes:-}" \ - "remedy: gh auth refresh -h github.com -s write:packages" ;; - esac -} - -check_repo_write() { - [ -n "$REPO" ] || { - skipped "write access — no repository resolved, so there is nothing to ask about" - return 0 - } - command -v "$GH" >/dev/null 2>&1 || { - skipped "write access to $REPO — gh is what would answer it" - return 0 - } - local push - push=$($GH api "repos/$REPO" -q .permissions.push 2>/dev/null || true) - case "$push" in - true) ok "write access to $REPO (workflow_dispatch is allowed)" ;; - false) missing "write access to $REPO" \ - "workflow_dispatch on qemu.yml needs it, and so does publishing packages under" \ - "this repository's namespace." \ - "remedy: ask for write access, or publish from an account that has it" ;; - *) missing "write access to $REPO" \ - "the repository did not answer — it may not exist on GitHub yet, or the token" \ - "cannot see it. gh api repos/$REPO returned nothing usable." \ - "remedy: create the repository, or set REPO=/ to the right one" ;; - esac -} - -# The QEMU half is a dispatch, and a dispatch resolves the workflow on the *default -# branch*, not in this checkout. So a qemu.yml that exists here and has not been pushed — -# or that has been disabled in the Actions UI — fails at `gh workflow run` with a 404 that -# reads as though the command were wrong. This turns it into a named precondition. -check_qemu_workflow() { - [ -n "$REPO" ] || { - skipped "the QEMU workflow — no repository resolved" - return 0 - } - command -v "$GH" >/dev/null 2>&1 || { - skipped "the QEMU workflow on $REPO — gh is what would answer it" - return 0 - } - local state - state=$($GH api "repos/$REPO/actions/workflows/qemu.yml" -q .state 2>/dev/null || true) - case "$state" in - active) ok "qemu.yml is dispatchable on $REPO" ;; - "") missing "qemu.yml on $REPO's default branch" \ - "a workflow_dispatch resolves the workflow on the default branch, not in this" \ - "checkout, so an unpushed .github/workflows/qemu.yml cannot be dispatched." \ - "remedy: push .github/workflows/qemu.yml to the default branch" ;; - *) missing "qemu.yml on $REPO is $state, not active" \ - "a disabled workflow accepts no dispatch." \ - "remedy: re-enable it under the repository's Actions tab" ;; - esac -} - -check_docker() { - command -v "$DOCKER" >/dev/null 2>&1 || { - missing "docker" \ - "the kernel is published as a one-file image built with buildx" \ - "remedy: install Docker" - return - } - if $DOCKER info >/dev/null 2>&1; then - ok "docker daemon reachable" - else - missing "a reachable docker daemon" \ - "docker is installed but 'docker info' failed" \ - "remedy: start the daemon (systemctl start docker), or fix DOCKER_HOST" - fi -} - -# The one precondition that genuinely needs a human, and the reason this task cannot be a -# workflow: storage does not build a kernel (ADR-0021). It has to come off a machine that -# has spinbox's already-built artefact, either at the canonical path (a previous `task -# guest:kernel:fetch`) or in a sibling checkout. -check_kernel() { - if [ -z "$KERNEL_SHA256" ]; then - missing "GUEST_KERNEL_SHA256" \ - "there is no pin, so nothing could say whether the right kernel was published" \ - "remedy: set GUEST_KERNEL_SHA256 in Taskfile.yml (task guest:kernel:pin -- )" - return - fi - local candidate - for candidate in "$KERNEL" "$SPINBOX_KERNEL"; do - [ -n "$candidate" ] && [ -f "$candidate" ] || continue - if [ "$(sha256 "$candidate")" = "$KERNEL_SHA256" ]; then - kernel_source=$candidate - ok "the pinned kernel ${KERNEL_VERSION:+$KERNEL_VERSION }at $candidate" - return - fi - # A file that is there and is the wrong one is worth saying out loud: it is how a - # lane certifies one kernel and publishes another. - printf ' %s is %s, not the pinned %s\n' "$candidate" "$(sha256 "$candidate")" "$KERNEL_SHA256" - done - missing "the pinned guest kernel (sha256 $KERNEL_SHA256)" \ - "looked at $KERNEL and ${SPINBOX_KERNEL:-}." \ - "storage does not build a kernel (ADR-0021): it has to come from a machine with a" \ - "sibling spinbox checkout that has built the pinned one." \ - "remedy: cd ../spinbox && task build:kernel # then re-run this task" \ - " (it may exit non-zero and still emit the artefact — cache permissions)" -} - -cmd_check() { - echo "preconditions for publishing the guest-backed lanes' inputs:" - check_repository - check_gh - check_repo_write - check_qemu_workflow - check_docker - check_kernel - echo - if [ "$failures" -ne 0 ]; then - echo "$failures precondition(s) missing — nothing was published." >&2 - return 1 - fi - echo "OK: everything publishing needs is present. Run: task guest:inputs:publish" -} - -# --- publish --------------------------------------------------------------------------- - -# published — is it already in the registry? Used to decide what to do and, at the -# end, to prove what was done. `docker manifest inspect` is what ci.yml's preflight uses, -# so the postcondition here is the same question the workflow asks. -published() { $DOCKER manifest inspect "$1" >/dev/null 2>&1; } - -# readable_by_ci — the question `published` cannot answer, and the reason a -# whole afternoon went into a package that existed. -# -# `docker manifest inspect` here runs as the human, with a personal token, so it succeeds -# for a package CI cannot read at all. From the workflow's side a private package linked -# to no repository is indistinguishable from one that was never pushed — same message, -# "not published". From this side they are perfectly distinguishable, so this is where the -# distinction belongs. -# -# **The linking claim in this file and in the Taskfile was too strong.** Both pushes carry -# org.opencontainers.image.source and that label is what links a package — *when Actions -# is the pusher*. Pushed from a laptop with a personal token, ghcr creates the package -# owned by the pusher, unlinked, and the label does not link it afterwards. Observed: -# storage/qemu, pushed by the QEMU workflow, came out linked; storage/guest-kernel, pushed -# from a developer machine minutes earlier with the same label, did not. -readable_by_ci() { - local pkg="$1" owner="${REPO%%/*}" name repo vis - pkg="${pkg#"$REGISTRY"/"$owner"/}"; pkg="${pkg%%:*}" - name="${REPO#*/}/${pkg#*/}" - # The org endpoint answers for a package the workflow cannot see; that asymmetry is - # the whole point. A user-owned repository answers on /user/packages instead. - local path="/orgs/$owner/packages/container/${name//\//%2F}" - vis=$($GH api "$path" -q '.visibility // "unknown"' 2>/dev/null) || return 0 - repo=$($GH api "$path" -q '.repository.full_name // ""' 2>/dev/null) || repo="" - if [ "$vis" = "public" ] || [ -n "$repo" ]; then - return 0 - fi - cat <&2 <&2 - exit 2 - ;; -esac diff --git a/hack/qemu-devices.mak b/hack/qemu-devices.mak deleted file mode 100644 index 84a09d5..0000000 --- a/hack/qemu-devices.mak +++ /dev/null @@ -1,133 +0,0 @@ -# The device set this project's QEMU is built with, passed as `--with-devices-x86_64=spin`. -# -# A file and not a `sed` over the source tree, because the source lives in a BuildKit cache -# mount keyed by version alone: an edit made there survives every later build, including -# ones meant to undo it, and nothing would show that the tree being compiled is not the -# tree upstream shipped. This is read on every build and reviewed like anything else. -# -# **One device per function, and the function's most compatible device.** For a Linux guest -# that is virtio, without qualification: virtio-blk, virtio-net and the rest have been in -# the mainline kernel since 2.6.25 (2008) and are in every distribution's kernel. e1000 and -# rtl8139 are "more compatible" only for a guest with no virtio drivers — a Windows install -# without them, or something pre-2008 — which is not what this fleet boots. Every extra -# model is a device a guest could be given by accident, a driver its kernel has to carry, -# and in the case of a NIC an option ROM QEMU refuses to start without. -# -# Only symbols upstream marks optional are touched, which at this level means the ones -# carrying `default y if PCI_DEVICES` (or ISA/PCIE/USB) in hw/*/Kconfig — nothing `select`s -# them, so each can go on its own without touching PCI_DEVICES itself. Turning off a symbol -# upstream does not offer produces a link failure twenty minutes in, which is what -# `CONFIG_CXL=n` did on 11.0.2 (ACPI and the PCI expander bridge still reference it). -# -# Every change here is checked by running scripts/minikconf.py against this file before a -# build is started; it names the symbol and says whether it is undefined or contradicted. -include ../i386-softmmu/default.mak - -# --- boards --------------------------------------------------------------------------- -# Q35 is what every lane boots and MICROVM is kept for the day one wants it (it needs FDT, -# which is why fdt is not disabled either). The rest are a 1996 PC and a machine type for -# AWS enclaves. -CONFIG_ISAPC=n -CONFIG_I440FX=n -CONFIG_NITRO_ENCLAVE=n - -# --- network: one card ------------------------------------------------------------------ -# virtio-net, and nothing else. This repository gives its guests no network at all, but the -# binary is the one a host runs for real (ADR-0021 — spin's runner launches QEMU) and a VM -# there gets a NIC. An emulated e1000 in a microVM is a performance bug with a driver, and -# each of these carries an option ROM: 1.6 MB of pc-bios for cards nobody would attach on -# purpose, in an image where firmware was already the largest thing. -CONFIG_E1000_PCI=n -CONFIG_E1000E_PCI_EXPRESS=n -CONFIG_IGB_PCI_EXPRESS=n -CONFIG_EEPRO100_PCI=n -CONFIG_NE2000_PCI=n -CONFIG_NE2000_ISA=n -CONFIG_PCNET_PCI=n -CONFIG_RTL8139_PCI=n -CONFIG_TULIP=n -CONFIG_VMXNET3_PCI=n -CONFIG_ROCKER=n -CONFIG_USB_NETWORK=n -CONFIG_CAN_SJA1000=n -CONFIG_CAN_PCI=n -CONFIG_CAN_CTUCANFD=n -CONFIG_CAN_CTUCANFD_PCI=n - -# --- storage: one disk -------------------------------------------------------------------- -# virtio-blk is how a volume reaches a guest here; virtio-scsi stays with it for a caller -# that wants more disks than PCI slots. The SCSI HBAs below emulate 1990s hardware, and NVMe -# is a storage device this system exists to replace. AHCI is not in the list because Q35 -# selects it (hw/i386/Kconfig) — the ICH9 southbridge has SATA whether or not anything -# plugs into it. -CONFIG_NVME_PCI=n -CONFIG_LSI_SCSI_PCI=n -CONFIG_MPTSAS_SCSI_PCI=n -CONFIG_MEGASAS_SCSI_PCI=n -CONFIG_VMW_PVSCSI_SCSI_PCI=n -CONFIG_ESP_PCI=n -CONFIG_SDHCI_PCI=n -CONFIG_UFS_PCI=n - -# --- display: one adapter ----------------------------------------------------------------- -# The standard VGA, whose vgabios-stdvga.bin is shipped with the firmware, so a guest with -# no serial console still has somewhere to print. Cirrus, VMware's SVGA, Bochs, ATI and -# Apple's paravirtual adapter are five more answers to a question a headless VM does not ask. -CONFIG_VGA_CIRRUS=n -CONFIG_VMWARE_VGA=n -CONFIG_BOCHS_DISPLAY=n -CONFIG_ATI_VGA=n -CONFIG_MAC_PVG_PCI=n - -# --- no sound, no USB, no serial over PCI ------------------------------------------------- -# The console is the ISA 16550 the kernel prints to on ttyS0, which is what -serial gives -# it. Audio has no backend compiled in at all (the configure flags disable every driver), -# so these are device models with nowhere to play; USB is a bus this fleet's guests have no -# devices on, and `usb=off` is already on every machine line here. -CONFIG_ES1370=n -CONFIG_AC97=n -CONFIG_HDA=n -CONFIG_SERIAL_PCI=n -CONFIG_SERIAL_PCI_MULTI=n -CONFIG_USB_UHCI=n -CONFIG_USB_OHCI_PCI=n -CONFIG_USB_EHCI_PCI=n -CONFIG_USB_XHCI_PCI=n -CONFIG_USB_XHCI_NEC=n - -# --- legacy PC and the rest --------------------------------------------------------------- -# A Mac's SMC, the qtest devices, Spice's QXL, IPMI over four buses, the Hyper-V -# enlightenments a Linux guest does not use, a watchdog, an industrial I/O carrier and the -# inter-VM shared memory device. -CONFIG_APPLESMC=n -CONFIG_TEST_DEVICES=n -CONFIG_QXL=n -CONFIG_ISA_DEBUG=n -CONFIG_ISA_IPMI_BT=n -CONFIG_ISA_IPMI_KCS=n -CONFIG_PCI_IPMI_BT=n -CONFIG_PCI_IPMI_KCS=n -CONFIG_IPMI_SSIF=n -CONFIG_HYPERV=n -CONFIG_WDT_IB6300ESB=n -CONFIG_TPCI200=n -CONFIG_IVSHMEM_DEVICE=n - -# Two that upstream's default.mak offers and 11.1.1 refuses, each checked by running -# scripts/minikconf.py against this file rather than by reading the list: -# CONFIG_SGA=n — "undefined symbol SGA". The serial graphics adapter is gone from the -# Kconfig tree; the line survives in upstream's default.mak alone. -# CONFIG_FDC=n — "contradiction between clauses when setting FDC". Q35's ICH9 brings an -# ISA_SUPERIO, which selects FDC_ISA, which selects FDC. A floppy -# controller is not optional while the board that needs it is kept. -# -# Deliberately left on, each for a reason: -# PCI_DEVICES — q35 still references controllers this would take with it, and every -# model above is reachable one at a time without it. -# PCI_BRIDGE, PCIE_PORT, XIO3130, IOH3420, I82801B11 — a q35's root ports; without them -# a device cannot be plugged into anything. -# VIRTIO_PCI, VIRTIO_NET, VIRTIO_BLK, VIRTIO_SCSI, VIRTIO_BALLOON, VIRTIO_RNG — the set -# a guest here is actually given. -# VTD, AMD_IOMMU — VFIO on q35 needs them; there is no passthrough here yet. -# HPET, PVPANIC — cheap, and pvpanic is how a guest reports a panic. -# SEV, TDX, SGX — confidential computing is a product decision, not debloat. diff --git a/hack/spin-machine b/hack/spin-machine new file mode 100755 index 0000000..a866d87 --- /dev/null +++ b/hack/spin-machine @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# +# Put a spin-machine release under _output/, from wherever one is available. +# +# This repository builds no QEMU and no kernel. It used to do both — a Dockerfile.qemu +# modelled on another project's, a device list that was byte-for-byte that project's, and a +# Dockerfile.guest-kernel that built nothing at all and existed only to move somebody +# else's kernel through a registry and back out. All three are gone: QEMU, the guest kernel +# and the firmware are one versioned artefact, and this is the only path by which one +# arrives. +# +# Two sources, in order: +# +# 1. a sibling spin-machine checkout that has already built one. No network. +# 2. the published release for the pinned version, fetched with `gh`. +# +# Pinned by version and by content, for the same reason the kernel used to be pinned by +# hash here: "which machine did this lane certify?" cannot be answerable only as "whatever +# was published that day". +set -euo pipefail + +VERSION="${SPIN_MACHINE_VERSION:?SPIN_MACHINE_VERSION must be set}" +SHA256="${SPIN_MACHINE_SHA256:-}" +REPO="${SPIN_MACHINE_REPO:-spin-stack/spin-machine}" +SIBLING="${SPIN_MACHINE_DIR:-../spin-machine}" +OUTPUT_DIR="${OUTPUT_DIR:-_output}" +CACHE="${SPIN_MACHINE_CACHE:-${OUTPUT_DIR}/machine-cache}" + +ARCH="$(uname -m)" +NAME="spin-machine-${VERSION}-linux-${ARCH}" + +# What this repository takes, and where it puts it. +# +# Both qemu-system binaries: the Agent runs on hosts with /dev/kvm and CI does not, and the +# split between a binary that refuses to emulate and one that can is the whole reason there +# are two. qemu-img because every qcow2 chain is created and inspected by running it — v6 +# forbids a parser of our own. The kernel goes to _output/guest/vmlinux, which is the one +# path the guest lane reads. +# +# Not taken: the base image. Nothing here boots from it. +place() { + local root="$1" missing=0 + copy() { + local src="${root}/$1" dest="${OUTPUT_DIR}/$2" + if [ ! -f "${src}" ]; then + echo " missing from the machine: $1" >&2 + missing=1 + return 0 + fi + mkdir -p "$(dirname "${dest}")" + rm -f "${dest}" 2>/dev/null || true + install -m "$([ -x "${src}" ] && echo 0755 || echo 0644)" "${src}" "${dest}" + } + + copy bin/qemu-system-x86_64 bin/qemu-system-x86_64 + copy bin/qemu-system-x86_64-tcg bin/qemu-system-x86_64-tcg + copy bin/qemu-img bin/qemu-img + copy kernel/vmlinux guest/vmlinux + for f in bios.bin bios-256k.bin pvh.bin kvmvapic.bin efi-virtio.rom; do + copy "qemu/${f}" "share/spin-stack/qemu/${f}" + done + + [ "${missing}" -eq 0 ] || { + echo "👹 that release does not contain a whole machine" >&2 + exit 1 + } +} + +# have_sibling only decides; it does no work. +# +# The decision and the work are separate because writing it as one `||` expression — try +# the sibling, fall back to the release — turns off errexit for everything inside the +# left-hand function, so a copy that failed halfway reports success. +have_sibling() { + local out="${SIBLING}/_output" + [ -f "${out}/bin/qemu-system-x86_64" ] && [ -f "${out}/vmlinux" ] +} + +from_sibling() { + local out="${SIBLING}/_output" staged + echo "Using the machine built in ${SIBLING}" + staged="$(mktemp -d)" + trap 'rm -rf "${staged}"' RETURN + mkdir -p "${staged}/bin" "${staged}/kernel" "${staged}/qemu" + cp "${out}/bin/qemu-system-x86_64" "${out}/bin/qemu-system-x86_64-tcg" \ + "${out}/bin/qemu-img" "${staged}/bin/" + cp "${out}/vmlinux" "${staged}/kernel/vmlinux" + cp "${out}"/share/spin-stack/qemu/* "${staged}/qemu/" + place "${staged}" +} + +from_release() { + local tarball="${CACHE}/${NAME}.tar.gz" unpacked + mkdir -p "${CACHE}" + + if [ ! -f "${tarball}" ]; then + # curl and not `gh`, now that spin-machine is public: a release asset has a plain URL + # and needs no authentication at all. `gh` needs a token even to read a public + # repository, which meant every job that reaches this — not only the one somebody + # remembered — had to carry GH_TOKEN, and the ones that did not failed with gh's + # advice about setting it rather than with anything about the machine. + # + # --fail so a 404 is an error instead of a tarball full of HTML; --location because + # the asset redirects to object storage; --retry because this is the one step here + # that depends on somebody else's network. + echo "Fetching spin-machine ${VERSION}" + curl --fail --location --show-error --silent \ + --retry 3 --retry-delay 2 --retry-all-errors --connect-timeout 30 \ + --output "${tarball}.part" \ + "https://github.com/${REPO}/releases/download/${VERSION}/${NAME}.tar.gz" || { + echo "👹 could not fetch ${NAME}.tar.gz from ${REPO} ${VERSION}" >&2 + rm -f "${tarball}.part" + exit 1 + } + # Renamed only once it is whole: an interrupted download left in place is a cached + # tarball that every later run would reuse and every later run would reject. + mv "${tarball}.part" "${tarball}" + fi + + # Checked on every run and not only after a download: the tarball is cached, so the copy + # being unpacked today may have arrived weeks ago. + if [ -n "${SHA256}" ]; then + echo "${SHA256} ${tarball}" | sha256sum -c - >/dev/null || { + echo "👹 ${tarball} is not the release SPIN_MACHINE_SHA256 pins." >&2 + exit 1 + } + else + echo " warning: SPIN_MACHINE_SHA256 is unset, so this release is unverified" >&2 + fi + + unpacked="$(mktemp -d)" + trap 'rm -rf "${unpacked}"' RETURN + tar -xzf "${tarball}" -C "${unpacked}" + place "${unpacked}/${NAME}/usr/share/spin-stack" +} + +if have_sibling; then + from_sibling +else + from_release +fi + +# Run, not just present. The binaries are statically linked, so a tree extracted here works +# here — which was not true when this repository built its own QEMU and had to carry the +# build image's loader and libraries beside it in order to run anything at all. +for b in qemu-system-x86_64 qemu-system-x86_64-tcg qemu-img; do + "${OUTPUT_DIR}/bin/${b}" --version >/dev/null || { + echo "👹 ${OUTPUT_DIR}/bin/${b} does not run on this host" >&2 + exit 1 + } +done +! "${OUTPUT_DIR}/bin/qemu-system-x86_64" -accel help | grep -qx tcg || { + echo "👹 qemu-system-x86_64 can emulate: a host with no /dev/kvm would run at a tenth of the speed" >&2 + exit 1 +} +"${OUTPUT_DIR}/bin/qemu-system-x86_64-tcg" -accel help | grep -qx tcg || { + echo "👹 qemu-system-x86_64-tcg cannot emulate, so a runner without /dev/kvm has nothing to boot with" >&2 + exit 1 +} + +echo "✓ machine ${VERSION}: qemu $("${OUTPUT_DIR}/bin/qemu-img" --version | head -1 | awk '{print $3}'), kernel $(stat -c%s "${OUTPUT_DIR}/guest/vmlinux") bytes" diff --git a/hack/stage1-demo.sh b/hack/stage1-demo.sh index e3f5441..dde6459 100755 --- a/hack/stage1-demo.sh +++ b/hack/stage1-demo.sh @@ -56,7 +56,7 @@ mkfifo "$DIR/ctl" # started yet. Read-write never blocks, and it keeps a writer open so the guest's # console does not see EOF the moment the stop word has been sent. exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -95,7 +95,7 @@ wait "$GUEST1" 2>/dev/null || true grep -m1 "GUESTINIT-PASS" "$DIR/logs/guest1.log" say "7. a second boot, reading only" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/hack/stage2-demo.sh b/hack/stage2-demo.sh index c6362ab..52b6fe0 100755 --- a/hack/stage2-demo.sh +++ b/hack/stage2-demo.sh @@ -63,7 +63,7 @@ mkfifo "$DIR/ctl" # Read-write, so opening does not block on a QEMU that has not started yet and the # guest's console does not see EOF the moment the stop word has been sent. exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -137,7 +137,7 @@ done echo " qemu-img check: all $LAYERS layers sound" say "8. a second boot, reading only, through the whole chain" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/hack/stage3-demo.sh b/hack/stage3-demo.sh index e88307b..0b99594 100755 --- a/hack/stage3-demo.sh +++ b/hack/stage3-demo.sh @@ -43,7 +43,7 @@ echo "volume $VOLUME, first layer $(basename "$FIRST")" say "3. a Linux guest boots off it and starts writing" mkfifo "$DIR/ctl" exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -134,7 +134,7 @@ grep -m1 "GUESTINIT-PASS" "$DIR/logs/guest1.log" say "9. a second boot, reading only" TIP=$(cat "$POINTER") -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/hack/stage4-demo.sh b/hack/stage4-demo.sh index 8045efd..fa2f5f5 100755 --- a/hack/stage4-demo.sh +++ b/hack/stage4-demo.sh @@ -44,7 +44,7 @@ FIRST=$(cat "$POINTER") mkfifo "$DIR/ctl" exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -139,7 +139,7 @@ done echo " qemu-img check: every downloaded layer is sound" say "9. a guest reads back what a guest on a host that no longer exists wrote" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/hack/stage5-demo.sh b/hack/stage5-demo.sh index 00126da..5609730 100755 --- a/hack/stage5-demo.sh +++ b/hack/stage5-demo.sh @@ -42,7 +42,7 @@ echo "volume $VOLUME, first layer $(basename "$FIRST")" say "3. a Linux guest boots off it and starts writing" mkfifo "$DIR/ctl" exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/hack/stage6-demo.sh b/hack/stage6-demo.sh index daffffb..cefbd5b 100755 --- a/hack/stage6-demo.sh +++ b/hack/stage6-demo.sh @@ -45,7 +45,7 @@ echo "parent volume $PARENT" mkfifo "$DIR/ctl" exec 9<>"$DIR/ctl" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -110,7 +110,7 @@ say "7. a second guest boots the clone and reads the FIRST guest's pattern" # reports and powers itself off, so waiting for the process is waiting for the answer — and # the socket stays free for the guest in step 8, which needs the Agent to reach it. CSOCK=$(grep "volume_id=$CLONE" "$DIR/logs/agent1.log" | grep -m1 -o 'qmp_socket=[^ ]*' | cut -d= -f2) -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -130,7 +130,7 @@ say "8. a guest on the clone, writing a slot of its own so the clone has a commi CIMAGE=$(cat "$CPOINTER") mkfifo "$DIR/ctl2" exec 8<>"$DIR/ctl2" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ @@ -180,7 +180,7 @@ test "$GDEPTH" -ge 3 || die "the grandchild's chain is $GDEPTH layer(s) deep: a echo " its chain is $GDEPTH layers deep" say "12. a guest boots the clone of the clone and reads the ORIGINAL guest's slot" -"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -monitor none -no-reboot \ +"$QEMU" -machine "q35,accel=$ACCEL" -m 512 -smp 1 -display none -vga none -monitor none -no-reboot \ -net none \ -L "$OUT/share/spin-stack/qemu" \ -kernel "$KERNEL" -initrd "$INITRAMFS" \ diff --git a/integration/e2e/fixture_test.go b/integration/e2e/fixture_test.go index 96f0d2b..859dfa3 100644 --- a/integration/e2e/fixture_test.go +++ b/integration/e2e/fixture_test.go @@ -69,7 +69,7 @@ func start(t *testing.T) *deployment { agentBin := testinfra.Binary(t, "volume-agent") // The Agent creates and inspects every qcow2 chain by running the pinned qemu-img // (v6 §7 forbids a parser of our own), and refuses to start without it — so this - // lane needs the artefact `task qemu:build` produces. testinfra.Binary fails naming + // lane needs the artefacts `task machine` fetches. testinfra.Binary fails naming // that task rather than skipping: a lane that quietly declined to start the Agent // would be the gate reporting success for work it did not do. qemuImg := testinfra.Binary(t, "qemu-img") diff --git a/taskfiles/guest.yml b/taskfiles/guest.yml index 89cfd58..1c97901 100644 --- a/taskfiles/guest.yml +++ b/taskfiles/guest.yml @@ -1,14 +1,17 @@ version: "3" -# The guest lane's two artefacts: the initramfs, which is built here from a static Go -# /init, and the kernel, which is not — it is spinbox's output, consumed and never -# compiled here. `guest:kernel:fetch` is the one path a kernel arrives by; nothing else -# resolves one. +# The guest lane's one artefact: the initramfs, built here from a static Go /init. +# +# The kernel is not here and never was built here. It arrives with the rest of the machine +# — see taskfiles/machine.yml — and this file used to carry five targets and two scripts +# whose only job was moving somebody else's kernel through a registry so that a machine +# with no sibling checkout could find one. That is what having the machine in one place +# replaced. tasks: guest:build: - desc: Build the initramfs the guest lane boots (a static Go /init). The kernel comes from `task guest:kernel:fetch`. + desc: Build the initramfs the guest lane boots (a static Go /init). The kernel comes with the machine (`task machine`). sources: - integration/guestinit/**/*.go - hack/build-guest-initramfs.sh @@ -17,23 +20,9 @@ tasks: cmds: - OUT_DIR={{.OUTPUT_DIR}}/guest bash hack/build-guest-initramfs.sh - guest:kernel:fetch: - desc: >- - Put the pinned guest kernel at _output/guest/vmlinux, from a sibling spinbox - checkout or from the mirrored image (ADR-0022). Idempotent; storage never builds one. - env: &guest_kernel_env - KERNEL: '{{.GUEST_KERNEL}}' - KERNEL_SHA256: '{{.GUEST_KERNEL_SHA256}}' - KERNEL_VERSION: '{{.GUEST_KERNEL_VERSION}}' - SPINBOX_KERNEL: '{{.SPINBOX_KERNEL}}' - KERNEL_IMAGE: '{{.GUEST_KERNEL_IMAGE}}' - cmds: - - bash hack/guest-kernel.sh fetch - guest:verify: - desc: Assert the guest lane's inputs — the initramfs we build, and that the kernel is the one we pinned. - deps: [guest:build, guest:kernel:fetch] - env: *guest_kernel_env + desc: Assert the guest lane's inputs — the initramfs we build, and the kernel that comes with the machine. + deps: [guest:build, machine] cmds: # The artefact is opened, not stat'ed: `test -f` says OK for a zero-byte file, a # half-written one and an archive with no /init, and `guest:build` fingerprints its @@ -56,88 +45,14 @@ tasks: echo "$img contains no ./init — the kernel would mount it and panic with no PID 1; rebuild: task guest:build" >&2 exit 1; } echo "OK: $img — gzip cpio carrying ./init" - - bash hack/guest-kernel.sh verify - - guest:kernel:pin: - desc: "Print the sha256 of a kernel artefact, for bumping GUEST_KERNEL_SHA256. Usage: task guest:kernel:pin -- " - env: *guest_kernel_env - cmds: - - bash hack/guest-kernel.sh pin {{.CLI_ARGS}} - - guest:kernel:tag: - desc: >- - Print the content tag of the pinned guest kernel (sha256-) — the tag - `guest:kernel:push` publishes alongside the version. CI resolves the mirror from - this so no workflow spells the pin out a second time, and the tag is - content-addressed for the same reason ADR-0022 pins by hash: a tag that can be - moved is a kernel that can be swapped under a lane that already certified it. - silent: true - cmds: - - echo "sha256-{{.GUEST_KERNEL_SHA256}}" - - guest:kernel:push: - desc: >- - Mirror the pinned kernel into a registry so machines with no spinbox checkout can - run the lane (ADR-0022). Usage: - task guest:kernel:push GUEST_KERNEL_IMAGE=ghcr.io///guest-kernel - Requires a `docker login` to that registry. - deps: [guest:kernel:fetch] - cmds: + # The kernel is checked by opening it too, and for the same reason as the initramfs: + # what makes it bootable here is not its size. QEMU enters this ELF through pvh.bin + # using the Xen notes in its program headers, and a kernel without them fails at + # start-up as a rom-open error that reads like a backend bug. - | set -euo pipefail - test -n "{{.GUEST_KERNEL_IMAGE}}" || { - echo "set GUEST_KERNEL_IMAGE=//guest-kernel"; exit 1; } - # A build context holding exactly the artefact: the image is one file on - # scratch, so its digest is a hash of the kernel and of nothing else. - ctx=$(mktemp -d); trap 'rm -rf "$ctx"' EXIT - cp {{.GUEST_KERNEL}} "$ctx/vmlinux" - # org.opencontainers.image.source links a ghcr.io package to a repository — and - # only when Actions is the pusher; see readable_by_ci in - # hack/publish-guest-inputs.sh. Derived from the target path rather than - # configured: the only namespace CI resolves is ghcr.io///... - # Same derivation in qemu:push. - src="" - case "{{.GUEST_KERNEL_IMAGE}}" in - ghcr.io/*/*/*) slug={{.GUEST_KERNEL_IMAGE}}; slug=${slug#ghcr.io/}; slug=${slug%/*} - src="--label org.opencontainers.image.source=https://github.com/$slug" ;; - esac - docker buildx build \ - --file Dockerfile.guest-kernel \ - --target package \ - --platform linux/amd64 \ - $src \ - --tag {{.GUEST_KERNEL_IMAGE}}:{{.GUEST_KERNEL_VERSION}} \ - --tag {{.GUEST_KERNEL_IMAGE}}:sha256-{{.GUEST_KERNEL_SHA256}} \ - --push \ - "$ctx" - echo - echo "pull it back with the digest, not the tag:" - echo " task guest:kernel:fetch GUEST_KERNEL_IMAGE={{.GUEST_KERNEL_IMAGE}}@\$(docker buildx imagetools inspect --format '{{`{{.Manifest.Digest}}`}}' {{.GUEST_KERNEL_IMAGE}}:{{.GUEST_KERNEL_VERSION}})" - - # The two artefacts CI's guest job needs and this repository does not build: the pinned - # QEMU and the pinned kernel. `guest:inputs:check` names every input publishing them - # needs; `guest:inputs:publish` performs the procedure rather than documenting it. - - guest:inputs:check: - desc: >- - Name every input publishing the guest artefacts needs (gh, a token with - write:packages, docker, the pinned kernel), and say which are missing. No side effects. - env: &publish_guest_env - REPO: '{{.REPO | default ""}}' - KERNEL: '{{.GUEST_KERNEL}}' - KERNEL_SHA256: '{{.GUEST_KERNEL_SHA256}}' - KERNEL_VERSION: '{{.GUEST_KERNEL_VERSION}}' - SPINBOX_KERNEL: '{{.SPINBOX_KERNEL}}' - QEMU_VERSION: '{{.QEMU_VERSION}}' - TASK_EXE: '{{.TASK_EXE}}' - cmds: - - bash hack/publish-guest-inputs.sh check - - guest:inputs:publish: - desc: >- - Publish both guest artefacts — the pinned QEMU image (by dispatching the QEMU - workflow) and the pinned guest kernel — then verify both resolve. Idempotent. - Usage: task guest:inputs:publish REPO=/ - env: *publish_guest_env - cmds: - - bash hack/publish-guest-inputs.sh publish + k={{.GUEST_KERNEL}} + test -f "$k" || { echo "no kernel at $k — run: task machine" >&2; exit 1; } + readelf -n "$k" 2>/dev/null | grep -qi xen || { + echo "$k carries no Xen PVH notes — QEMU has no entry point into it" >&2; exit 1; } + echo "OK: $k — PVH notes present" diff --git a/taskfiles/machine.yml b/taskfiles/machine.yml new file mode 100644 index 0000000..f93ad0e --- /dev/null +++ b/taskfiles/machine.yml @@ -0,0 +1,40 @@ +version: "3" + +# The machine this repository runs guests on: QEMU, the guest kernel, the firmware. +# +# None of it is built here. It used to be — a Dockerfile.qemu modelled on another +# project's, a device list that was byte-for-byte that project's, and a +# Dockerfile.guest-kernel that built nothing and existed only to move somebody else's +# kernel through a registry and back out (ADR-0022). All of that has one home now, and +# `task machine` is the only path by which a machine arrives. + +tasks: + + machine: + desc: >- + Put the pinned machine under _output/ — both qemu-system binaries, qemu-img and the + guest kernel — from a sibling spin-machine checkout or its published release. + Idempotent. + status: + # "it runs", not "the file is there". Those came apart once already, when the + # binaries were dynamically linked and an extracted one died at exit 127 in whatever + # lane needed it first. They are static now, and this still asks. + - "{{.OUTPUT_DIR}}/bin/qemu-img --version" + - "{{.OUTPUT_DIR}}/bin/qemu-system-x86_64 --version" + - "{{.OUTPUT_DIR}}/bin/qemu-system-x86_64-tcg --version" + - "test -f {{.OUTPUT_DIR}}/guest/vmlinux" + - "test -f {{.OUTPUT_DIR}}/share/spin-stack/qemu/pvh.bin" + cmds: + - | + set -euo pipefail + SPIN_MACHINE_VERSION="{{.SPIN_MACHINE_VERSION}}" \ + SPIN_MACHINE_SHA256="{{.SPIN_MACHINE_SHA256}}" \ + SPIN_MACHINE_DIR="{{.SPIN_MACHINE_DIR}}" \ + OUTPUT_DIR="{{.OUTPUT_DIR}}" \ + bash hack/spin-machine + + machine:version: + desc: Print the pinned machine version (the single source of truth for CI). + silent: true + cmds: + - echo "{{.SPIN_MACHINE_VERSION}}" diff --git a/taskfiles/qemu.yml b/taskfiles/qemu.yml deleted file mode 100644 index 33e34c7..0000000 --- a/taskfiles/qemu.yml +++ /dev/null @@ -1,275 +0,0 @@ -version: "3" - -# The pinned QEMU. It is built from Dockerfile.qemu — once into _output/ for the local -# lanes, once as the runtime image CI's guest jobs pull — and verified in two halves, -# because the extracted binary links against the build image's libraries and does not run -# on a bare runner (ADR-0025). - -tasks: - - qemu:build: - desc: "Build the pinned QEMU into _output/: the system binary, qemu-img and the firmware the guest lane boots with." - cmds: - # Emptied first: `--output type=local` writes files and removes none, so a binary or - # a firmware blob this build no longer produces stays on disk, indistinguishable from - # one it does. That is how a 331 MB firmware tree and a qemu-storage-daemon outlived - # the builds that stopped shipping them. - - rm -rf {{.OUTPUT_DIR}}/bin/qemu-* {{.OUTPUT_DIR}}/share/spin-stack/qemu {{.OUTPUT_DIR}}/lib/qemu - - mkdir -p {{.OUTPUT_DIR}} - - | - docker buildx build \ - --file Dockerfile.qemu \ - --target extract \ - --platform linux/amd64 \ - --cache-from {{.QEMU_CACHE_FROM}} \ - --cache-to {{.QEMU_CACHE_TO}} \ - --build-arg QEMU_VERSION={{.QEMU_VERSION}} \ - --build-arg JOBS={{.QEMU_JOBS}} \ - --output type=local,dest={{.OUTPUT_DIR}} \ - . - - task: qemu:verify - - qemu:verify: - desc: Verify the built QEMU is the pinned version and exposes the devices we depend on. - cmds: - # Two halves: the files are checked where they were extracted, the behaviour where - # the binary can actually run. The extracted binary is dynamically linked against the - # build image's libraries, so it does not run on a bare runner (ADR-0025). - - | - set -euo pipefail - bin={{.OUTPUT_DIR}}/bin/qemu-system-x86_64 - test -x "$bin" || { echo "missing $bin — run: task qemu:build"; exit 1; } - # qemu-img is not a convenience: the Volume Agent refuses to start without - # -qemu-img, because every qcow2 chain is created and inspected by running it - # (v6 §7 forbids a parser of our own). Presence only — see qemu:verify:behaviour. - test -x {{.OUTPUT_DIR}}/bin/qemu-img || { - echo "missing {{.OUTPUT_DIR}}/bin/qemu-img — the Agent cannot create a chain without it; run: task qemu:build"; exit 1; } - # The CI binary. It is the one with TCG compiled in, and it exists so that the - # production one does not have to carry it: a host serving tenants must fail loudly - # with no /dev/kvm rather than emulate. A tree with only one of the two boots - # somewhere and not somewhere else, which is the confusion this checks for. - test -x {{.OUTPUT_DIR}}/bin/qemu-system-x86_64-tcg || { - echo "missing {{.OUTPUT_DIR}}/bin/qemu-system-x86_64-tcg — a runner without KVM has nothing to boot with; run: task qemu:build"; exit 1; } - test -f {{.OUTPUT_DIR}}/share/spin-stack/qemu/bios-256k.bin - # pvh.bin is how -kernel enters a PVH ELF kernel, which is the guest lane's - # only boot path. Its absence fails at run time as a rom-open error, which - # reads like a backend bug. - test -f {{.OUTPUT_DIR}}/share/spin-stack/qemu/pvh.bin || { - echo "missing pvh.bin — a direct kernel boot (-kernel) cannot work; rebuild with: task qemu:build"; exit 1; } - # The option ROM of the one NIC this build keeps. A device model without its ROM is - # a VM that dies at start-up, and no lane here attaches a NIC to find out. - test -f {{.OUTPUT_DIR}}/share/spin-stack/qemu/efi-virtio.rom || { - echo "missing efi-virtio.rom — a VM given a virtio-net-pci cannot start; rebuild with: task qemu:build"; exit 1; } - echo "OK: the extracted tree carries both qemu-system binaries, qemu-img and the firmware (incl. pvh)" - - task: qemu:verify:behaviour - - qemu:verify:behaviour: - desc: Run the pinned QEMU and assert its version and devices, wherever that QEMU is actually runnable. - cmds: - # Ask whichever QEMU this caller will execute. The extracted binary runs on a - # developer machine and inside CI's guest jobs (ADR-0025's runtime image, where there - # is no docker to ask about an image — building one unconditionally failed there with - # `"docker": executable file not found in $PATH`), and not on a bare runner building - # the artefact for others: `liburing.so.2: cannot open shared object file`. So run it - # if it runs and fall back to the image if it does not, printing which branch - # answered. - - | - set -euo pipefail - bin={{.OUTPUT_DIR}}/bin/qemu-system-x86_64 - # qemu-img is asked the same question in the same breath, because the Agent - # refuses to start without one and a tree that carries a qemu-system nobody can - # fault while its qemu-img is the wrong version is a lane that fails three tests - # away from the cause. - if out=$("$bin" --version 2>/dev/null && echo --- && "$bin" -device help 2>&1 && \ - {{.OUTPUT_DIR}}/bin/qemu-img --version); then - where="the binaries at {{.OUTPUT_DIR}}/bin" - else - img={{.QEMU_IMAGE}}:{{.QEMU_VERSION}} - echo "$bin cannot run here (its libraries are the build image's) — asking the runtime image instead" - task qemu:image - out=$(docker run --rm --entrypoint /bin/sh "$img" -c \ - '/usr/local/bin/qemu-system-x86_64 --version; echo ---; /usr/local/bin/qemu-system-x86_64 -device help 2>&1; /usr/local/bin/qemu-img --version') - where="$img" - fi - printf '%s' "$out" | grep -c 'version {{.QEMU_VERSION}}' | grep -qx 2 || { - echo "in $where, qemu-system and qemu-img are not both the pinned {{.QEMU_VERSION}}:" - printf '%s\n' "$out" | grep -i version; exit 1; } - printf '%s' "$out" | grep -q '{{.QEMU_VERSION}}' || { - echo "the QEMU in $where is not the pinned {{.QEMU_VERSION}}:"; printf '%s\n' "$out" | head -3; exit 1; } - # virtio-blk-pci, not vhost-user-blk-pci: the guest gets an ordinary virtio disk - # backed by a qcow2 QEMU opens itself. The vhost-user path is not built at all. - printf '%s' "$out" | grep -q 'virtio-blk-pci' || { - echo "the QEMU in $where has no virtio-blk-pci device, so no guest can be given a qcow2"; exit 1; } - # And the split, asserted on the tree the lanes actually execute rather than only - # inside the build: the binary a tenant's host runs must not be able to emulate, - # and CI's must. Backwards, this is a fleet running at a tenth of the speed or a - # guest lane that cannot boot — and neither is visible in a file listing. - tcgbin={{.OUTPUT_DIR}}/bin/qemu-system-x86_64-tcg - if "$bin" -accel help >/dev/null 2>&1 && [ -x "$tcgbin" ]; then - ! "$bin" -accel help | grep -qx tcg || { - echo "$bin has TCG compiled in: a host with no /dev/kvm would emulate instead of refusing"; exit 1; } - "$tcgbin" -accel help | grep -qx tcg || { - echo "$tcgbin cannot emulate, so a runner without /dev/kvm has nothing to boot with"; exit 1; } - echo "OK: the production binary refuses to emulate and the CI one can" - # And the one network card this build keeps actually starts, against the firmware - # tree the lanes pass with -L. This is the assertion the firmware trim needed and - # did not have: a NIC whose option ROM is not shipped fails at *start-up* with - # `failed to find romfile`, and nothing here attaches a NIC, so no lane would have - # noticed until spin's runner did. Emulated on purpose — loading a ROM has nothing - # to do with the accelerator, and this must run where there is no /dev/kvm. - fw={{.OUTPUT_DIR}}/share/spin-stack/qemu - nic=$(printf '{"execute":"qmp_capabilities"}\n{"execute":"quit"}\n' | \ - "$tcgbin" -machine q35,accel=tcg -m 64 -display none -serial none -monitor none \ - -L "$fw" -netdev socket,id=n0,listen=127.0.0.1:0 \ - -device virtio-net-pci,netdev=n0 -qmp stdio 2>&1 >/dev/null) || { - echo "a VM with the one NIC this build keeps does not start: $nic"; exit 1; } - echo "OK: a VM with virtio-net-pci starts against the shipped firmware" - fi - echo "OK: $(printf '%s' "$out" | head -1), virtio-blk-pci present — verified in $where" - - qemu:image: - desc: >- - Build the QEMU runtime image locally (--load, not pushed). Its caller is - `qemu:verify:behaviour`: the extracted binary links against the build image's - libraries and cannot run on a bare host (ADR-0025), so the behaviour half of the - verification runs QEMU inside this image instead. - cmds: - - | - docker buildx build \ - --file Dockerfile.qemu \ - --target runtime \ - --platform linux/amd64 \ - --cache-from {{.QEMU_CACHE_FROM}} \ - --build-arg QEMU_VERSION={{.QEMU_VERSION}} \ - --build-arg JOBS={{.QEMU_JOBS}} \ - --tag {{.QEMU_IMAGE}}:{{.QEMU_VERSION}} \ - --load \ - . - - qemu:version: - desc: Print the pinned QEMU version (the single source of truth for CI tags). - silent: true - cmds: - - echo "{{.QEMU_VERSION}}" - - qemu:push: - desc: >- - Build and push the QEMU runtime image. Usage: - task qemu:push QEMU_IMAGE=ghcr.io///qemu QEMU_IMAGE_EXTRA_TAG= - Requires a `docker login` to that registry. - vars: - QEMU_IMAGE_EXTRA_TAG: '{{.QEMU_IMAGE_EXTRA_TAG | default ""}}' - cmds: - - | - set -euo pipefail - # The package→repository link, exactly as at guest:kernel:push and for the same - # reason: an unlinked private package reads to CI as an unpublished one. - src="" - case "{{.QEMU_IMAGE}}" in - ghcr.io/*/*/*) slug={{.QEMU_IMAGE}}; slug=${slug#ghcr.io/}; slug=${slug%/*} - src="--label org.opencontainers.image.source=https://github.com/$slug" ;; - esac - docker buildx build \ - --file Dockerfile.qemu \ - --target runtime \ - --platform linux/amd64 \ - --cache-from {{.QEMU_CACHE_FROM}} \ - --cache-to {{.QEMU_CACHE_TO}} \ - --build-arg QEMU_VERSION={{.QEMU_VERSION}} \ - --build-arg JOBS={{.QEMU_JOBS}} \ - $src \ - --tag {{.QEMU_IMAGE}}:{{.QEMU_VERSION}} \ - {{if .QEMU_IMAGE_EXTRA_TAG}}--tag {{.QEMU_IMAGE}}:{{.QEMU_IMAGE_EXTRA_TAG}} \{{end}} - --push \ - . - - qemu:tools: - desc: >- - Put the pinned qemu-img, both qemu-system binaries and the firmware under _output/, extracting - them from the published runtime image when `task qemu:build` has not run here. - Idempotent. - status: - # "it runs", not "the file is there". The two came apart once already: an extracted - # binary whose libraries had not come with it is executable, and dies at exit 127 in - # whatever lane first needs it — which was three e2e tests, not this task. - - "{{.OUTPUT_DIR}}/bin/qemu-img --version" - - "{{.OUTPUT_DIR}}/bin/qemu-system-x86_64 --version" - - "{{.OUTPUT_DIR}}/bin/qemu-system-x86_64-tcg --version" - - "test -f {{.OUTPUT_DIR}}/share/spin-stack/qemu/pvh.bin" - cmds: - # A second *source* for one artefact, not a second definition of the build — the - # shape hack/guest-kernel.sh has for the kernel. `task qemu:build` compiles QEMU from - # source, tens of minutes, and every lane that starts an Agent needs qemu-img. The - # image is the one the QEMU workflow publishes, from the same builder stage. - - | - set -euo pipefail - img={{.QEMU_IMAGE}}:{{.QEMU_VERSION}} - BINARIES="qemu-img qemu-system-x86_64 qemu-system-x86_64-tcg" - command -v docker >/dev/null || { - echo "no qemu-img at {{.OUTPUT_DIR}}/bin and no docker to fetch one — run: task qemu:build" >&2; exit 1; } - docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img" || { - # Which of the two causes this is matters, because the fixes are opposites. - # The tag is QEMU_VERSION, and the image is published by .github/workflows/qemu.yml - # — a separate workflow. So the first push of a version bump races: this gate - # starts while the image for the new tag is still being built, and no amount of - # rebuilding *here* helps. Waiting does. That happened on the 11.0.2 → 11.1.1 bump, - # where the advice below ("build it") was the wrong advice, confidently given. - echo "no $img locally and it could not be pulled." >&2 - echo " If QEMU_VERSION changed in this commit: .github/workflows/qemu.yml publishes" >&2 - echo " that tag and takes tens of minutes. Let it finish, then re-run this." >&2 - echo " Otherwise the tag should already exist — build it locally: task qemu:build" >&2 - exit 1; } - mkdir -p {{.OUTPUT_DIR}}/bin - cid=$(docker create "$img") - trap 'docker rm -f "$cid" >/dev/null' EXIT - # Two binaries and one firmware tree, because two different lanes need different - # halves and neither is worth a second task: an Agent needs qemu-img to create and - # inspect a chain, and the guest lane also needs a QEMU to boot and the firmware - # to boot *with* — pvh.bin above all, since a direct kernel boot has no entry - # point without it and fails as a rom-open error that reads like a backend bug. - for b in $BINARIES; do - docker cp -L "$cid":/usr/local/bin/"$b" {{.OUTPUT_DIR}}/bin/"$b".real - done - rm -rf {{.OUTPUT_DIR}}/share/spin-stack/qemu - mkdir -p {{.OUTPUT_DIR}}/share/spin-stack - docker cp "$cid":/usr/share/spin-stack/qemu {{.OUTPUT_DIR}}/share/spin-stack/qemu - - # And the libraries it links against, computed *inside the image*: the binary is - # dynamically linked (liburing, libaio, glib), and extracting it alone produced a - # file that ran on a developer's machine and died on a CI runner with - # "liburing.so.2: cannot open shared object file". A hand-written apt-get list is a - # second statement of that, kept in step by nobody. The interpreter counts too — - # ldd names the loader on its own line, with no `=>`. - rm -rf {{.OUTPUT_DIR}}/lib/qemu && mkdir -p {{.OUTPUT_DIR}}/lib/qemu - for so in $(docker run --rm --entrypoint sh "$img" -c \ - "for b in $BINARIES; do ldd /usr/local/bin/\$b; done" \ - | sed -n -e "s|.*=> \(/[^ ]*\).*|\1|p" -e "s|^\t\(/[^ ]*\) (0x.*|\1|p" | sort -u); do - # -L, because these are symlinks: /lib/x86_64-linux-gnu/liburing.so.2 points at - # liburing.so.2.14 and /lib64/ld-linux-x86-64.so.2 points out of its own - # directory. Copied as links they land dangling, and the failure that produces is - # indistinguishable from having copied nothing at all — which is how the first - # attempt at this looked like it had worked. - docker cp -L "$cid":"$so" {{.OUTPUT_DIR}}/lib/qemu/ - done - - # A wrapper and not an RPATH rewrite: patchelf would be one more pinned tool in the - # gate. It invokes the image's *loader* rather than setting LD_LIBRARY_PATH, so the - # host's glibc is not in the picture at all: LD_LIBRARY_PATH would have this - # binary's libc.so.6 loaded by whatever loader the host has, and those two are one - # unit. Verified by running it on debian:12, whose glibc is older than the image's. - ld=$(basename $(ls {{.OUTPUT_DIR}}/lib/qemu/ld-linux-*)) - for b in $BINARIES; do - cat > {{.OUTPUT_DIR}}/bin/"$b" </dev/null || { - echo "extracted $b does not run on this host" >&2; exit 1; } - done - echo "extracted $BINARIES, their loader and $(ls {{.OUTPUT_DIR}}/lib/qemu | wc -l) libraries from $img"