diff --git a/docs/user/images.md b/docs/user/images.md index 11799542..ea9d7b5d 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -351,11 +351,10 @@ when needed: set its `sign_expected_pcr` to false to opt out; - `image.ddi` renders ordered Starlark partition definitions and uses offline `systemd-repart` to create a GPT disk and independent partition artifacts in one `image.DdiInfo`; its disk field is absent for a - split-only invocation, `output_size` composes the disk with free space behind its partitions, and - `strip_pkgdb` leaves the package database out of them, and `mkfs_options` tunes the filesystems it - creates; + split-only invocation, `strip_pkgdb` leaves the package database out of the partitions, and + `mkfs_options` tunes the filesystems it creates; - `image.disk_convert` re-encodes a raw disk with an explicitly selected box and provides - `image.DiskConversionInfo`; + `image.DiskConversionInfo`; `virtual_size` sizes the disk a qcow2 describes; - `image.bootable` selects a kernel and matching initrd from a logical image, exposed as `[uki]`, `[kernel]`, and `[initrd]` subtargets; - `image.sysext` builds a systemd-sysext(8) DDI with `systemd-repart`, containing `/usr`, `/opt`, and @@ -425,13 +424,10 @@ Optional attributes: `image.ddi()`. - `secure_boot_key` (target providing `image.SigningKeyInfo`): Signs the UKIs and systemd-boot; see "Secure Boot signing" below. -- `output_size` (size string such as `"20G"`): Ships the disk at this size instead of at the size its - partitions need, so an installed system finds room past them and does not have to resize its medium - before first boot; passed on to `image.ddi()`. The partitions keep the sizes their definitions ask - for, and the composed file is enlarged behind the last one, so the added room costs nothing on disk and, - as with `image.vm`'s `grow`, sits past the GPT backup header until something rewrites the table. A size - the - partitions do not fit in fails the build. +- `virtual_size` (size string such as `"20G"`): The size of a virtual disk like `[qcow2]`. A system booted + from it can claim the extra space during first boot, e.g. for a root partition. The raw image stays at + the size of its partitions. As with `image.vm`'s `grow`, the extra space sits past the GPT backup header + until something rewrites the table. A size the partitions do not fit in fails the build. - `mkfs_options` (dict of filesystem to option list, default `{}`): Options `mkfs` is given when creating a partition of that filesystem, passed on to `image.ddi()`, which hands each list to `systemd-repart` as `SYSTEMD_REPART_MKFS_OPTIONS_`. Naming a filesystem the disk never formats @@ -855,7 +851,7 @@ and the boot loader read (`io.systemd.stub.kernel-cmdline-extra` and its `boot` ephemeral guest is given room the built partitions do not occupy; the guest claims that room itself, for example with `systemd-repart`. vmspawn grows the built artifact in place, and the added space sits past the GPT backup header until something rewrites the table. Use it to give a guest more room than the shipped -disk carries; a disk that should always carry that room sets `output_size` on the image instead. +disk carries; a qcow2 that should always carry that room comes from setting `virtual_size` on the image. ```sh tine buck run //examples/image:boot-demo-vm.fedora diff --git a/examples/image/BUCK b/examples/image/BUCK index be8b314b..69c51457 100644 --- a/examples/image/BUCK +++ b/examples/image/BUCK @@ -221,10 +221,10 @@ image.bootable_disk( splash = distribution.select({"arch": "/usr/share/systemd/bootctl/splash-arch.bmp"}, default = None), # A shipped appliance has no terminal to read a man page on. install_docs = False, - # Ship the disk with free space past its partitions, so an installed system can claim it without - # resizing the medium first. The space is sparse and this image boots from tmpfs, so nothing here - # consumes it; a real appliance creates a root partition in it on first boot. - output_size = "2G", + # Ship the qcow2 with free space past its partitions, so a VM booting it can claim it without + # resizing the disk first. This image boots from tmpfs, so nothing here consumes it; a real + # appliance creates a root partition in it on first boot. + virtual_size = "2G", ) image.vm( @@ -262,12 +262,21 @@ tests.vm_smoke( "test -s /usr/lib/udev/hwdb.bin", # The runner's cmdline_extra reached the kernel the UKI booted. "grep -q tine.smoke=1 /proc/cmdline", - # The guest sees the disk at its output_size, and the partitions it boots from still - # occupy only the front of it. - "test $(blockdev --getsize64 /dev/vda) -eq 2147483648", ], ) +# The qcow2 describes a disk of the image's virtual_size, while the raw stays as small as its partitions. +box.sh_test( + name = "virtual-size-test", + labels = ["image"], + box = "@tine//catalog:fedora.rawhide.box", + script = """ +test "$(qemu-img info --output=json "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)["virtual-size"])')" = 2147483648 +test "$(stat -c %s "$2")" -lt $((900 << 20)) +""", + args = ["$(location :boot-demo.fedora[qcow2])", "$(location :boot-demo.fedora)"], +) + # System-extension DDI extending :boot-demo: only the delta above the base lands in the # image, and its extension-release pins the base's ID/VERSION_ID. `boot-demo-vm` exposes # it under /var/lib/extensions. diff --git a/examples/image/expected.json b/examples/image/expected.json index 06ade67a..69762c29 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -1,7 +1,7 @@ { "arm64": { "boot-demo.fedora": { - "sha256": "76733eaf5e248b9d6004557c9bfc20f200df0bc7c66dd7a742bde969b52bbbcb" + "sha256": "0783b7ef59cebeef993e7cf6d727769d9e828d3a7a544e65a3c9de2418ab6a4a" }, "boot-demo.fedora[boot][initrd]": { "sha256": "401d67515ee8cfcd6b546fc6ab8424c0a53999baadeb391f33b66e9e213a293a" @@ -36,7 +36,7 @@ }, "x86_64": { "boot-demo.fedora": { - "sha256": "c2cec78b3489475f18be9db1cfba8f7ffe47614910b51f7ece9d0bf92bf98fdb" + "sha256": "6a7b4796d2216d9bf9d0db587d08e9c8d44fdc1ebc817c2c4ba3f428aa6f3195" }, "boot-demo.fedora[boot][initrd]": { "sha256": "d0081aa88c2729fb5363d30eb815e3cdc563eb9c61a5c19e93413ca80a3a0933" diff --git a/image/compose.bzl b/image/compose.bzl index 6893ae65..702805b9 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -28,6 +28,8 @@ load( ":ddi.bzl", "DISK_FORMATS", "Partition", # @unused Used as a type. + "VIRTUAL_SIZE_ATTR", + "VIRTUAL_SIZE_FORMATS", "declare_ddi", "declare_disk_conversion", "encode_definitions", @@ -441,7 +443,6 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: imported = [system.info], imported_root_hash = system.info.root_hash, mkfs_options = ctx.attrs.mkfs_options, - output_size = ctx.attrs.output_size, seed = ctx.attrs.disk_seed, ) raw = disk.info.disk @@ -456,6 +457,7 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: disk = disk.info, box = esp.box, format = format, + virtual_size = ctx.attrs.virtual_size if format in VIRTUAL_SIZE_FORMATS else None, identifier = format, ) for format in DISK_FORMATS @@ -559,11 +561,6 @@ _bootable_disk_image = rule( default = {}, doc = "filesystem -> the options mkfs is given when formatting a partition of that type", ), - "output_size": attrs.option( - attrs.string(), - default = None, - doc = 'size to compose the disk at, e.g. "20G"; the room past the partitions stays free', - ), "package_manager": attrs.option( attrs.dep(providers = [PackageManagerInfo]), default = None, @@ -578,6 +575,7 @@ _bootable_disk_image = rule( doc = "leave the package database out of the system partitions; the image still carries it", ), "verity_key": VERITY_KEY_ATTR, + "virtual_size": VIRTUAL_SIZE_ATTR, }, ) diff --git a/image/convert.py b/image/convert.py index b6c54639..0e06f054 100755 --- a/image/convert.py +++ b/image/convert.py @@ -17,6 +17,7 @@ class Spec(TypedDict): format: str input: str out: str + virtual_size: str | None def main(argv: list[str] | None = None) -> None: @@ -41,6 +42,11 @@ def main(argv: list[str] | None = None) -> None: "-o", str(out), str(source), ] # fmt: skip subprocess.run(cmd, check=True) + if size := spec["virtual_size"]: + # Like vmspawn's --grow-image, the added room sits past the GPT backup header until first + # boot's repart rewrites the table. qemu-img refuses to shrink, so a size the partitions do + # not fit in fails here. + subprocess.run(["qemu-img", "resize", "-q", "-f", "qcow2", str(out), size], check=True) if __name__ == "__main__": diff --git a/image/ddi.bzl b/image/ddi.bzl index 751c4d7c..8e9b2001 100644 --- a/image/ddi.bzl +++ b/image/ddi.bzl @@ -292,7 +292,6 @@ def declare_ddi( imported_root_hash: RootHashInfo | None = None, seed: str | None = None, verity_key: SigningKeyInfo | None = None, - output_size: str | None = None, strip_pkgdb: bool = False, mkfs_options: dict[str, list[str]] = {}, basename: str = "image", @@ -315,12 +314,6 @@ def declare_ddi( split = split, ) - if output_size != None: - if not disk: - fail("ddi: output_size applies to a composed disk, which this invocation does not emit") - if not regex_match(SIZE_PATTERN, output_size): - fail("ddi: invalid output_size {!r}".format(output_size)) - # repart hands mkfs one variable per filesystem and splits it on whitespace, so an option # carrying any would silently arrive as two. for filesystem, options in mkfs_options.items(): @@ -392,7 +385,6 @@ def declare_ddi( "manifest": None, "mkfs_options": mkfs_options, "out": None, - "output_size": output_size, "partitions": [ { "blocks": value.blocks, @@ -504,7 +496,6 @@ def _ddi_impl(ctx: AnalysisContext) -> list[Provider]: split = ctx.attrs.split, strip_pkgdb = ctx.attrs.strip_pkgdb, verity_key = resolve_signing_key(ctx.attrs.verity_key), - output_size = ctx.attrs.output_size, ) return [ DefaultInfo(default_outputs = result.outputs, sub_targets = result.sub_targets), @@ -523,11 +514,6 @@ DDI_ATTRS = { default = {}, doc = "filesystem -> the options mkfs is given when formatting a partition of that type", ), - "output_size": attrs.option( - attrs.string(), - default = None, - doc = 'size to compose the disk at, e.g. "20G"; the room past the partitions stays free', - ), "seed": attrs.option( attrs.string(), default = None, @@ -587,6 +573,16 @@ def ddi( # The disk conversion output formats, doubling as each artifact's extension. DISK_FORMATS = ["qcow2", "raw.zst"] +# The formats whose disk has a size of its own, so it can leave room past the partitions for first boot to +# claim. The raw itself stays as small as its partitions. +VIRTUAL_SIZE_FORMATS = ["qcow2"] + +VIRTUAL_SIZE_ATTR = attrs.option( + attrs.string(), + default = None, + doc = 'size of the disk a qcow2 describes, e.g. "20G"; the room past the partitions stays free', +) + DiskConversionInfo = provider( doc = "A composed raw disk re-encoded into one distributable format.", fields = { @@ -602,12 +598,22 @@ def declare_disk_conversion( disk: DdiInfo, box: Dependency, format: str, + virtual_size: str | None = None, basename: str = "image", identifier: str | None = None, ) -> DiskConversionInfo: - """Declare a disk format conversion from a resolved raw disk provider.""" + """Declare a disk format conversion from a resolved raw disk provider. + + `virtual_size` sizes the disk of a format in VIRTUAL_SIZE_FORMATS, which otherwise is as large as + the raw. + """ if disk.disk == None: fail("disk_convert: DdiInfo does not contain a composed disk") + if virtual_size != None: + if format not in VIRTUAL_SIZE_FORMATS: + fail("disk_convert: a {} disk is the raw's size and takes no virtual_size".format(format)) + if not regex_match(SIZE_PATTERN, virtual_size): + fail("disk_convert: invalid virtual_size {!r}".format(virtual_size)) out = declare_out(ctx, identifier, basename + "." + format) cmd = cmd_args( box_run(box = box[BoxInfo], exe = tools.convert), @@ -618,6 +624,7 @@ def declare_disk_conversion( "format": format, "input": disk.disk, "out": out.as_output(), + "virtual_size": virtual_size, }, ), ) @@ -632,6 +639,7 @@ def _disk_convert_impl(ctx: AnalysisContext) -> list[Provider]: disk = ctx.attrs.disk[DdiInfo], box = ctx.attrs.box, format = ctx.attrs.format, + virtual_size = ctx.attrs.virtual_size, ) return [DefaultInfo(default_output = info.image), info] @@ -642,6 +650,7 @@ disk_convert = rule( "box": attrs.exec_dep(providers = [BoxInfo], doc = "execution environment supplying conversion tools"), "disk": attrs.dep(providers = [DdiInfo], doc = "the composed raw disk to re-encode"), "format": attrs.enum(DISK_FORMATS), + "virtual_size": VIRTUAL_SIZE_ATTR, } | IMAGE_TOOLS_ATTR, ) diff --git a/image/ddi.py b/image/ddi.py index 944038a1..c2f046b0 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -14,7 +14,6 @@ import tempfile import uuid from dataclasses import dataclass -from decimal import Decimal from pathlib import Path from typing import Any, Self, TypedDict @@ -80,7 +79,6 @@ class Spec(finalize.ImageSpec): basename: str # Stable target identity the seed derives from, unless one is given outright. identity: str - output_size: str | None seed: str | None signing: repart.KeySpec | None # The partitions this invocation writes, and the independent ones copied into the result. @@ -329,26 +327,6 @@ def _copy_partition(row: dict[str, Any], name: str, blocks: Path, metadata: Path metadata.write_text(json.dumps(described, sort_keys=True, separators=(",", ":")) + "\n") -def _grow(disk: Path, size: str) -> None: - """Extend a composed disk to its requested size, leaving the added room a hole. - - repart sizes the disk to what its partitions need; the extra room is not something any of them - should claim, so the file is enlarged after the fact instead. As with vmspawn's --grow-image, - the room sits past the GPT backup header until something rewrites the table. - """ - suffixes = "KMGTPE" # systemd reads them base-1024 - target = ( - int(Decimal(size[:-1]) * 1024 ** (suffixes.index(size[-1]) + 1)) - if size[-1] in suffixes - else int(size) - ) - composed = disk.stat().st_size - if composed > target: - util.fail(f"ddi: the partitions need {composed} bytes, more than the requested {size}") - with disk.open("r+b") as f: - f.truncate(target) - - def main(argv: list[str] | None = None) -> None: spec = specs.parse(Spec, "ddi", argv) @@ -485,8 +463,6 @@ def main(argv: list[str] | None = None) -> None: repart.write_root_hash(rows, Path(spec["root_hash_out"])) if out and splits: shutil.copyfile(disk, out) - if out and spec["output_size"]: - _grow(out, spec["output_size"]) artifacts = [] if out: