image: replace output_size with a qcow2 virtual_size

docs/user/images.md

@@ -351,11 +351,10 @@ when needed:
   set its `sign_expected_pcr` to false to opt out;
 - `image.ddi` renders ordered Starlark partition definitions and uses offline `systemd-repart` to create a
   GPT disk and independent partition artifacts in one `image.DdiInfo`; its disk field is absent for a
-  split-only invocation, `output_size` composes the disk with free space behind its partitions, and
-  `strip_pkgdb` leaves the package database out of them, and `mkfs_options` tunes the filesystems it
-  creates;
+  split-only invocation, `strip_pkgdb` leaves the package database out of the partitions, and
+  `mkfs_options` tunes the filesystems it creates;
 - `image.disk_convert` re-encodes a raw disk with an explicitly selected box and provides
-  `image.DiskConversionInfo`;
+  `image.DiskConversionInfo`; `virtual_size` sizes the disk a qcow2 describes;
 - `image.bootable` selects a kernel and matching initrd from a logical image, exposed as `[uki]`, `[kernel]`,
   and `[initrd]` subtargets;
 - `image.sysext` builds a systemd-sysext(8) DDI with `systemd-repart`, containing `/usr`, `/opt`, and
@@ -425,13 +424,10 @@ Optional attributes:
   `image.ddi()`.
 - `secure_boot_key` (target providing `image.SigningKeyInfo`): Signs the UKIs and systemd-boot; see "Secure
   Boot signing" below.
-- `output_size` (size string such as `"20G"`): Ships the disk at this size instead of at the size its
-  partitions need, so an installed system finds room past them and does not have to resize its medium
-  before first boot; passed on to `image.ddi()`. The partitions keep the sizes their definitions ask
-  for, and the composed file is enlarged behind the last one, so the added room costs nothing on disk and,
-  as with `image.vm`'s `grow`, sits past the GPT backup header until something rewrites the table. A size
-  the
-  partitions do not fit in fails the build.
+- `virtual_size` (size string such as `"20G"`): The size of a virtual disk like `[qcow2]`. A system booted
+  from it can claim the extra space during first boot, e.g. for a root partition. The raw image stays at
+  the size of its partitions. As with `image.vm`'s `grow`, the extra space sits past the GPT backup header
+  until something rewrites the table. A size the partitions do not fit in fails the build.
 - `mkfs_options` (dict of filesystem to option list, default `{}`): Options `mkfs` is given when
   creating a partition of that filesystem, passed on to `image.ddi()`, which hands each list to
   `systemd-repart` as `SYSTEMD_REPART_MKFS_OPTIONS_<FSTYPE>`. Naming a filesystem the disk never formats
@@ -855,7 +851,7 @@ and the boot loader read (`io.systemd.stub.kernel-cmdline-extra` and its `boot`
 ephemeral guest is given room the built partitions do not occupy; the guest claims that room itself, for
 example with `systemd-repart`. vmspawn grows the built artifact in place, and the added space sits past the
 GPT backup header until something rewrites the table. Use it to give a guest more room than the shipped
-disk carries; a disk that should always carry that room sets `output_size` on the image instead.
+disk carries; a qcow2 that should always carry that room comes from setting `virtual_size` on the image.
 
 ```sh
 tine buck run //examples/image:boot-demo-vm.fedora

examples/image/BUCK

@@ -221,10 +221,10 @@ image.bootable_disk(
     splash = distribution.select({"arch": "/usr/share/systemd/bootctl/splash-arch.bmp"}, default = None),
     # A shipped appliance has no terminal to read a man page on.
     install_docs = False,
-    # Ship the disk with free space past its partitions, so an installed system can claim it without
-    # resizing the medium first. The space is sparse and this image boots from tmpfs, so nothing here
-    # consumes it; a real appliance creates a root partition in it on first boot.
-    output_size = "2G",
+    # Ship the qcow2 with free space past its partitions, so a VM booting it can claim it without
+    # resizing the disk first. This image boots from tmpfs, so nothing here consumes it; a real
+    # appliance creates a root partition in it on first boot.
+    virtual_size = "2G",
 )
 
 image.vm(
@@ -262,12 +262,21 @@ tests.vm_smoke(
         "test -s /usr/lib/udev/hwdb.bin",
         # The runner's cmdline_extra reached the kernel the UKI booted.
         "grep -q tine.smoke=1 /proc/cmdline",
-        # The guest sees the disk at its output_size, and the partitions it boots from still
-        # occupy only the front of it.
-        "test $(blockdev --getsize64 /dev/vda) -eq 2147483648",
     ],
 )
 
+# The qcow2 describes a disk of the image's virtual_size, while the raw stays as small as its partitions.
+box.sh_test(
+    name = "virtual-size-test",
+    labels = ["image"],
+    box = "@tine//catalog:fedora.rawhide.box",
+    script = """
+test "$(qemu-img info --output=json "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)["virtual-size"])')" = 2147483648
+test "$(stat -c %s "$2")" -lt $((900 << 20))
+""",
+    args = ["$(location :boot-demo.fedora[qcow2])", "$(location :boot-demo.fedora)"],
+)
+
 # System-extension DDI extending :boot-demo: only the delta above the base lands in the
 # image, and its extension-release pins the base's ID/VERSION_ID. `boot-demo-vm` exposes
 # it under /var/lib/extensions.

examples/image/expected.json

@@ -1,7 +1,7 @@
 {
   "arm64": {
     "boot-demo.fedora": {
-      "sha256": "76733eaf5e248b9d6004557c9bfc20f200df0bc7c66dd7a742bde969b52bbbcb"
+      "sha256": "0783b7ef59cebeef993e7cf6d727769d9e828d3a7a544e65a3c9de2418ab6a4a"
     },
     "boot-demo.fedora[boot][initrd]": {
       "sha256": "401d67515ee8cfcd6b546fc6ab8424c0a53999baadeb391f33b66e9e213a293a"
@@ -36,7 +36,7 @@
   },
   "x86_64": {
     "boot-demo.fedora": {
-      "sha256": "c2cec78b3489475f18be9db1cfba8f7ffe47614910b51f7ece9d0bf92bf98fdb"
+      "sha256": "6a7b4796d2216d9bf9d0db587d08e9c8d44fdc1ebc817c2c4ba3f428aa6f3195"
     },
     "boot-demo.fedora[boot][initrd]": {
       "sha256": "d0081aa88c2729fb5363d30eb815e3cdc563eb9c61a5c19e93413ca80a3a0933"

image/compose.bzl

@@ -28,6 +28,8 @@ load(
     ":ddi.bzl",
     "DISK_FORMATS",
     "Partition",  # @unused Used as a type.
+    "VIRTUAL_SIZE_ATTR",
+    "VIRTUAL_SIZE_FORMATS",
     "declare_ddi",
     "declare_disk_conversion",
     "encode_definitions",
@@ -441,7 +443,6 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]:
         imported = [system.info],
         imported_root_hash = system.info.root_hash,
         mkfs_options = ctx.attrs.mkfs_options,
-        output_size = ctx.attrs.output_size,
         seed = ctx.attrs.disk_seed,
     )
     raw = disk.info.disk
@@ -456,6 +457,7 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]:
             disk = disk.info,
             box = esp.box,
             format = format,
+            virtual_size = ctx.attrs.virtual_size if format in VIRTUAL_SIZE_FORMATS else None,
             identifier = format,
         )
         for format in DISK_FORMATS
@@ -559,11 +561,6 @@ _bootable_disk_image = rule(
             default = {},
             doc = "filesystem -> the options mkfs is given when formatting a partition of that type",
         ),
-        "output_size": attrs.option(
-            attrs.string(),
-            default = None,
-            doc = 'size to compose the disk at, e.g. "20G"; the room past the partitions stays free',
-        ),
         "package_manager": attrs.option(
             attrs.dep(providers = [PackageManagerInfo]),
             default = None,
@@ -578,6 +575,7 @@ _bootable_disk_image = rule(
             doc = "leave the package database out of the system partitions; the image still carries it",
         ),
         "verity_key": VERITY_KEY_ATTR,
+        "virtual_size": VIRTUAL_SIZE_ATTR,
     },
 )
 

image/convert.py

@@ -17,6 +17,7 @@ class Spec(TypedDict):
     format: str
     input: str
     out: str
+    virtual_size: str | None
 
 
 def main(argv: list[str] | None = None) -> None:
@@ -41,6 +42,11 @@ def main(argv: list[str] | None = None) -> None:
             "-o", str(out), str(source),
         ]  # fmt: skip
     subprocess.run(cmd, check=True)
+    if size := spec["virtual_size"]:
+        # Like vmspawn's --grow-image, the added room sits past the GPT backup header until first
+        # boot's repart rewrites the table. qemu-img refuses to shrink, so a size the partitions do
+        # not fit in fails here.
+        subprocess.run(["qemu-img", "resize", "-q", "-f", "qcow2", str(out), size], check=True)
 
 
 if __name__ == "__main__":

image/ddi.bzl

@@ -292,7 +292,6 @@ def declare_ddi(
     imported_root_hash: RootHashInfo | None = None,
     seed: str | None = None,
     verity_key: SigningKeyInfo | None = None,
-    output_size: str | None = None,
     strip_pkgdb: bool = False,
     mkfs_options: dict[str, list[str]] = {},
     basename: str = "image",
@@ -315,12 +314,6 @@ def declare_ddi(
         split = split,
     )
 
-    if output_size != None:
-        if not disk:
-            fail("ddi: output_size applies to a composed disk, which this invocation does not emit")
-        if not regex_match(SIZE_PATTERN, output_size):
-            fail("ddi: invalid output_size {!r}".format(output_size))
-
     # repart hands mkfs one variable per filesystem and splits it on whitespace, so an option
     # carrying any would silently arrive as two.
     for filesystem, options in mkfs_options.items():
@@ -392,7 +385,6 @@ def declare_ddi(
         "manifest": None,
         "mkfs_options": mkfs_options,
         "out": None,
-        "output_size": output_size,
         "partitions": [
             {
                 "blocks": value.blocks,
@@ -504,7 +496,6 @@ def _ddi_impl(ctx: AnalysisContext) -> list[Provider]:
         split = ctx.attrs.split,
         strip_pkgdb = ctx.attrs.strip_pkgdb,
         verity_key = resolve_signing_key(ctx.attrs.verity_key),
-        output_size = ctx.attrs.output_size,
     )
     return [
         DefaultInfo(default_outputs = result.outputs, sub_targets = result.sub_targets),
@@ -523,11 +514,6 @@ DDI_ATTRS = {
         default = {},
         doc = "filesystem -> the options mkfs is given when formatting a partition of that type",
     ),
-    "output_size": attrs.option(
-        attrs.string(),
-        default = None,
-        doc = 'size to compose the disk at, e.g. "20G"; the room past the partitions stays free',
-    ),
     "seed": attrs.option(
         attrs.string(),
         default = None,
@@ -587,6 +573,16 @@ def ddi(
 # The disk conversion output formats, doubling as each artifact's extension.
 DISK_FORMATS = ["qcow2", "raw.zst"]
 
+# The formats whose disk has a size of its own, so it can leave room past the partitions for first boot to
+# claim. The raw itself stays as small as its partitions.
+VIRTUAL_SIZE_FORMATS = ["qcow2"]
+
+VIRTUAL_SIZE_ATTR = attrs.option(
+    attrs.string(),
+    default = None,
+    doc = 'size of the disk a qcow2 describes, e.g. "20G"; the room past the partitions stays free',
+)
+
 DiskConversionInfo = provider(
     doc = "A composed raw disk re-encoded into one distributable format.",
     fields = {
@@ -602,12 +598,22 @@ def declare_disk_conversion(
     disk: DdiInfo,
     box: Dependency,
     format: str,
+    virtual_size: str | None = None,
     basename: str = "image",
     identifier: str | None = None,
 ) -> DiskConversionInfo:
-    """Declare a disk format conversion from a resolved raw disk provider."""
+    """Declare a disk format conversion from a resolved raw disk provider.
+
+    `virtual_size` sizes the disk of a format in VIRTUAL_SIZE_FORMATS, which otherwise is as large as
+    the raw.
+    """
     if disk.disk == None:
         fail("disk_convert: DdiInfo does not contain a composed disk")
+    if virtual_size != None:
+        if format not in VIRTUAL_SIZE_FORMATS:
+            fail("disk_convert: a {} disk is the raw's size and takes no virtual_size".format(format))
+        if not regex_match(SIZE_PATTERN, virtual_size):
+            fail("disk_convert: invalid virtual_size {!r}".format(virtual_size))
     out = declare_out(ctx, identifier, basename + "." + format)
     cmd = cmd_args(
         box_run(box = box[BoxInfo], exe = tools.convert),
@@ -618,6 +624,7 @@ def declare_disk_conversion(
                 "format": format,
                 "input": disk.disk,
                 "out": out.as_output(),
+                "virtual_size": virtual_size,
             },
         ),
     )
@@ -632,6 +639,7 @@ def _disk_convert_impl(ctx: AnalysisContext) -> list[Provider]:
         disk = ctx.attrs.disk[DdiInfo],
         box = ctx.attrs.box,
         format = ctx.attrs.format,
+        virtual_size = ctx.attrs.virtual_size,
     )
     return [DefaultInfo(default_output = info.image), info]
 
@@ -642,6 +650,7 @@ disk_convert = rule(
         "box": attrs.exec_dep(providers = [BoxInfo], doc = "execution environment supplying conversion tools"),
         "disk": attrs.dep(providers = [DdiInfo], doc = "the composed raw disk to re-encode"),
         "format": attrs.enum(DISK_FORMATS),
+        "virtual_size": VIRTUAL_SIZE_ATTR,
     }
     | IMAGE_TOOLS_ATTR,
 )

image/ddi.py

@@ -14,7 +14,6 @@
 import tempfile
 import uuid
 from dataclasses import dataclass
-from decimal import Decimal
 from pathlib import Path
 from typing import Any, Self, TypedDict
 
@@ -80,7 +79,6 @@ class Spec(finalize.ImageSpec):
     basename: str
     # Stable target identity the seed derives from, unless one is given outright.
     identity: str
-    output_size: str | None
     seed: str | None
     signing: repart.KeySpec | None
     # The partitions this invocation writes, and the independent ones copied into the result.
@@ -329,26 +327,6 @@ def _copy_partition(row: dict[str, Any], name: str, blocks: Path, metadata: Path
     metadata.write_text(json.dumps(described, sort_keys=True, separators=(",", ":")) + "\n")
 
 
-def _grow(disk: Path, size: str) -> None:
-    """Extend a composed disk to its requested size, leaving the added room a hole.
-
-    repart sizes the disk to what its partitions need; the extra room is not something any of them
-    should claim, so the file is enlarged after the fact instead. As with vmspawn's --grow-image,
-    the room sits past the GPT backup header until something rewrites the table.
-    """
-    suffixes = "KMGTPE"  # systemd reads them base-1024
-    target = (
-        int(Decimal(size[:-1]) * 1024 ** (suffixes.index(size[-1]) + 1))
-        if size[-1] in suffixes
-        else int(size)
-    )
-    composed = disk.stat().st_size
-    if composed > target:
-        util.fail(f"ddi: the partitions need {composed} bytes, more than the requested {size}")
-    with disk.open("r+b") as f:
-        f.truncate(target)
-
-
 def main(argv: list[str] | None = None) -> None:
     spec = specs.parse(Spec, "ddi", argv)
 
@@ -485,8 +463,6 @@ def main(argv: list[str] | None = None) -> None:
             repart.write_root_hash(rows, Path(spec["root_hash_out"]))
         if out and splits:
             shutil.copyfile(disk, out)
-        if out and spec["output_size"]:
-            _grow(out, spec["output_size"])
 
     artifacts = []
     if out: