Build custom image/AMI with preloaded extra extensions

docs/user/images.md

@@ -870,6 +870,87 @@ boot is not exposed the way the same package in the running system is. The union
 everything in the UKI, provided the image keeps the kernel package installed in its own tree, which is where
 the UKI's kernel and modules come from.
 
+### image.ddi_preloaded_disk
+
+`image.ddi_preloaded_disk()` adds extension and portable service DDIs to the ESP of an already built
+disk. It keeps all existing signatures intact, and fails if any file on the ESP already exists.
+
+```Starlark
+image.ddi_preloaded_disk(
+    name = "appliance",
+    base = ":base-os",
+    extensions = [":debug-tools", ":site-config"],
+    portables = [":webapp", ":webapp-config"],
+)
+```
+
+- `base` (target providing `image.DdiInfo`): The disk (raw or zstd-compressed), such as an
+  `image.disk()`, another `image.ddi_preloaded_disk()`, or an `image.imported_disk()` (see below).
+  It needs exactly one ESP partition, with room for the DDIs.
+- `extensions` (list of extension targets): Each must have the `system` scope. A DDI goes to
+  `/extensions/<name>.raw.v/<name>_<version>_<arch>.raw` on the ESP, regardless of its classes.
+- `portables` (list of portable images and extensions with the `portable` scope): A DDI goes to
+  `/portables/<name>.raw.v/<name>_<version>_<arch>.raw` on the ESP.
+- `virtual_size` (size string such as `"20G"`): Same as for `image.disk()`. It defaults to the base's.
+
+These paths are compatible with `systemd-sysupdate` transfer files and `systemd-vpick` versioning,
+so that updates replace the preloaded extensions.
+
+Nothing currently merges or attaches the DDIs at boot: systemd does not look for extensions and
+portable images in the ESP. Booting/instantiation needs to do that with creating symlinks to `/run`
+or `/var`, followed by `systemd-{conf,sys}ext refresh` (as `systemd-tmpfiles-setup.service` runs
+after the initial confext/sysext merge). For portable services, call `portablectl attach` as usual.
+
+The target defaults to the disk as `<name>.raw.zst`, and offers `[qcow2]` and `[vmdk]` subtargets.
+It can be booted by `image.vm`.
+
+### External disks as a base
+
+`image.imported_disk()` declares a base disk for preloading from a disk that was built elsewhere,
+such as a published release of an OS. `disk` is its `.raw` or `.raw.zst`, and the optional `ami` its
+`.ami.json`, which the target provides as `[ami]`, plus optionally a `virtual_size`. Pin each file in the
+project's `tine.lock.json`, and download it with [`http.file()`](http.md#declaring-a-download):
+
+```json
+{
+  "os-disk": {
+    "url": "https://releases.example.com",
+    "release": "1.2",
+    "platforms": {
+      "x86_64": {"artifact": "os_1.2_x86-64.raw.zst", "sha256": "…", "size": 382105381}
+    }
+  },
+  "os-ami": {
+    "url": "https://releases.example.com",
+    "release": "1.2",
+    "platforms": {
+      "x86_64": {"artifact": "os_1.2_x86-64.ami.json", "sha256": "…", "size": 1414}
+    }
+  }
+}
+```
+
+`tine bump` fills in `sha256` and `size`, and with a `releases` index moves the pins to the newest
+release, see [The pin](http.md#the-pin).
+
+```Starlark
+load("//:tine.lock.json", pins = "value")
+load("@tine//http:defs.bzl", "http")
+
+http.file(name = "os.raw.zst", spec = pins["os-disk"])
+
+http.file(name = "os.ami.json", spec = pins["os-ami"])
+
+image.imported_disk(
+    name = "os",
+    disk = ":os.raw.zst",
+    ami = ":os.ami.json",
+    virtual_size = "20G",
+)
+```
+
+Then `image.ddi_preloaded_disk()` takes `:os` as a `base =`, see above.
+
 ### Kernel modules in the UKI
 
 A UKI carries the kernel, the initrds it was built from, and one further initrd the `image.uki` rule
@@ -1137,7 +1218,11 @@ versioned UKI creation, semantic boot-artifact extraction, ESP-layer assembly, a
 `demo-portable` is a portable service image, and `demo-portable-ext` an extension for it. Running
 `boot-demo-vm` validates the interactive VM runner. `boot-demo-vm-smoke` boots `boot-demo`, merges
 `demo-ext` and `demo-confext` at boot, and attaches the two portable DDIs with
-`portablectl attach --extension demo-portable-ext demo-portable`.
+`portablectl attach --extension demo-portable-ext demo-portable`. `boot-demo-preloaded` is `boot-demo`
+with these four DDIs on its ESP. `boot-demo-preloaded-vm-smoke` boots it, activates the DDIs from the ESP,
+and checks that they merge and attach. `boot-demo-imported` imports the `[raw.zst]` and `[ami]` of
+`boot-demo` the way a project that builds on a release would, and `imported-disk-test` checks that
+preloading it gives the same disk.
 
 A host that merges a confext or attaches a portable image needs two things, which `boot-demo` provides:
 

examples/image/BUCK

@@ -385,6 +385,125 @@ image.extension(
     ],
 )
 
+_PRELOADED_EXTENSIONS = [":demo-confext", ":demo-ext"]
+
+_PRELOADED_PORTABLES = [":demo-portable", ":demo-portable-ext"]
+
+# boot-demo with the demo DDIs on its ESP, where systemd-sysupdate would update them. Nothing merges or
+# attaches them at boot yet: whoever instantiates the disk does that.
+image.ddi_preloaded_disk(
+    name = "boot-demo-preloaded",
+    base = ":boot-demo",
+    extensions = _PRELOADED_EXTENSIONS,
+    portables = _PRELOADED_PORTABLES,
+)
+
+# boot-demo's published raw.zst and AMI parameters, the way a project that builds on a release of it
+# imports them, and the same DDIs on its ESP.
+image.imported_disk(
+    name = "boot-demo-imported",
+    ami = ":boot-demo[ami]",
+    disk = ":boot-demo[raw.zst]",
+    # As boot-demo's own, which a release would publish beside the disk.
+    virtual_size = "2G",
+)
+
+image.ddi_preloaded_disk(
+    name = "boot-demo-imported-preloaded",
+    base = ":boot-demo-imported",
+    extensions = _PRELOADED_EXTENSIONS,
+    portables = _PRELOADED_PORTABLES,
+)
+
+# The preloaded disk defaults to the virtual_size of the imported one.
+box.sh_test(
+    name = "imported-virtual-size-test",
+    labels = ["image"],
+    box = "@tine//tools:dev.box",
+    script = """test "$(qemu-img info --output=json "$1" | jq '."virtual-size"')" = 2147483648""",
+    args = ["$(location :boot-demo-imported-preloaded.fedora[qcow2])"],
+    distro = _FEDORA,
+)
+
+image.vm(
+    name = "boot-demo-preloaded-vm",
+    autologin = "root",
+    credentials = {
+        "firstboot.locale": "C.UTF-8",
+        "firstboot.timezone": "UTC",
+    },
+    image = ":boot-demo-preloaded",
+)
+
+# The checks activate the DDIs on the ESP the way an instance would, see "image.ddi_preloaded_disk" in images.md.
+# The ESP carries one version of each, so the links can name the .raw.v directories.
+tests.vm_smoke(
+    name = "boot-demo-preloaded-vm-smoke",
+    vm = ":boot-demo-preloaded-vm",
+    checks = [
+        # The disk has boot-demo's virtual_size
+        'test "$(lsblk --nodeps --noheadings --bytes --output SIZE /dev/vda)" = 2147483648',
+        # The ESP is where systemd-gpt-auto-generator mounts it, which is /boot on this image.
+        'test -d "$(bootctl --print-esp-path)/extensions"',
+        "mkdir -p /run/extensions /run/confexts",
+        'ln -s "$(bootctl --print-esp-path)/extensions/demo-ext.raw.v" /run/extensions/',
+        'ln -s "$(bootctl --print-esp-path)/extensions/demo-confext.raw.v" /run/confexts/',
+        "systemd-sysext refresh",
+        "systemd-confext refresh",
+        # systemd-vpick picked each newest version out of its directory, and the extension-release of
+        # each matches boot-demo.
+        "test -x /usr/bin/zsh",
+        "test -f /etc/tine/from-confext",
+        'portablectl attach --now --runtime --extension "$(bootctl --print-esp-path)/portables/demo-portable-ext.raw.v" "$(bootctl --print-esp-path)/portables/demo-portable.raw.v"',
+        "systemctl is-active demo-portable.service",
+    ],
+)
+
+# The preloaded disk registers with the AMI parameters of boot-demo, under its own name.
+box.sh_test(
+    name = "preloaded-ami-test",
+    labels = ["image"],
+    box = "@tine//tools:dev.box",
+    script = """
+python3 -c '
+import json, sys
+base, preloaded = (json.load(open(path)) for path in sys.argv[1:])
+assert preloaded["Name"] == preloaded["Description"] == "boot-demo-preloaded", preloaded
+names = ("Name", "Description")
+assert {k: v for k, v in preloaded.items() if k not in names} == {k: v for k, v in base.items() if k not in names}, (base, preloaded)
+' "$@"
+""",
+    args = ["$(location :boot-demo.fedora[ami])", "$(location :boot-demo-preloaded.fedora[ami])"],
+    distro = _FEDORA,
+)
+
+# Preloading the imported raw.zst writes the same disk as preloading boot-demo itself, and registers it
+# with the same AMI parameters, under its own name.
+box.sh_test(
+    name = "imported-disk-test",
+    labels = ["image"],
+    box = "@tine//tools:dev.box",
+    script = """
+python3 -c '
+import hashlib, json, sys
+disk, imported_disk, ami, imported_ami = sys.argv[1:]
+digests = [hashlib.file_digest(open(path, "rb"), "sha256").hexdigest() for path in (disk, imported_disk)]
+assert digests[0] == digests[1], digests
+ami, imported_ami = json.load(open(ami)), json.load(open(imported_ami))
+assert imported_ami["Name"] == "boot-demo-imported-preloaded", imported_ami
+names = ("Name", "Description")
+assert {k: v for k, v in ami.items() if k not in names} == {k: v for k, v in imported_ami.items() if k not in names}
+' "$@"
+""",
+    args = [
+        "$(location :boot-demo-preloaded.fedora)",
+        "$(location :boot-demo-imported-preloaded.fedora)",
+        "$(location :boot-demo-preloaded.fedora[ami])",
+        "$(location :boot-demo-imported-preloaded.fedora[ami])",
+    ],
+    distro = _FEDORA,
+)
+
 # A file tree that differs per distribution is one copy whose source is a select, rather than a
 # target per distribution and an alias choosing between them. A distribution the select names no
 # tree for gets `None`, which copies nothing; only arch ships one so every other exercises that.

examples/image/expected.json

@@ -1,5 +1,8 @@
 {
   "arm64": {
+    "boot-demo-preloaded.fedora": {
+      "sha256": "eb0291105e6dda37fbe5b77bf603ece5c46ef75077238949aa0faa1191a4dacf"
+    },
     "boot-demo.fedora": {
       "sha256": "e3377bc790d7f40a25bd0126b5d18904d29bcc9cbb287047113019845d971b1f"
     },
@@ -35,6 +38,9 @@
     }
   },
   "x86_64": {
+    "boot-demo-preloaded.fedora": {
+      "sha256": "5639ec39438ad8de3facbc193b55163785b36956abec603061c95dce4229a8d2"
+    },
     "boot-demo.fedora": {
       "sha256": "810f4d8cd0ae75dd5d58c09784615409a4c0c5983f44b9a7e68b3e2b758aaab0"
     },

image/BUCK

@@ -17,6 +17,7 @@ image_tools(
     ddi = ":ddi",
     layer = ":layer",
     manifest = ":manifest",
+    preload = ":preload",
     python = "//tools:python3",
     sbom = ":sbom",
     syft = "//tools:syft",
@@ -71,6 +72,21 @@ python.binary(
     visibility = ["PUBLIC"],
 )
 
+# A driver that writes a disk converts it in its own action, rather than handing Buck a raw for the
+# conversion to read.
+python.library(
+    name = "convert-lib",
+    srcs = ["convert.py"],
+    deps = ["//:specs", "//:util"],
+)
+
+python.binary(
+    name = "preload",
+    main = "preload.py",
+    deps = [":convert-lib", "//:specs", "//:util"],
+    visibility = ["PUBLIC"],
+)
+
 python.binary(
     name = "uki",
     main = "uki.py",
@@ -110,13 +126,16 @@ python.test(
         "layer_test.py",
         "manifest.py",
         "manifest_test.py",
+        "preload.py",
+        "preload_test.py",
         "substitute.py",
         "substitute_test.py",
         "uki.py",
         "uki_test.py",
         "version_test.py",
     ],
     deps = [
+        ":convert-lib",
         ":finalize",
         ":version",
         "//:specs",

image/ami.bzl

@@ -86,14 +86,80 @@ def declare_ami(
                 uefi_data = _uefi_data(actions, box_info, secure_boot_certificate),
             ),
         )
+    return _ami_providers(basename = basename, disk = disk, importer = importer, parameters = parameters, version = version)
+
+def declare_ami_from_base(
+    actions: AnalysisActions,
+    *,
+    base_parameters: Artifact,
+    basename: str,
+    disk: Artifact,
+    importer: RunInfo,
+    virtual_size: str | None,
+) -> list[Provider]:
+    """Declare the AMI RegisterImage parameters of a disk built on another, for its `[ami]` subtarget.
+
+    The disk registers with the `base_parameters` of the disk it builds on, under its own name, so that an
+    instance gets the base's UEFI variable store, which this build has no certificate to write.
+    `virtual_size` replaces the base's root volume size.
+    """
+    parameters = actions.declare_output("{}.ami.json".format(basename))
+    actions.dynamic_output_new(
+        _from_base(
+            base = base_parameters,
+            name = basename,
+            out = parameters.as_output(),
+            volume_size = _gib(virtual_size) if virtual_size != None else None,
+        ),
+    )
+    return _ami_providers(basename = basename, disk = disk, importer = importer, parameters = parameters, version = None)
+
+def _from_base_impl(
+    actions: AnalysisActions,
+    base: ArtifactValue,
+    name: str,
+    out: OutputArtifact,
+    volume_size: int | None,
+) -> list[Provider]:
+    register = base.read_json() | {"Description": name, "Name": _ami_name(name)}
+    if volume_size != None:
+        mappings = [dict(mapping) for mapping in register["BlockDeviceMappings"]]
+        mappings[0]["Ebs"] = dict(mappings[0]["Ebs"]) | {"VolumeSize": volume_size}
+        register["BlockDeviceMappings"] = mappings
+    actions.write_json(out, register)
+    return []
+
+_from_base = dynamic_actions(
+    impl = _from_base_impl,
+    attrs = {
+        "base": dynattrs.artifact_value(),
+        "name": dynattrs.value(str),
+        "out": dynattrs.output(),
+        "volume_size": dynattrs.value(int | None),
+    },
+)
+
+def _ami_providers(
+    *,
+    basename: str,
+    disk: Artifact,
+    importer: RunInfo,
+    parameters: Artifact,
+    version: str | None,
+) -> list[Provider]:
     return [
         # Building an AMI builds the disk, as for any other format; only the parameters are published,
         # as a release carries the disk already.
         DefaultInfo(default_output = parameters, other_outputs = [disk]),
-        PublishedInfo(artifacts = {name: PublishedFile(artifact = parameters, kind = "ami")}, image = basename, version = version),
+        PublishedInfo(artifacts = {parameters.basename: PublishedFile(artifact = parameters, kind = "ami")}, image = basename, version = version),
         RunInfo(args = cmd_args(importer, disk, parameters)),
     ]
 
+# The importer that running an `[ami]` subtarget runs.
+AMI_IMPORT_ATTR = {
+    "_ami_import": attrs.dep(providers = [RunInfo], default = "tine//image:ami-import"),
+}
+
 def _with_uefi_data_impl(
     actions: AnalysisActions,
     out: OutputArtifact,