From bc7d813af2163167c6e4c238031cff84ec785f86 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 14 Oct 2026 16:33:26 +0000 Subject: [PATCH 1/5] tools: add a box for the AWS CLI Publishing images to EC2 needs the `aws` CLI, both to log in and to drive the EC2 and EBS APIs. Ship it in its own box, as it bundles an entire boto stack and is *huge*, and it's not always needed. Relaxed entry point so that it uses the host's AWS login. tine then never has to handle credentials itself. Signed-off-by: Martin Pitt --- tools/BUCK | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/BUCK b/tools/BUCK index 27156309..15a51238 100644 --- a/tools/BUCK +++ b/tools/BUCK @@ -171,6 +171,25 @@ box_python_binary( visibility = ["PUBLIC"], ) +# AWS tooling in its own box, as it's big and not always needed. +box.new( + name = "aws.box", + packages = ["awscli2"], + release = _RELEASE, + visibility = ["PUBLIC"], +) + +# PUBLIC: the AWS CLI, relaxed so that it logs in to and reads the host's ~/.aws, e.g. +# `buck run tine//tools:aws -- login`. +command_alias( + name = "aws", + args = ["aws"], + # The box has no pager, neither aws' `less` default nor the host's $PAGER; aws breaks without it. + env = {"AWS_PAGER": ""}, + exe = ":aws.box", + visibility = ["PUBLIC"], +) + _SOURCE_ARGS = [ "--starlark-fmt", "$(location_exec :starlark-fmt)", From f1fb4d45c8cc9beb53496f8193a759b4301bd465 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 15 Oct 2026 14:11:19 +0000 Subject: [PATCH 2/5] image: add a VMDK disk format Useful for VMware or cloud importers such as AWS VM Import (you need an S3 bucket and the `vmimport` role). Like the qcow2 it adjusts to the image's `virtual_size` attribute. However, qemu-img can neither resize a VMDK nor convert into a larger streamOptimized one; so both formats now convert from a qcow2 overlay of that size. Signed-off-by: Martin Pitt --- docs/user/images.md | 18 ++++++++------- examples/image/BUCK | 14 ++++++++---- image/convert.py | 55 +++++++++++++++++++++++++++++++++++---------- image/ddi.bzl | 6 ++--- 4 files changed, 66 insertions(+), 27 deletions(-) diff --git a/docs/user/images.md b/docs/user/images.md index 8a9ea877..d1188981 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -360,7 +360,7 @@ when needed: split-only invocation, `strip_pkgdb` leaves the package database out of the partitions, and `mkfs_options` tunes the filesystems it creates; - `image.disk_convert` re-encodes a raw disk with an explicitly selected box and provides - `image.DiskConversionInfo`; `virtual_size` sizes the disk a qcow2 describes; + `image.DiskConversionInfo`; `virtual_size` sizes the disk a qcow2 or VMDK describes; - `image.bootable` selects a kernel and matching initrd from a logical image, exposed as `[uki]`, `[kernel]`, and `[initrd]` subtargets; - `image.vm` runs the raw image ephemerally with its explicitly selected box's `systemd-vmspawn`, QEMU, @@ -530,10 +530,10 @@ Optional attributes: `image.ddi()`. - `secure_boot_key` (target providing `image.SigningKeyInfo`): Signs the UKIs and systemd-boot; see "Secure Boot signing" below. -- `virtual_size` (size string such as `"20G"`): The size of a virtual disk like `[qcow2]`. A system booted - from it can claim the extra space during first boot, e.g. for a root partition. The raw image stays at - the size of its partitions. As with `image.vm`'s `grow`, the extra space sits past the GPT backup header - until something rewrites the table. A size the partitions do not fit in fails the build. +- `virtual_size` (size string such as `"20G"`): The size of a virtual disk (`[qcow2]` or `[vmdk]`). A + system booted from it can claim the extra space during first boot, e.g. for a root partition. The raw + image stays at the size of its partitions. As with `image.vm`'s `grow`, the extra space sits past the GPT + backup header until something rewrites the table. A size the partitions do not fit in fails the build. - `mkfs_options` (dict of filesystem to option list, default `{}`): Options `mkfs` is given when creating a partition of that filesystem, passed on to `image.ddi()`, which hands each list to `systemd-repart` as `SYSTEMD_REPART_MKFS_OPTIONS_`. Naming a filesystem the disk never formats @@ -619,6 +619,7 @@ views and supply-chain artifacts are lazy subtargets: ├── [directory] ├── [qcow2] ├── [raw.zst] +├── [vmdk] ├── [manifest] ├── [pkgdb] ├── [sbom] @@ -642,8 +643,9 @@ views and supply-chain artifacts are lazy subtargets: The disk and its re-encodings are files named `__.`, so they keep the image identity when copied out of the build, exactly matching the UKI's `__.efi`. -The `[qcow2]` and `[raw.zst]` subtargets re-encode the raw disk into a compact qcow2 or a compressed raw on -demand, each publishing one `image.DiskConversionInfo`; the encodings are reachable only through those +The `[qcow2]`, `[raw.zst]` and `[vmdk]` subtargets re-encode the raw disk into a compact qcow2, a compressed +raw, or a compressed (streamOptimized) VMDK for VMware and cloud importers on demand, each publishing one +`image.DiskConversionInfo`; the encodings are reachable only through those subtargets, because one result cannot carry the same provider type twice. The target has no aggregate bootable-image provider: it returns `image.ImageInfo`, `image.DdiInfo`, `image.InitrdInfo`, `image.ImageDirectoryInfo`, @@ -656,7 +658,7 @@ names instead of composing names of its own. A partition is the exception and tr `PartitionInfo`: its name contains the type and UUID repart assigned, so it exists only after the build, and a consumer reads it back out of the metadata written beside the partition. The ESP is not published: what it carries is transferred by other -means, and no update writes the partition back. `[qcow2]` and `[raw.zst]` publish themselves rather than +means, and no update writes the partition back. The format subtargets publish themselves rather than travelling in the disk's own set, so gathering a release does not build every encoding of it. So do the metadata views: `[sbom]` publishes both formats and `[pkgdb]` one file named after the format its package system keeps the database in, each named after the image they describe, and for a disk `[initrd][sbom]` diff --git a/examples/image/BUCK b/examples/image/BUCK index 3c2fbce0..daacde7a 100644 --- a/examples/image/BUCK +++ b/examples/image/BUCK @@ -267,16 +267,22 @@ tests.vm_smoke( ], ) -# The qcow2 describes a disk of the image's virtual_size, while the raw stays as small as its partitions. +# The qcow2 and vmdk describe a disk of the image's virtual_size with the raw's content, while the raw stays +# as small as its partitions. box.sh_test( name = "virtual-size-test", labels = ["image"], box = "@tine//catalog:fedora.rawhide.box", script = """ -test "$(qemu-img info --output=json "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)["virtual-size"])')" = 2147483648 -test "$(stat -c %s "$2")" -lt $((900 << 20)) +raw=$1 +shift +test "$(stat -c %s "$raw")" -lt $((900 << 20)) +for disk in "$@"; do + test "$(qemu-img info --output=json "$disk" | python3 -c 'import json, sys; print(json.load(sys.stdin)["virtual-size"])')" = 2147483648 + qemu-img compare -q -f raw "$raw" "$disk" +done """, - args = ["$(location :boot-demo.fedora[qcow2])", "$(location :boot-demo.fedora)"], + args = ["$(location :boot-demo.fedora)", "$(location :boot-demo.fedora[qcow2])", "$(location :boot-demo.fedora[vmdk])"], ) # System-extension DDI extending :boot-demo: only the delta above the base lands in the diff --git a/image/convert.py b/image/convert.py index 0e06f054..4a448fc7 100755 --- a/image/convert.py +++ b/image/convert.py @@ -5,7 +5,9 @@ """Convert a composed raw disk image into a distributable output format.""" +import json import subprocess +import tempfile from pathlib import Path from typing import TypedDict @@ -25,12 +27,8 @@ def main(argv: list[str] | None = None) -> None: source = Path(spec["input"]) out = Path(spec["out"]) - if spec["format"] not in ("qcow2", "raw.zst"): - fail(f"convert: unknown format {spec['format']!r}") - if spec["format"] == "qcow2": - # convert drops zero clusters, so the qcow2 stays compact regardless of the raw size. - cmd = ["qemu-img", "convert", "-f", "raw", "-O", "qcow2", str(source), str(out)] - else: + fmt = spec["format"] + if fmt == "raw.zst": cmd = [ "zstd", "-q", "-f", # use all physical cores; don't use --adapt, it breaks reproducibility @@ -41,12 +39,45 @@ def main(argv: list[str] | None = None) -> None: "-15", # well above the default level 3, short of the --ultra maximum "-o", str(out), str(source), ] # fmt: skip - subprocess.run(cmd, check=True) - if size := spec["virtual_size"]: - # Like vmspawn's --grow-image, the added room sits past the GPT backup header until first - # boot's repart rewrites the table. qemu-img refuses to shrink, so a size the partitions do - # not fit in fails here. - subprocess.run(["qemu-img", "resize", "-q", "-f", "qcow2", str(out), size], check=True) + subprocess.run(cmd, check=True) + return + + if fmt == "qcow2": + options = [] + elif fmt == "vmdk": + # streamOptimized is compressed and skips zero grains, and is the variant cloud importers + # such as AWS VM Import take. Not reproducible: qemu-img writes a random content ID. + options = ["-o", "subformat=streamOptimized"] + else: + fail(f"convert: unknown format {fmt!r}") + + with tempfile.TemporaryDirectory() as tmp: + if size := spec["virtual_size"]: + # A VMDK cannot be resized after the fact, and a streamOptimized one cannot be converted + # into at a larger size either, as the raw ends within a grain. So convert from a qcow2 + # overlay of that size instead, which reads the raw and zeros past its end, and costs no + # copy. Like vmspawn's --grow-image, the added room sits past the GPT backup + # header until first boot's repart rewrites the table. + overlay = Path(tmp) / "overlay.qcow2" + # absolute: qemu-img resolves a backing file relative to the overlay's directory + backing = ["-b", str(source.absolute()), "-F", "raw"] + subprocess.run( + ["qemu-img", "create", "-q", "-f", "qcow2", *backing, str(overlay), size], check=True + ) + info = subprocess.run( + ["qemu-img", "info", "--output=json", str(overlay)], check=True, stdout=subprocess.PIPE + ) + # an overlay smaller than its backing file silently cuts it off + if json.loads(info.stdout)["virtual-size"] < source.stat().st_size: + fail(f"convert: the partitions need {source.stat().st_size} bytes, more than {size}") + source_format, source = "qcow2", overlay + else: + source_format = "raw" + # convert drops zero clusters, so the output stays compact regardless of the disk size. + subprocess.run( + ["qemu-img", "convert", "-f", source_format, "-O", fmt, *options, str(source), str(out)], + check=True, + ) if __name__ == "__main__": diff --git a/image/ddi.bzl b/image/ddi.bzl index 6300b46d..4d783eed 100644 --- a/image/ddi.bzl +++ b/image/ddi.bzl @@ -571,16 +571,16 @@ def ddi( ) # The disk conversion output formats, doubling as each artifact's extension. -DISK_FORMATS = ["qcow2", "raw.zst"] +DISK_FORMATS = ["qcow2", "raw.zst", "vmdk"] # The formats whose disk has a size of its own, so it can leave room past the partitions for first boot to # claim. The raw itself stays as small as its partitions. -VIRTUAL_SIZE_FORMATS = ["qcow2"] +VIRTUAL_SIZE_FORMATS = ["qcow2", "vmdk"] VIRTUAL_SIZE_ATTR = attrs.option( attrs.string(), default = None, - doc = 'size of the disk a qcow2 describes, e.g. "20G"; the room past the partitions stays free', + doc = 'size of the disk a qcow2 or vmdk describes, e.g. "20G"; the room past the partitions stays free', ) DiskConversionInfo = provider( From 6e3fbbb963a4357817f4b64637671efde1024e42 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 16 Oct 2026 12:04:22 +0000 Subject: [PATCH 3/5] image: add an AMI bundle format An AMI is not a file format but an EC2 object referring to a disk snapshot and its boot parameters. With every initrd taking credentials from IMDS, any bootable disk can be one, so add an `[ami]` subtarget next to the disk formats: the raw disk and what EC2's `RegisterImage` needs to boot it, with UEFI, IMDSv2, a TPM, and a root volume of the image's virtual_size. Signed-off-by: Martin Pitt --- docs/user/images.md | 4 ++- examples/image/BUCK | 19 +++++++++++ image/ami.bzl | 65 ++++++++++++++++++++++++++++++++++++++ image/compose.bzl | 8 +++++ platforms/architecture.bzl | 3 ++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 image/ami.bzl diff --git a/docs/user/images.md b/docs/user/images.md index d1188981..ca892142 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -620,6 +620,7 @@ views and supply-chain artifacts are lazy subtargets: ├── [qcow2] ├── [raw.zst] ├── [vmdk] +├── [ami] ├── [manifest] ├── [pkgdb] ├── [sbom] @@ -646,7 +647,8 @@ identity when copied out of the build, exactly matching the UKI's `_ +# SPDX-License-Identifier: MPL-2.0 + +"""Amazon Machine Images: a bootable disk with what EC2 needs to register it.""" + +load("//platforms:architecture.bzl", "ARCHITECTURES") + +# What `RegisterImage` accepts in an image name besides letters and digits. +_NAME_PUNCTUATION = "()[] ./-'@_" + +# EC2's label for the root volume, which ties its BlockDeviceMappings to the AMI's RootDeviceName. This +# is the conventional (but arbitrary) value of most Linux AMIs. Does not appear in guests, that's +# `/dev/nvmen1` on Nitro instances. +_ROOT_DEVICE = "/dev/xvda" + +def _ami_name(basename: str) -> str: + return "".join([c if c.isalnum() or c in _NAME_PUNCTUATION else "-" for c in basename.elems()]) + +def _gib(size: str) -> int: + """`size`, a SIZE_PATTERN string read base-1024 as systemd does, in GiB rounded up.""" + exponent = "KMGTPE".find(size[-1]) + 1 + number = float(size[:-1] if exponent else size) + + # Starlark has no math.ceil; flooring the negation rounds up. + return int(-(-number * (1 << 10 * exponent) // (1 << 30))) + +def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Artifact, virtual_size: str | None) -> list[Provider]: + """Declare the AMI bundle of a bootable disk, for its `[ami]` subtarget. + + The bundle holds the disk and the `RegisterImage` parameters, which `aws ec2 register-image + --cli-input-json` takes as they are once the import adds the snapshot. + """ + register = { + "Architecture": ARCHITECTURES[arch].ec2, + "BlockDeviceMappings": [ + { + "DeviceName": _ROOT_DEVICE, + "Ebs": {"DeleteOnTermination": True, "VolumeType": "gp3"}, + }, + ], + # Not "uefi-preferred": instance types without UEFI would boot the disk in legacy BIOS mode. + "BootMode": "uefi", + "Description": basename, + "EnaSupport": True, + "ImdsSupport": "v2.0", + "Name": _ami_name(basename), + "RootDeviceName": _ROOT_DEVICE, + # Limits the AMI to instance types with a NitroTPM, which leaves out bare metal ones. + "TpmSupport": "v2.0", + "VirtualizationType": "hvm", + } + + # Without it, the volume is as large as the snapshot, which is as large as the disk. + if virtual_size != None: + register["BlockDeviceMappings"][0]["Ebs"]["VolumeSize"] = _gib(virtual_size) + + bundle = actions.symlinked_dir( + "ami", + { + # Links, not copies: the disk is sparse, as its partitions are mostly empty. + "disk.raw": disk, + "register.json": actions.write_json("ami-register.json", register), + }, + ) + return [DefaultInfo(default_output = bundle)] diff --git a/image/compose.bzl b/image/compose.bzl index f29f588d..b862be63 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -6,6 +6,7 @@ load("//distribution:defs.bzl", "distribution") load("//package:manager.bzl", "PackageManagerInfo") load("//platforms:architecture.bzl", "ARCHITECTURES") +load(":ami.bzl", "declare_ami") load( ":archive.bzl", "ARCHIVE_ATTRS", @@ -562,6 +563,13 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: ], "uki": [DefaultInfo(default_output = uki.ukis, sub_targets = uki_subtargets(uki)), uki], }) + sub_targets["ami"] = declare_ami( + ctx.actions, + arch = ctx.attrs.arch, + basename = basename, + disk = raw, + virtual_size = ctx.attrs.virtual_size, + ) for conversion in conversions: sub_targets[conversion.format] = [ DefaultInfo(default_output = conversion.image), diff --git a/platforms/architecture.bzl b/platforms/architecture.bzl index 162bfdf7..72d4fdd8 100644 --- a/platforms/architecture.bzl +++ b/platforms/architecture.bzl @@ -7,6 +7,7 @@ The canonical names are systemd's `ConditionArchitecture=` spellings, but with ` Everything with its own naming schema reads it from this table: - `config`: the Buck2 prelude's constraint value, which `select()` keys on +- `ec2`: EC2's AMI `Architecture` - `efi`: ukify's `--efi-arch`, which also names the boot stubs and systemd-boot - `go`: Go's `GOARCH` - `systemd`: the `%a` specifier, which names published artifacts @@ -18,6 +19,7 @@ ARCHITECTURES = { "arm64": struct( config = "config//cpu:arm64", deb = "arm64", + ec2 = "arm64", efi = "aa64", go = "arm64", pacman = "aarch64", @@ -28,6 +30,7 @@ ARCHITECTURES = { "x86_64": struct( config = "config//cpu:x86_64", deb = "amd64", + ec2 = "x86_64", efi = "x64", go = "amd64", pacman = "x86_64", From deceb7fdc57f6ca9c8a669f5479914d3f4b05483 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 16 Oct 2026 12:16:00 +0000 Subject: [PATCH 4/5] image: import an AMI bundle into EC2 Turn the previous commit's bundle into an actual EC2 AMI image: running `[ami]` uploads the disk as an EBS snapshot and registers it with the bundle's parameters, through the AWS CLI with the user's login. `--no-tpm` registers it without a NitroTPM, for bare metal instance types. Only blocks holding data are uploaded, as disks commonly still have lots of empty space on partitions. That needs an unencrypted snapshot, so encrypted ones are not supported for now. Signed-off-by: Martin Pitt --- README.md | 2 + docs/user/aws.md | 113 ++++++++++++++++++++++++ docs/user/images.md | 2 +- image/BUCK | 12 ++- image/ami.bzl | 7 +- image/ami_import.py | 179 +++++++++++++++++++++++++++++++++++++++ image/ami_import_test.py | 67 +++++++++++++++ image/compose.bzl | 2 + 8 files changed, 379 insertions(+), 5 deletions(-) create mode 100644 docs/user/aws.md create mode 100644 image/ami_import.py create mode 100644 image/ami_import_test.py diff --git a/README.md b/README.md index 6a560616..829394ae 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,8 @@ User guides: - [Building images](docs/user/images.md): host requirements, declaring images and layers, output formats, the VM runner +- [Publishing images to AWS EC2](docs/user/aws.md): importing a disk as an AMI, and booting a test + instance - [Building Rust projects](docs/user/cargo.md): building a checked-out Rust project offline, with its crate graph in the executables and image SBOM - [Building Go projects](docs/user/go.md): the same for a checked-out Go project, whose module list go diff --git a/docs/user/aws.md b/docs/user/aws.md new file mode 100644 index 00000000..abd92465 --- /dev/null +++ b/docs/user/aws.md @@ -0,0 +1,113 @@ + + +# Publishing images to AWS EC2 + +An [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AMI) is not a +file format but an object in one EC2 region of one account: it refers to an EBS snapshot holding the disk, +and records how instances boot it (architecture, boot mode, TPM, network driver). tine builds everything +that is a function of the image, and uploads it as a separate `buck run` step that uses your AWS login. + +## The AMI bundle + +Every [image.bootable_disk](images.md#imagebootable_disk) has an `[ami]` subtarget: the raw disk and the +parameters to register it with. `buck build` it to get the bundle, `buck run` it to +[import](#importing-an-ami) it. + +Nothing about the disk is specific to EC2: every initrd supports +[systemd IMDS](https://www.freedesktop.org/software/systemd/man/devel/systemd-imds.html) for passing +[credentials](https://systemd.io/CREDENTIALS/) and carries the `ena` EC2 network driver. The disk is only +as large as its partitions. The image's `virtual_size` sets the default size of an instance's root volume, +rounded up to GiB, and a launch can override that. On first boot, `systemd-repart` then creates or grows +partitions according to their definitions. + +The AMI boots with UEFI only, and has a NitroTPM, which a disk needs if it seals anything to a TPM, such as +a root partition with `Encrypt=tpm2`. That limits it to instance types with NitroTPM support, which leaves +out the bare metal ones. Import with `--no-tpm` for arm64 bare metal; x86 bare metal only boots legacy +BIOS, which a tine disk does not support. + +The AMI is named after the disk's published name: `__`. Characters which +EC2 does not accept in names (such as the `^` in versions) get replaced by `-`. The unchanged name is the +AMI's description. + +TODO: Secure Boot. Without the image's keys in the AMI's UEFI variable store, an instance boots in setup +mode with Secure Boot disabled. + +## Logging in + +tine never handles AWS credentials. It uses the Fedora packaged [AWS CLI](https://aws.amazon.com/de/cli/) +in a box, which runs with your home directory and environment: + +```sh +tine buck run tine//tools:aws -- login +tine buck run tine//tools:aws -- configure set region eu-central-1 +``` + +Any other `aws` command works the same way, so you do not need the AWS CLI installed on the host. + +The import needs these IAM permissions: + +- `ebs:StartSnapshot`, `ebs:PutSnapshotBlock`, `ebs:CompleteSnapshot` +- `ec2:GetEbsEncryptionByDefault`, `ec2:DescribeImages`, `ec2:DescribeSnapshots`, `ec2:RegisterImage`, + `ec2:CreateTags` + +## Importing an AMI + +Run the disk's `[ami]` subtarget: + +```sh +tine buck run '//examples/image:boot-demo.fedora[ami]' +``` + +This prints the new AMI ID. Options: + +- `--name`: register under another name. EC2 requires names to be unique per account and region, and the + import fails before uploading anything if the name is taken, so pass this to import the same image + version again. +- `--region`: import into another region than the configured one. +- `--jobs`: the number of blocks uploaded in parallel, 16 by default. +- `--no-tpm`: register the AMI without a NitroTPM, for arm64 bare metal instance types. + +Disk partitions often have a lot of empty space, e.g. an ESP sized for future updates. Only the blocks +holding data are uploaded. This only works with unencrypted snapshots, so the import refuses to run while +[EBS encryption by default](https://docs.aws.amazon.com/ebs/latest/userguide/encryption-by-default.html) +is enabled for the region. + +If the import fails while uploading, EC2 cancels the incomplete snapshot after an hour. If it fails while +registering, the completed snapshot stays; the import printed its ID, delete it with +`aws ec2 delete-snapshot`. + +## Running an instance with credentials + +System credentials go into the instance's user data, as a JSON object with a `systemd.credentials` list. + +**For testing only**, this one answers the first boot's questions and logs root in on the consoles +without a password, as `image.vm`'s `autologin` does for a local VM: + +```json +{"systemd.credentials": [ + {"name": "firstboot.locale", "text": "C.UTF-8"}, + {"name": "firstboot.timezone", "text": "UTC"}, + {"name": "passwd.hashed-password.root", "text": "!*"}, + {"name": "agetty.autologin", "text": "root"}, + {"name": "login.noauth", "text": "yes"} +]} +``` + +For a root password, put a hash from `mkpasswd` into `passwd.hashed-password.root` and leave out the last +two. An EC2 key pair (`--key-name`) gets specified in the `ssh.authorized_keys.root` credential (for +production instances). + +Launch an instance in the default VPC with that user data: + +```sh +tine buck run tine//tools:aws -- ec2 run-instances \ + --image-id ami-0123456789abcdef0 \ + --instance-type t3.small \ + --user-data file://user-data.json +``` + +Add `--block-device-mappings 'DeviceName=/dev/xvda,Ebs={VolumeSize=20}'` for a volume size different than +the AMI's default. diff --git a/docs/user/images.md b/docs/user/images.md index ca892142..635d250d 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -648,7 +648,7 @@ The `[qcow2]`, `[raw.zst]` and `[vmdk]` subtargets re-encode the raw disk into a raw, or a compressed (streamOptimized) VMDK for VMware and cloud importers on demand, each publishing one `image.DiskConversionInfo`; the encodings are reachable only through those subtargets, because one result cannot carry the same provider type twice. `[ami]` is the raw disk with the -parameters an Amazon Machine Image is registered with. The target has no aggregate +parameters an Amazon Machine Image is registered with, see [aws.md](aws.md). The target has no aggregate bootable-image provider: it returns `image.ImageInfo`, `image.DdiInfo`, `image.InitrdInfo`, `image.ImageDirectoryInfo`, `image.UkiInfo`, and `image.PublishedInfo` independently. diff --git a/image/BUCK b/image/BUCK index da11c8e0..80c217d3 100644 --- a/image/BUCK +++ b/image/BUCK @@ -5,7 +5,7 @@ load("@prelude//:rules.bzl", "constraint_setting", "constraint_value") load("//box:test.bzl", "box_python_test") -load("//python:defs.bzl", "tine_python_binary", "tine_python_library") +load("//python:defs.bzl", "box_python_binary", "tine_python_binary", "tine_python_library") load(":image.bzl", "image_tools") # The one bundle every image rule resolves its drivers through. @@ -108,6 +108,8 @@ box_python_test( name = "test", box = "//tools:dev.box", srcs = [ + "ami_import.py", + "ami_import_test.py", "ddi.py", "ddi_test.py", "finalize_test.py", @@ -211,6 +213,14 @@ tine_python_binary( visibility = ["PUBLIC"], ) +# Run by a bootable disk's `[ami]` subtarget, relaxed, with the host's AWS login. +box_python_binary( + name = "ami-import", + box = "//tools:aws.box", + main = "ami_import.py", + visibility = ["PUBLIC"], +) + # Nothing satisfies this value, so a target requiring it cannot be built. A key whose section has no # token requires it, which keeps it out of `//...` and makes an image that signs with it fail naming # this target. `prelude//:none` would do the same without naming a reason. diff --git a/image/ami.bzl b/image/ami.bzl index 5eb68af4..a2a16cd8 100644 --- a/image/ami.bzl +++ b/image/ami.bzl @@ -24,11 +24,12 @@ def _gib(size: str) -> int: # Starlark has no math.ceil; flooring the negation rounds up. return int(-(-number * (1 << 10 * exponent) // (1 << 30))) -def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Artifact, virtual_size: str | None) -> list[Provider]: +def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Artifact, importer: RunInfo, virtual_size: str | None) -> list[Provider]: """Declare the AMI bundle of a bootable disk, for its `[ami]` subtarget. The bundle holds the disk and the `RegisterImage` parameters, which `aws ec2 register-image - --cli-input-json` takes as they are once the import adds the snapshot. + --cli-input-json` takes as they are once the import adds the snapshot. Running the subtarget + imports the bundle with `importer`. """ register = { "Architecture": ARCHITECTURES[arch].ec2, @@ -62,4 +63,4 @@ def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Art "register.json": actions.write_json("ami-register.json", register), }, ) - return [DefaultInfo(default_output = bundle)] + return [DefaultInfo(default_output = bundle), RunInfo(args = cmd_args(importer, bundle))] diff --git a/image/ami_import.py b/image/ami_import.py new file mode 100644 index 00000000..24f06a97 --- /dev/null +++ b/image/ami_import.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Import an AMI bundle into EC2: upload its disk as an EBS snapshot, then register the image. + +Every AWS call goes through the `aws` CLI, so this uses whatever login and region the CLI is +configured with, and never handles credentials itself. +""" + +import argparse +import base64 +import concurrent.futures +import errno +import hashlib +import json +import os +import subprocess +import sys +import tempfile +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +# The only block size the EBS direct APIs take. +BLOCK_SIZE = 512 * 1024 +GIB = 1024**3 + + +def data_blocks(disk: Path) -> Iterator[tuple[int, bytes]]: + """Yield each block of `disk` that holds anything but zeros, with its index. + + Holes are never read, so a sparse disk costs its allocated size rather than its apparent one. + """ + with disk.open("rb") as f: + fd = f.fileno() + size = os.fstat(fd).st_size + offset = 0 + while offset < size: + try: + start = os.lseek(fd, offset, os.SEEK_DATA) + except OSError as e: + # nothing but a trailing hole past `offset` + if e.errno == errno.ENXIO: + return + raise + end = os.lseek(fd, start, os.SEEK_HOLE) + for index in range(start // BLOCK_SIZE, (end + BLOCK_SIZE - 1) // BLOCK_SIZE): + block = os.pread(fd, BLOCK_SIZE, index * BLOCK_SIZE).ljust(BLOCK_SIZE, b"\0") + if block.count(0) != BLOCK_SIZE: + yield index, block + offset = (end + BLOCK_SIZE - 1) // BLOCK_SIZE * BLOCK_SIZE + + +def checksum(block: bytes) -> str: + """The per-block checksum `PutSnapshotBlock` requires: base64 of its SHA256.""" + return base64.b64encode(hashlib.sha256(block).digest()).decode() + + +class Aws: + """The `aws` CLI, with JSON output and an optional region for every call.""" + + def __init__(self, region: str | None) -> None: + self.region = ["--region", region] if region else [] + # The CLI records a session ID in an SQLite database; the parallel uploads contend for it and + # the losing ones print a traceback. + self.env = os.environ | {"AWS_CLI_SESSION_ID_DISABLED": "true"} + + def __call__(self, *args: str) -> dict[str, Any]: + out = subprocess.run( + ["aws", *args, *self.region, "--output", "json"], + check=True, + stdout=subprocess.PIPE, + text=True, + env=self.env, + ).stdout + return json.loads(out) if out.strip() else {} + + +def upload(aws: Aws, disk: Path, name: str, jobs: int) -> str: + """Write the non-zero blocks of `disk` into a new snapshot and return its ID once completed.""" + # An encrypted snapshot reads back garbage, not zeros, for blocks never written; skipping zero + # blocks is only correct for an unencrypted one, and the account default would encrypt it. + if aws("ec2", "get-ebs-encryption-by-default")["EbsEncryptionByDefault"]: + sys.exit("ami-import: EBS encryption by default is enabled for this account and region; disable it") + + size = (disk.stat().st_size + GIB - 1) // GIB + snapshot = str( + aws( + "ebs", + "start-snapshot", + f"--volume-size={size}", + "--no-encrypted", + f"--description={name}", + f"--tags=Key=Name,Value={name}", + # minutes until EBS cancels it should this die before completing it + "--timeout=60", + )["SnapshotId"] + ) + print(f"ami-import: uploading {disk} into {snapshot} ({size} GiB)", file=sys.stderr) + + def put(index: int, block: bytes, scratch: Path) -> None: + data = scratch / str(index) + data.write_bytes(block) + aws( + "ebs", + "put-snapshot-block", + f"--snapshot-id={snapshot}", + f"--block-index={index}", + f"--block-data={data}", + f"--data-length={BLOCK_SIZE}", + f"--checksum={checksum(block)}", + "--checksum-algorithm=SHA256", + ) + data.unlink() + + written = 0 + with tempfile.TemporaryDirectory() as tmp, concurrent.futures.ThreadPoolExecutor(jobs) as pool: + # Bounded, so that memory holds a few blocks per job rather than the whole disk. + pending: set[concurrent.futures.Future[None]] = set() + for index, block in data_blocks(disk): + if len(pending) >= 2 * jobs: + done, pending = concurrent.futures.wait( + pending, return_when=concurrent.futures.FIRST_COMPLETED + ) + for future in done: + future.result() + pending.add(pool.submit(put, index, block, Path(tmp))) + written += 1 + if written % 100 == 0: + print(f"ami-import: {written} blocks", file=sys.stderr) + for future in concurrent.futures.wait(pending).done: + future.result() + print(f"ami-import: {written} blocks written, completing", file=sys.stderr) + + aws("ebs", "complete-snapshot", f"--snapshot-id={snapshot}", f"--changed-blocks-count={written}") + aws("ec2", "wait", "snapshot-completed", f"--snapshot-ids={snapshot}") + return snapshot + + +def register(aws: Aws, bundle: Path, name: str, snapshot: str, tpm: bool) -> str: + """Register the image from the bundle's parameters, booting `snapshot`, and return its ID.""" + parameters = json.loads((bundle / "register.json").read_text()) + if not tpm: + del parameters["TpmSupport"] + parameters["Name"] = name + parameters["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] = snapshot + parameters["TagSpecifications"] = [{"ResourceType": "image", "Tags": [{"Key": "Name", "Value": name}]}] + with tempfile.NamedTemporaryFile("w", suffix=".json") as f: + json.dump(parameters, f) + f.flush() + return str(aws("ec2", "register-image", "--cli-input-json", f"file://{f.name}")["ImageId"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundle", type=Path, help="the AMI bundle to import") + parser.add_argument("--name", help="the image name, instead of the bundle's; EC2 requires it unique") + parser.add_argument("--region", help="the region to import into, instead of the CLI's configured one") + parser.add_argument("--jobs", type=int, default=16, help="blocks to upload in parallel") + parser.add_argument( + "--no-tpm", + dest="tpm", + action="store_false", + help="register without a NitroTPM, which bare metal instance types lack", + ) + args = parser.parse_args() + + aws = Aws(args.region) + name = args.name or json.loads((args.bundle / "register.json").read_text())["Name"] + # Names are unique per account and region; find out before spending the upload. + if aws("ec2", "describe-images", "--owners=self", f"--filters=Name=name,Values={name}")["Images"]: + sys.exit(f"ami-import: an image named {name!r} exists already; pass another --name") + + snapshot = upload(aws, args.bundle / "disk.raw", name, args.jobs) + print(register(aws, args.bundle, name, snapshot, args.tpm)) + + +if __name__ == "__main__": + main() diff --git a/image/ami_import_test.py b/image/ami_import_test.py new file mode 100644 index 00000000..96a8077b --- /dev/null +++ b/image/ami_import_test.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Tests for picking the blocks of an AMI disk to upload. + +buck test tine//image:test +""" + +import base64 +import hashlib +import tempfile +import unittest +from pathlib import Path + +import ami_import +from ami_import import BLOCK_SIZE + + +class DataBlocksTest(unittest.TestCase): + def disk(self, size: int, writes: dict[int, bytes]) -> Path: + tmp = tempfile.TemporaryDirectory(prefix="ami-import-test.") + self.addCleanup(tmp.cleanup) + path = Path(tmp.name) / "disk.raw" + with path.open("wb") as f: + f.truncate(size) + for offset, data in writes.items(): + f.seek(offset) + f.write(data) + return path + + def test_yields_only_blocks_with_data(self) -> None: + disk = self.disk( + 8 * BLOCK_SIZE, + { + # allocated, but zero: nothing to upload + 0: bytes(BLOCK_SIZE), + # stretches across blocks 2 and 3 + 3 * BLOCK_SIZE - 1: b"ab", + 7 * BLOCK_SIZE + 5: b"c", + }, + ) + self.assertEqual([index for index, _ in ami_import.data_blocks(disk)], [2, 3, 7]) + + def test_blocks_are_whole_and_in_place(self) -> None: + disk = self.disk(4 * BLOCK_SIZE, {BLOCK_SIZE + 3: b"x"}) + [(index, block)] = ami_import.data_blocks(disk) + self.assertEqual(index, 1) + self.assertEqual(block, bytes(3) + b"x" + bytes(BLOCK_SIZE - 4)) + + def test_pads_a_short_last_block(self) -> None: + disk = self.disk(BLOCK_SIZE + 10, {BLOCK_SIZE: b"y"}) + [(index, block)] = ami_import.data_blocks(disk) + self.assertEqual(index, 1) + self.assertEqual(block, b"y" + bytes(BLOCK_SIZE - 1)) + + def test_empty_disk_has_no_blocks(self) -> None: + self.assertEqual(list(ami_import.data_blocks(self.disk(3 * BLOCK_SIZE, {}))), []) + + +class ChecksumTest(unittest.TestCase): + def test_is_base64_sha256(self) -> None: + block = b"z" * BLOCK_SIZE + self.assertEqual(base64.b64decode(ami_import.checksum(block)), hashlib.sha256(block).digest()) + + +if __name__ == "__main__": + unittest.main() diff --git a/image/compose.bzl b/image/compose.bzl index b862be63..ccc177ad 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -568,6 +568,7 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: arch = ctx.attrs.arch, basename = basename, disk = raw, + importer = ctx.attrs._ami_import[RunInfo], virtual_size = ctx.attrs.virtual_size, ) for conversion in conversions: @@ -655,6 +656,7 @@ _bootable_disk_image = rule( ), "verity_key": VERITY_KEY_ATTR, "virtual_size": VIRTUAL_SIZE_ATTR, + "_ami_import": attrs.exec_dep(providers = [RunInfo], default = "tine//image:ami-import"), }, ) From 940a62b0ec927d3eb8743b66bac153fec382afc4 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 16 Oct 2026 12:20:51 +0000 Subject: [PATCH 5/5] image: enroll an AMI's Secure Boot key An instance of a signed AMI booted in Secure Boot setup mode, so it ran without enforcement until something enrolled keys, and whatever booted first could enroll its own. Register the AMI with a UEFI variable store that enrolls the disk's signing certificate, and test it in the secureboot example. Signed-off-by: Martin Pitt --- docs/user/aws.md | 6 ++- examples/image-secureboot/BUCK | 18 +++++++++ image/ami.bzl | 69 +++++++++++++++++++++++++++++----- image/ami_import.py | 4 ++ image/compose.bzl | 8 ++++ tools/BUCK | 6 ++- 6 files changed, 98 insertions(+), 13 deletions(-) diff --git a/docs/user/aws.md b/docs/user/aws.md index abd92465..d304d9b4 100644 --- a/docs/user/aws.md +++ b/docs/user/aws.md @@ -32,8 +32,10 @@ The AMI is named after the disk's published name: `__n1` on Nitro instances. @@ -24,12 +29,24 @@ def _gib(size: str) -> int: # Starlark has no math.ceil; flooring the negation rounds up. return int(-(-number * (1 << 10 * exponent) // (1 << 30))) -def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Artifact, importer: RunInfo, virtual_size: str | None) -> list[Provider]: +def declare_ami( + actions: AnalysisActions, + *, + arch: str, + basename: str, + box: BoxInfo, + disk: Artifact, + importer: RunInfo, + secure_boot_certificate: Artifact | None, + virtual_size: str | None, +) -> list[Provider]: """Declare the AMI bundle of a bootable disk, for its `[ami]` subtarget. The bundle holds the disk and the `RegisterImage` parameters, which `aws ec2 register-image --cli-input-json` takes as they are once the import adds the snapshot. Running the subtarget - imports the bundle with `importer`. + imports the bundle with `importer`. With a `secure_boot_certificate`, the bundle also holds the + UEFI variable store that enrolls it, also as the subtarget's `[uefi-data]`; `box` provides the + tool that writes it. """ register = { "Architecture": ARCHITECTURES[arch].ec2, @@ -55,12 +72,44 @@ def declare_ami(actions: AnalysisActions, *, arch: str, basename: str, disk: Art if virtual_size != None: register["BlockDeviceMappings"][0]["Ebs"]["VolumeSize"] = _gib(virtual_size) - bundle = actions.symlinked_dir( - "ami", - { - # Links, not copies: the disk is sparse, as its partitions are mostly empty. - "disk.raw": disk, - "register.json": actions.write_json("ami-register.json", register), - }, + contents = { + # Links, not copies: the disk is sparse, as its partitions are mostly empty. + "disk.raw": disk, + "register.json": actions.write_json("ami-register.json", register), + } + sub_targets = {} + if secure_boot_certificate != None: + contents["uefi-data"] = _uefi_data(actions, box, secure_boot_certificate) + sub_targets["uefi-data"] = [DefaultInfo(default_output = contents["uefi-data"])] + bundle = actions.symlinked_dir("ami", contents) + return [ + DefaultInfo(default_output = bundle, sub_targets = sub_targets), + RunInfo(args = cmd_args(importer, bundle)), + ] + +def _uefi_data(actions: AnalysisActions, box: BoxInfo, certificate: Artifact) -> Artifact: + """The instance's initial UEFI variable store, which trusts `certificate` as PK, KEK, and db. + + Without it, an instance boots in Secure Boot setup mode, and the first thing it boots may enroll + whichever keys it likes. + """ + uefi_data = actions.declare_output("ami-uefi-data") + actions.run( + cmd_args( + box_run(box = box), + "virt-fw-vars", + "--set-pk", + _SIGNATURE_OWNER, + certificate, + "--add-kek", + _SIGNATURE_OWNER, + certificate, + "--add-db", + _SIGNATURE_OWNER, + certificate, + "--output-aws", + uefi_data.as_output(), + ), + category = "uefi_data", ) - return [DefaultInfo(default_output = bundle), RunInfo(args = cmd_args(importer, bundle))] + return uefi_data diff --git a/image/ami_import.py b/image/ami_import.py index 24f06a97..024b266a 100644 --- a/image/ami_import.py +++ b/image/ami_import.py @@ -145,6 +145,10 @@ def register(aws: Aws, bundle: Path, name: str, snapshot: str, tpm: bool) -> str parameters["Name"] = name parameters["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] = snapshot parameters["TagSpecifications"] = [{"ResourceType": "image", "Tags": [{"Key": "Name", "Value": name}]}] + # Only a disk signed for Secure Boot has one, see `_uefi_data` in ami.bzl. + uefi_data = bundle / "uefi-data" + if uefi_data.exists(): + parameters["UefiData"] = uefi_data.read_text().strip() with tempfile.NamedTemporaryFile("w", suffix=".json") as f: json.dump(parameters, f) f.flush() diff --git a/image/compose.bzl b/image/compose.bzl index ccc177ad..ec3e76ba 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -3,6 +3,7 @@ """Convenience compositions for image products.""" +load("//box:runtime.bzl", "BoxInfo") load("//distribution:defs.bzl", "distribution") load("//package:manager.bzl", "PackageManagerInfo") load("//platforms:architecture.bzl", "ARCHITECTURES") @@ -567,8 +568,14 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: ctx.actions, arch = ctx.attrs.arch, basename = basename, + box = ctx.attrs._ami_box[BoxInfo], disk = raw, importer = ctx.attrs._ami_import[RunInfo], + # A key held by a token has a URI, not a file, for a certificate; every key rule publishes the + # file as `[cert]`. + secure_boot_certificate = ( + ctx.attrs.secure_boot_key[DefaultInfo].sub_targets["cert"][DefaultInfo].default_outputs[0] if secure_boot_key != None else None + ), virtual_size = ctx.attrs.virtual_size, ) for conversion in conversions: @@ -656,6 +663,7 @@ _bootable_disk_image = rule( ), "verity_key": VERITY_KEY_ATTR, "virtual_size": VIRTUAL_SIZE_ATTR, + "_ami_box": attrs.exec_dep(providers = [BoxInfo], default = "tine//tools:aws.box"), "_ami_import": attrs.exec_dep(providers = [RunInfo], default = "tine//image:ami-import"), }, ) diff --git a/tools/BUCK b/tools/BUCK index 15a51238..5df8e6c7 100644 --- a/tools/BUCK +++ b/tools/BUCK @@ -174,7 +174,11 @@ box_python_binary( # AWS tooling in its own box, as it's big and not always needed. box.new( name = "aws.box", - packages = ["awscli2"], + packages = [ + "awscli2", + # virt-fw-vars, for an AMI's UEFI variable store + "python3-virt-firmware", + ], release = _RELEASE, visibility = ["PUBLIC"], )