diff --git a/AGENTS.md b/AGENTS.md index cf6b8f1b..370b78c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ Cell layout and architecture: [the design plan](docs/design/architecture.md). Us `starlark_fmt` owns the layout of every `.bzl` and `BUCK` file and wraps at 160 columns. - Leave a trailing comma after the last argument to keep a signature or call broken across lines. -- It sorts dict keys unconditionally; mark a dict whose order is a contract (`disk.bzl:partition` feeds +- It sorts dict keys unconditionally; mark a dict whose order is a contract (`ddi.bzl:partition` feeds partition UUIDs) `# @unsorted-dict-items`. - It drops load symbols it cannot see used; annotate type-only imports `# @unused`. diff --git a/README.md b/README.md index 87c63631..b77e0775 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,8 @@ image.rootfs_archive( package_manager = ":image.package-manager", packages = ["bash", "coreutils"], ops = [ - image.copy("marker.txt", "/etc/tine/marker"), - image.run(["/usr/bin/bash", "-c", "echo built-by-tine >> /etc/tine/marker"], chroot = True), + image.ops.copy("marker.txt", "/etc/tine/marker"), + image.ops.run(["/usr/bin/bash", "-c", "echo built-by-tine >> /etc/tine/marker"], chroot = True), ], format = "tar", ) diff --git a/catalog/BUCK b/catalog/BUCK index 967bdc82..87776793 100644 --- a/catalog/BUCK +++ b/catalog/BUCK @@ -164,7 +164,7 @@ box.new( "e2fsprogs", "erofs-utils", "kmod", - # image/kmod.py loads libkmod.so.2 with dlopen(), and no package in this list depends on it. + # kmod/kmod.py loads libkmod.so.2 with dlopen(), and no package in this list depends on it. "libkmod2", # The relaxed sandbox binds the host's nsswitch.conf. Without libnss_systemd, getpwuid() cannot find # a systemd-homed caller, and ssh-keygen aborts. diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 3f138bb0..2dfdeccd 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -51,8 +51,8 @@ tine//package_system/ one directory per package system: its repository, reso extractor, and optional indexer and builder tine//box/ box bootstrap and sandbox command construction tine//rootfs/ bind/overlay mounting and stored-delta translation -tine//image/ layers, boot artifacts, composition macros, and VM runners -tine//image_format/ archive, directory, and raw-disk output rules and drivers +tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and + drivers, and VM runners tine//git/ pinned Git repositories with local checkout overrides tine//cargo/ vendored crate trees and offline Rust source builds tine//go/ go.sum-verified module fetches and offline Go source builds @@ -1126,7 +1126,7 @@ allowing reusable helpers to return ordered groups of operations. Every rule tha wrapped in a declaration macro that flattens them the same way, so a helper returning an ordered group works identically in `image.layer()` and in a composition; the underlying attribute stays a flat, typed list so Buck can track every embedded source dependency. An `image.install()` target carries `packages` -beside its operations, so a reusable target declares what it needs and `image.install_from()` folds that +beside its operations, so a reusable target declares what it needs and `image.ops.install_from()` folds that into the installing layer's own request. `install_langs` narrows the install to the translated files of the named languages, and `install_docs` @@ -1170,7 +1170,7 @@ locally backed in later solves. Unlike a static `local_repository`, only the pac pulls in are built; the universe target itself never forces a package build. Logical images and terminal outputs are separate rule families. Terminal rules merge the stack only when -needed. The catalog of terminal rules (`image.archive`, `image.directory`, `image.uki`, `image.repart`, +needed. The catalog of terminal rules (`image.archive`, `image.directory`, `image.uki`, `image.ddi`, `image.bootable`, `image.sysext`, and `image.vm`) and the `image.rootfs_archive`, `image.sysext_image`, and `image.bootable_disk` composition rules are documented in [images.md](../user/images.md). All of them, with the operation helpers and conventional partition layouts, are members of the `image` struct exported @@ -1207,30 +1207,30 @@ multi-threaded output is byte-identical to its single-threaded output, so this s uses it, while the kernel-modules cpio that `uki.py` appends stays raw because the distribution already ships each module compressed; the kernel unpacks the concatenation as independently compressed segments. -`image.repart` deliberately distinguishes `definitions` from `partitions`. Definitions describe new +`image.ddi` deliberately distinguishes `definitions` from `partitions`. Definitions describe new partitions to populate directly from `image.ImageInfo`: repart mounts the delta stack with a disposable overlay upper instead of first copying a directory artifact. Partition inputs are -`image.RepartInfo` outputs from an earlier split call; their blocks are copied into the new disk with +`image.DdiInfo` outputs from an earlier split call; their blocks are copied into the new disk with their resolved type and UUID preserved. Calls emit a disk by default. `split = True` additionally exposes each newly defined partition with normalized metadata alongside its block artifact; `disk = False` makes such a call partition-only. No partial disk is passed between -actions. The partition artifacts are portable, so `image.RepartInfo` carries no box: repart uses the +actions. The partition artifacts are portable, so `image.DdiInfo` carries no box: repart uses the destination `image.ImageInfo.box`, while standalone conversion and VM rules select a box explicitly. `image.directory` is an independent terminal view and is never an input to repart. -Partition layouts are always explicit inputs; neither `image.repart` nor `image.bootable_disk` chooses one +Partition layouts are always explicit inputs; neither `image.ddi` nor `image.bootable_disk` chooses one implicitly. The reusable conventional layouts are listed in [images.md](../user/images.md). Partition labels may contain `{image_id}` and `{version}` placeholders. `image.format_partition_labels()` -renders them — `image.bootable_disk()` calls it with its own image identity, raw `image.repart()` users +renders them — `image.bootable_disk()` calls it with its own image identity, raw `image.ddi()` users call it themselves, and unrendered placeholders fail the build. The usr-verity layouts carry such labels to produce the `_[_verity[_sig]]` names that systemd-sysupdate A/B slot matching expects. Rendered labels pass through `image.partition()` again, so GPT's 36-character label limit is enforced on the final value. Verity data, hash, and optional signature partitions are produced together in the split action. The root or -usr hash is an artifact because its value is known only after execution. `image.RepartInfo.root_hash` carries +usr hash is an artifact because its value is known only after execution. `image.DdiInfo.root_hash` carries an optional `image.RootHashInfo`, allowing `image.uki` to consume the hash without another top-level provider. One split call may produce at most one such hash. This artifact boundary also ensures the final disk contains exactly the partition bytes whose hash was embedded in the UKI. The hash is also available @@ -1305,7 +1305,7 @@ UKI is named `__.efi` from the image identity. For now an image holds exactly one kernel — the name (and sysupdate's matching of it) could not distinguish more. If several kernels per image ever become a requirement, add naming configuration to `image.uki()` to disambiguate them. The ESP layer copies the UKI -directory into `EFI/Linux` and includes the operations returned by `image.install_systemd_boot()`. Those +directory into `EFI/Linux` and includes the operations returned by `image.ops.install_systemd_boot()`. Those create the ESP path, run the box's `bootctl` with its paths in the command environment, and remove the random seed. The final repart action creates the ESP while copying the previously split system partitions into the same disk. It splits nothing itself: the system partitions arrive already split, and nothing updates @@ -1330,7 +1330,7 @@ artifact, so one provider covers every encoding instead of one provider type per Bootability and output format remain independent terminal capabilities, but the artifacts declared by one `image.bootable_disk()` invocation form one concrete product. The composition publishes one target at -the requested name. Its default output and `image.RepartInfo.disk` are the raw disk; its other terminal +the requested name. Its default output and `image.DdiInfo.disk` are the raw disk; its other terminal views, supply-chain metadata, and constituents are lazy subtargets: @@ -1358,8 +1358,8 @@ supply-chain metadata, and constituents are lazy subtargets: ``` Typed providers are the composition API; subtargets are the command-line interface. There is no aggregate -bootable-image provider. The rule returns its completed `image.ImageInfo`, `image.RepartInfo`, -`image.InitrdInfo`, `image.ImageDirectoryInfo`, and `image.UkiInfo` independently. `image.RepartInfo` +bootable-image provider. The rule returns its completed `image.ImageInfo`, `image.DdiInfo`, +`image.InitrdInfo`, `image.ImageDirectoryInfo`, and `image.UkiInfo` independently. `image.DdiInfo` holds the composed raw disk, its independent partitions, and optional `image.RootHashInfo`, rather than exposing separate top-level providers for these facets. Its qcow2 and compressed-raw encodings are reachable only through their subtargets, each publishing one @@ -1376,7 +1376,7 @@ describe, so a release gathers what a scanner reads while an image's own publish update transfers. `image.bootable_disk` is one rule that owns the complete composition action graph. Provider-oriented action -helpers are shared with the standalone `image.layer`, `image.repart`, `image.uki`, and terminal-format +helpers are shared with the standalone `image.layer`, `image.ddi`, `image.uki`, and terminal-format rules, so the composition reuses their implementations without declaring private sibling targets or forwarding through a result rule. This aggregation remains scoped to the concrete bootable-image product; it does not restore a generic `image_result` around every logical `image.ImageInfo`. @@ -1426,15 +1426,14 @@ gets the content its operations ask for and nothing else, which is the same reas package state alone. Each generator is a no-op for an image that carries none of what it acts on, so none of them needs -per-distribution knowledge: `image.depmod()` is silent without kernels, `image.hwdb()` without `hwdb.d`, -`image.locale_gen()` -without an `/etc/locale.gen` asking for something. That is what lets the compositions, which declare a whole -product rather than one layer, end every image they build with all three at their defaults without knowing -what went into it. A generator named in `ops` replaces the composition's copy instead of adding a second, so -placing or configuring one stays possible. `sysext_image` is the exception that generates nothing: an -extension merges onto a system it does not own, where a database built from the extension's own tree would -shadow that system's while describing only what the extension carries, exactly as its package database -would. +per-distribution knowledge: `image.ops.depmod()` is silent without kernels, `image.ops.hwdb()` without +`hwdb.d`, `image.ops.locale_gen()` without an `/etc/locale.gen` asking for something. That is what lets the +compositions, which declare a whole product rather than one layer, end every image they build with all three +at their defaults without knowing what went into it. A generator named in `ops` replaces the composition's +copy instead of adding a second, so placing or configuring one stays possible. `sysext_image` is the +exception that generates nothing: an extension merges onto a system it does not own, where a database built +from the extension's own tree would shadow that system's while describing only what the extension carries, +exactly as its package database would. Their tools are the box's, applied to the mounted image through the `--root` interface systemd gives them, which is the same relationship every other layer driver has to the image and is what lets an image that @@ -1754,7 +1753,7 @@ Useful implementation entry points: `package/{href,installer,transaction}.py` - each package system's `rules.bzl` and drivers under `package_system/`, listed in its own section above - `box/{build,runtime}.bzl`, `box/sandbox.py`, and `rootfs/rootfs.py` -- `image/{image,compose,defs,sign,vm}.bzl` and `image_format/{archive,boot,disk,sysext,uki}.bzl` +- `image/{image,compose,defs,sign,vm,archive,boot,ddi,sysext}.bzl` - `cargo/{defs,rules,vendor}.bzl` and `cargo/{lock,vendor,build}.py` - `go/rules.bzl` and `go/{fetch,build}.py` - `tools/catalog.py` and `catalog/BUCK` diff --git a/docs/user/cargo.md b/docs/user/cargo.md index f53f1ba0..e94cda2e 100644 --- a/docs/user/cargo.md +++ b/docs/user/cargo.md @@ -45,14 +45,14 @@ cargo.package( - `box` is the build environment, declared by the consumer because only the consumer knows what its projects link against. -The binaries are ordinary artifacts, so an image installs one with an `image.copy()` operation: +The binaries are ordinary artifacts, so an image installs one with an `image.ops.copy()` operation: ```Starlark image.rootfs_archive( name = "demo", package_manager = ":image.package-manager", packages = ["glibc"], - ops = [image.copy(":hello[hello-cli]", "/usr/bin/hello-cli")], + ops = [image.ops.copy(":hello[hello-cli]", "/usr/bin/hello-cli")], format = "tar", ) ``` diff --git a/docs/user/go.md b/docs/user/go.md index 7a6ff652..7e7c01c7 100644 --- a/docs/user/go.md +++ b/docs/user/go.md @@ -75,7 +75,7 @@ go.package( executables, which the kernel loads at a random address on each run. Unset, the box toolchain's default applies, which is `"exe"` on x86-64 and AArch64 Linux. -The binaries are ordinary artifacts, so an image installs one with an `image.copy()` operation; see the +The binaries are ordinary artifacts, so an image installs one with an `image.ops.copy()` operation; see the example. For example, etcd's build scripts build the server in `server/`, which contains the server module. The diff --git a/docs/user/images.md b/docs/user/images.md index 116b34d7..11799542 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -77,8 +77,8 @@ Everything below is re-exported from one facade, so a `BUCK` file needs a single load("@tine//image:defs.bzl", "image") ``` -The modules behind the facade (`image.bzl`, `compose.bzl`, and the `image_format` package) are -implementation structure and may be rearranged; load them directly only from inside the cell. +The modules behind the facade, such as `image.bzl` and `compose.bzl`, are implementation structure and +may be rearranged; load them directly only from inside the cell. An initial image fixes one package manager for its whole lifetime; every derived image and terminal output inherits it and its box. Take a catalog package manager, optionally extend it with project repositories, @@ -106,7 +106,7 @@ image.layer( image.layer( name = "project-configured.image", parent = ":project.image", - ops = [image.run(["/usr/bin/project", "configure"], chroot = True)], + ops = [image.ops.run(["/usr/bin/project", "configure"], chroot = True)], ) ``` @@ -134,28 +134,29 @@ ordered operation sequence in one action and persists exactly one delta: request before any of its operations run, so every operation sees what the layer adds. `package_sets` names symbolic sets the image's OS release supplies and `packages` names concrete ones; a layer may use both, and duplicates between them collapse. -- `image.run([...])` executes a command against the image. By default the box supplies the userspace and +- `image.ops.run([...])` executes a command against the image. By default the box supplies the userspace and the image is mounted at `/buildroot`; with `chroot = True` the command runs inside the image with its own binaries instead. Either takes an `env` argument that overlays variables on that command's environment. An argument names a build artifact by being one, or by spelling `$(location //target)` in a BUCK file, and it resolves the same in both modes: a chrooted command gets the project bind-mounted at a fixed path under `/run`, which is also its working directory. Running a script the repository - owns is therefore just `image.run(["/usr/bin/bash", "$(location :setup.sh)"], chroot = True)`. Buck's macro - parser claims `$(...)`, so a shell substitution has to be written `\$(...)`; an unescaped one fails to - parse rather than silently reaching the shell. -- `image.python([...])` runs a python script against the image without the image needing python: it is an - `image.run` of the relocatable interpreter Buck already pins for its own bootstrap, named through the + owns is therefore just `image.ops.run(["/usr/bin/bash", "$(location :setup.sh)"], chroot = True)`. Buck's + macro parser claims `$(...)`, so a shell substitution has to be written `\$(...)`; an unescaped one fails + to parse rather than silently reaching the shell. +- `image.ops.python([...])` runs a python script against the image without the image needing python: it is an + `image.ops.run` of the relocatable interpreter Buck already pins for its own bootstrap, named through the project like any other artifact. It takes the same `env` and `chroot` arguments, so the script either sees the image at `/buildroot` or has it as its own root; in the chrooted case nothing of the interpreter reaches the delta, because the project bind carrying it lives under `/run`. -- `image.copy` introduces a declared Buck artifact at an absolute image path; `image.mkdir` and - `image.symlink` mutate the same root. `image.remove` treats its absolute path as a glob pattern, +- `image.ops.copy` introduces a declared Buck artifact at an absolute image path; `image.ops.mkdir` and + `image.ops.symlink` mutate the same root. `image.ops.remove` treats its absolute path as a glob pattern, including recursive `**`, and removes every matching file, symlink, or directory tree. A pattern that matches nothing does nothing. -- `image.depmod()`, `image.hwdb()` and `image.locale_gen()` build the state installed packages only describe - (below). -- `image.install_from` installs what another target needs and applies the operations it attaches (see below). +- `image.ops.depmod()`, `image.ops.hwdb()` and `image.ops.locale_gen()` build the state installed packages + only describe (below). +- `image.ops.install_from` installs what another target needs and applies the operations it attaches (see + below). `image.layer()` recursively flattens operation lists, allowing reusable helpers to return ordered groups of operations; `image.rootfs_archive`, `image.sysext_image`, and `image.bootable_disk` accept the same nested @@ -175,44 +176,43 @@ image.layer( package_manager = ":image.package-manager", package_sets = ["bootable"], ops = [ - image.install_from(":project.install"), - image.depmod(), - image.hwdb(), + image.ops.install_from(":project.install"), + image.ops.depmod(), + image.ops.hwdb(), ], ) ``` -- `image.depmod()` rebuilds `modules.dep` and its `.bin` indexes for every kernel the image installs, and +- `image.ops.depmod()` rebuilds `modules.dep` and its `.bin` indexes for every kernel the image installs, and does nothing for an image that installs none. It runs the image's own `depmod`, because the tool reads its search-order configuration from absolute paths that `--basedir` does not move, and writes index files whose compatibility is its own kmod's business. An image with modules but no `depmod` fails the build by name rather than shipping a stale index. -- `image.hwdb(usr = True, strict = True)` compiles `hwdb.d` into the binary database udev actually reads. It - writes `/usr/lib/udev/hwdb.bin`, where the image ships it and nothing writable shadows it, and drops +- `image.ops.hwdb(usr = True, strict = True)` compiles `hwdb.d` into the binary database udev actually reads. + It writes `/usr/lib/udev/hwdb.bin`, where the image ships it and nothing writable shadows it, and drops the `/etc` copy; `usr = False` writes that copy instead. `strict = False` accepts a source file the image cannot parse. An image with no `hwdb.d` is left alone. -- `image.locale_gen()` runs the image's own `locale-gen` when `/etc/locale.gen` asks for something, which is - how Debian and Arch generate locales; a distribution that ships them as packages has no such file and +- `image.ops.locale_gen()` runs the image's own `locale-gen` when `/etc/locale.gen` asks for something, which + is how Debian and Arch generate locales; a distribution that ships them as packages has no such file and the operation does nothing. -`image.hwdb()` is the one that runs a box tool against the mounted image, exactly as `image.run()` does by -default, so an image that installs no systemd of its own still gets a database; the other two must be +`image.ops.hwdb()` is the one that runs a box tool against the mounted image, exactly as `image.ops.run()` +does by default, so an image that installs no systemd of its own still gets a database; the other two must be the image's own. What they write is captured by the layer that runs them, so it is built once and cached rather than repeated by every terminal output. -`image.rootfs_archive`, `image.bootable_disk`'s root filesystem, and `image.initrd` end their operations -with `image.depmod()`, `image.hwdb()` and `image.locale_gen()` at their defaults, because a composition -declares a whole product rather than one layer: the three cover everything a package can leave described, -and each does nothing on -an image carrying none of it. They land on the layer the composition declares, so a composition handed a -`parent` and given no operations or packages of its own declares no layer and places none; that parent is -where they belong, after the operations whose packages they read. An initrd generates before it prunes, so -it ships the compiled database rather than the sources it was compiled from. Naming a generator in `ops` -overrides the composition's copy instead of adding a second, so `image.hwdb(usr = False)`, or an -`image.depmod()` placed before the operations that strip modules, is taken exactly as written. -`image.sysext_image` runs none of them: an extension merges onto a system it does not own, where a database -built from the extension's own tree would -shadow that system's while describing only what the extension carries. +`image.rootfs_archive`, `image.bootable_disk`'s root filesystem, and `image.initrd` end their operations with +`image.ops.depmod()`, `image.ops.hwdb()` and `image.ops.locale_gen()` at their defaults, because a +composition declares a whole product rather than one layer: the three cover everything a package can leave +described, and each does nothing on an image carrying none of it. They land on the layer the composition +declares, so a composition handed a `parent` and given no operations or packages of its own declares no layer +and places none; that parent is where they belong, after the operations whose packages they read. An initrd +generates before it prunes, so it ships the compiled database rather than the sources it was compiled from. +Naming a generator in `ops` overrides the composition's copy instead of adding a second, so +`image.ops.hwdb(usr = False)`, or an `image.ops.depmod()` placed before the operations that strip modules, is +taken exactly as written. `image.sysext_image` runs none of them: an extension merges onto a system it does +not own, where a database built from the extension's own tree would shadow that system's while describing +only what the extension carries. ### Declaring the initrd @@ -224,7 +224,7 @@ image.initrd( name = "os.initrd", package_manager = ":image.package-manager", packages = ["cryptsetup"], - ops = [image.copy(":modprobe.conf", "/usr/lib/modprobe.d/local.conf")], + ops = [image.ops.copy(":modprobe.conf", "/usr/lib/modprobe.d/local.conf")], compression = "none", ) @@ -248,35 +248,35 @@ be shared by several disk images. ### Attaching install operations to a target How a project installs is a property of the project, not of each image carrying it. `image.install()` -attaches packages and operations to a target, and an image applies both with one `image.install_from()`: +attaches packages and operations to a target, and an image applies both with one `image.ops.install_from()`: ```Starlark image.install( name = "project.install", packages = ["glibc"], ops = [ - image.copy(":project[project-cli]", "/usr/bin/project-cli"), - image.copy(":project.checkout[tmpfiles.d]", "/usr/lib/tmpfiles.d"), + image.ops.copy(":project[project-cli]", "/usr/bin/project-cli"), + image.ops.copy(":project.checkout[tmpfiles.d]", "/usr/lib/tmpfiles.d"), ], ) image.bootable_disk( name = "os", package_sets = ["bootable"], - ops = [image.install_from(":project.install")], + ops = [image.ops.install_from(":project.install")], ... ) ``` -`image.copy()` copies a checkout path as it is on disk, including the files that Git ignores, such as build -output or editor backups. The image then contains the ignored files. Build release images in a fresh +`image.ops.copy()` copies a checkout path as it is on disk, including the files that Git ignores, such as +build output or editor backups. The image then contains the ignored files. Build release images in a fresh clone, or after `git clean -fdx`. A target declaring its own `packages` is what keeps every image carrying it from having to know: they join the installing layer's own request, so two targets that both need packages compose without the caller merging anything. Any operation may be attached, not only copies, and the operations are spliced in place, so the image still decides where in its own order they land. An `image.install` target may -itself `image.install_from()` another, which composes; a cycle is rejected by Buck as a target cycle. +itself `image.ops.install_from()` another, which composes; a cycle is rejected by Buck as a target cycle. ### Installing a file a project ships as a template @@ -291,7 +291,7 @@ image.substitute( replacements = {"@bindir@": "/usr/bin", "@port@": "555"}, ) -image.copy(":project-http.service", "/usr/lib/systemd/system/project-http.service") +image.ops.copy(":project-http.service", "/usr/lib/systemd/system/project-http.service") ``` The output is named after the target. Each placeholder has to appear in the template, so a project @@ -349,8 +349,8 @@ when needed: kernel are signed for Secure Boot. With a `sign_expected_pcr_key`, a signed expected-PCR 11 policy covers each profile; set its `sign_expected_pcr` to false to opt out; -- `image.repart` renders ordered Starlark partition definitions and uses offline `systemd-repart` to create a - GPT disk and independent partition artifacts in one `image.RepartInfo`; its disk field is absent for a +- `image.ddi` renders ordered Starlark partition definitions and uses offline `systemd-repart` to create a + GPT disk and independent partition artifacts in one `image.DdiInfo`; its disk field is absent for a split-only invocation, `output_size` composes the disk with free space behind its partitions, and `strip_pkgdb` leaves the package database out of them, and `mkfs_options` tunes the filesystems it creates; @@ -416,24 +416,24 @@ Required attributes: passed on to `image.layer`. - `definitions` (list of `image.partition()` descriptors): Partition layout; `image.DEFAULT_ROOT_PARTITIONS`, `image.DEFAULT_USR_VERITY_PARTITIONS`, and `image.DEFAULT_SIGNED_USR_VERITY_PARTITIONS` are reusable - conventional layouts; it must contain system and ESP partitions; passed on to `image.repart()`. + conventional layouts; it must contain system and ESP partitions; passed on to `image.ddi()`. Optional attributes: -- `disk_seed` (string): Seeds stable partition UUIDs; passed on to `image.repart()`. +- `disk_seed` (string): Seeds stable partition UUIDs; passed on to `image.ddi()`. - `verity_key` (target providing `image.SigningKeyInfo`): Signs the verity signature partition; passed on to - `image.repart()`. + `image.ddi()`. - `secure_boot_key` (target providing `image.SigningKeyInfo`): Signs the UKIs and systemd-boot; see "Secure Boot signing" below. - `output_size` (size string such as `"20G"`): Ships the disk at this size instead of at the size its partitions need, so an installed system finds room past them and does not have to resize its medium - before first boot; passed on to `image.repart()`. The partitions keep the sizes their definitions ask + before first boot; passed on to `image.ddi()`. The partitions keep the sizes their definitions ask for, and the composed file is enlarged behind the last one, so the added room costs nothing on disk and, as with `image.vm`'s `grow`, sits past the GPT backup header until something rewrites the table. A size the partitions do not fit in fails the build. - `mkfs_options` (dict of filesystem to option list, default `{}`): Options `mkfs` is given when - creating a partition of that filesystem, passed on to `image.repart()`, which hands each list to + creating a partition of that filesystem, passed on to `image.ddi()`, which hands each list to `systemd-repart` as `SYSTEMD_REPART_MKFS_OPTIONS_`. Naming a filesystem the disk never formats fails the build rather than going nowhere, and an option holding whitespace is refused because repart splits the variable on it. This is where a disk's compression is really decided: an `image.partition()`'s @@ -444,7 +444,7 @@ Optional attributes: entry with the wall clock even under `SOURCE_DATE_EPOCH`. - `strip_pkgdb` (bool, default `False`): Leaves the package database out of the system partitions, for a system that ships without its package manager and never resolves a package again; passed on to - `image.repart()`. The logical image keeps it, so `[pkgdb]` still captures the database and `[sbom]` still + `image.ddi()`. The logical image keeps it, so `[pkgdb]` still captures the database and `[sbom]` still reports every installed package rather than what a binary scan can guess. The ESP carries no database either way. - `sign_expected_pcr_key` (target providing `image.SigningKeyInfo`): Seals the expected-PCR policy; @@ -543,7 +543,7 @@ identity when copied out of the build, exactly matching the UKI's `_ /etc/tine/marker"], chroot = True), + image.ops.run(["/usr/bin/bash", "-c", "echo built-by-\\$(echo tine) > /etc/tine/marker"], chroot = True), # A repository-owned script, run inside the image. The project is mounted there, so # `$(location)` names the script and its data exactly as it would outside. - image.run( + image.ops.run( [ "/usr/bin/bash", "$(location :configure.sh)", @@ -86,11 +86,11 @@ image.rootfs_archive( chroot = True, ), # The same, in python, which this image does not install: the interpreter is Buck's own. - image.python(["$(location :configure.py)", "$(location :marker.txt)", "from-python-argument"], chroot = True), + image.ops.python(["$(location :configure.py)", "$(location :marker.txt)", "from-python-argument"], chroot = True), # Unchrooted, the same interpreter sees the image at /buildroot, like any box command. - image.python(["-c", "from pathlib import Path; Path('/buildroot/etc/tine/from-python-box').write_text('x\\n')"]), + image.ops.python(["-c", "from pathlib import Path; Path('/buildroot/etc/tine/from-python-box').write_text('x\\n')"]), # Ensure we don't leak the /tine sandbox mount into image builds - image.run(["/usr/bin/test", "!", "-d", "/tine"], chroot = True), + image.ops.run(["/usr/bin/test", "!", "-d", "/tine"], chroot = True), ], format = "tar", ) @@ -109,8 +109,8 @@ image.layer( package_manager = ":image.package-manager", packages = ["bash", "coreutils"], ops = [ - image.write_file("/etc/tine/from-base-kept", "base"), - image.write_file("/etc/tine/from-base-gone", "base"), + image.ops.write_file("/etc/tine/from-base-kept", "base"), + image.ops.write_file("/etc/tine/from-base-gone", "base"), ], ) @@ -118,8 +118,8 @@ image.layer( name = "chained-child.layer", parent = ":chained-base", ops = [ - image.write_file("/etc/tine/from-child", "child"), - image.remove("/etc/tine/from-base-gone"), + image.ops.write_file("/etc/tine/from-child", "child"), + image.ops.remove("/etc/tine/from-base-gone"), ], ) @@ -135,7 +135,7 @@ image.layer( package_manager = ":image.package-manager", packages = ["bash"], ops = [ - image.run(["/usr/bin/bash", "-c", "echo base > /etc/from-base-install"], chroot = True), + image.ops.run(["/usr/bin/bash", "-c", "echo base > /etc/from-base-install"], chroot = True), ], ) @@ -144,7 +144,7 @@ image.layer( parent = ":layered-install-base", packages = ["coreutils"], ops = [ - image.run(["/usr/bin/bash", "-c", "ls /usr/bin/ls > /etc/from-child-install"], chroot = True), + image.ops.run(["/usr/bin/bash", "-c", "ls /usr/bin/ls > /etc/from-child-install"], chroot = True), ], ) @@ -160,8 +160,8 @@ image.layer( package_manager = ":image.package-manager", packages = ["bash", "coreutils"], ops = [ - image.mkdir("/etc/tine"), - image.run( + image.ops.mkdir("/etc/tine"), + image.ops.run( [ "/usr/bin/bash", "-c", @@ -175,7 +175,7 @@ image.layer( image.layer( name = "escaped-child.layer", parent = ":escaped-base", - ops = [image.remove("/etc/tine/esc\\x2dgone")], + ops = [image.ops.remove("/etc/tine/esc\\x2dgone")], ) image.archive( @@ -277,7 +277,7 @@ image.sysext_image( packages = ["zsh"], install_docs = False, # install a file from source tree; covers reproducibility check under varying umasks - ops = [image.copy("from-source.txt", "/usr/lib/tine/from-source")], + ops = [image.ops.copy("from-source.txt", "/usr/lib/tine/from-source")], ) # A file tree that differs per distribution is one copy whose source is a select, rather than a @@ -287,8 +287,8 @@ image.layer( name = "tree.layer", parent = ":base", ops = [ - image.copy("tree/os", "/"), - image.copy(distribution.select({"arch": "tree/os.arch"}, default = None), "/"), + image.ops.copy("tree/os", "/"), + image.ops.copy(distribution.select({"arch": "tree/os.arch"}, default = None), "/"), ], ) @@ -315,7 +315,7 @@ image.layer( package_manager = ":image.package-manager", packages = ["bash", "coreutils"], ops = [ - image.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo old > /etc/tine/sub/old"], chroot = True), + image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo old > /etc/tine/sub/old"], chroot = True), ], ) @@ -323,8 +323,8 @@ image.layer( name = "opaque-child.layer", parent = ":opaque-base", ops = [ - image.remove("/etc/tine/sub"), - image.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo new > /etc/tine/sub/fresh"], chroot = True), + image.ops.remove("/etc/tine/sub"), + image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo new > /etc/tine/sub/fresh"], chroot = True), ], ) diff --git a/image/BUCK b/image/BUCK index 44d25078..364d30c4 100644 --- a/image/BUCK +++ b/image/BUCK @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Logical image mutation and boot assembly drivers.""" +"""Logical images, their compositions and output formats, and the drivers that build them.""" load("@prelude//:rules.bzl", "constraint_setting", "constraint_value") load("//box:test.bzl", "box_python_test") @@ -11,17 +11,17 @@ load(":image.bzl", "image_tools") # The one bundle every image rule resolves its drivers through. image_tools( name = "tools", - archive = "//image_format:archive", + archive = ":archive", artifacts = ":artifacts", boot = ":boot", - convert = "//image_format:convert", - disk = "//image_format:disk", + convert = ":convert", + ddi = ":ddi", layer = ":layer", manifest = ":manifest", python = "//tools:python3", sbom = ":sbom", syft = "//tools:syft", - sysext = "//image_format:sysext", + sysext = ":sysext", uki = ":uki", visibility = ["PUBLIC"], ) @@ -40,24 +40,71 @@ tine_python_binary( ) tine_python_binary( - name = "uki", - main = "uki.py", + name = "archive", + main = "archive.py", deps = [ ":finalize", - ":kmod", "//:specs", "//:util", "//archive:cpio", + "//archive:tar", + ], + visibility = ["PUBLIC"], +) + +tine_python_binary( + name = "ddi", + main = "ddi.py", + deps = [ + ":finalize", + ":manifest-lib", + ":repart", + "//:specs", + "//:util", ], visibility = ["PUBLIC"], ) tine_python_library( - name = "kmod", - srcs = ["kmod.py"], + name = "repart", + srcs = ["repart.py"], deps = ["//:util"], ) +tine_python_binary( + name = "convert", + main = "convert.py", + deps = ["//:specs", "//:util"], + visibility = ["PUBLIC"], +) + +tine_python_binary( + name = "sysext", + main = "sysext.py", + deps = [ + ":finalize", + ":manifest-lib", + ":repart", + "//:specs", + "//:util", + "//rootfs:rootfs", + ], + visibility = ["PUBLIC"], +) + +tine_python_binary( + name = "uki", + main = "uki.py", + deps = [ + ":finalize", + "//:specs", + "//:util", + "//archive:cpio", + "//kmod:kmod", + ], + visibility = ["PUBLIC"], +) + tine_python_library( name = "version", srcs = ["version.py"], @@ -75,13 +122,14 @@ box_python_test( name = "test", box = "//tools:dev.box", srcs = [ + "ddi.py", + "ddi_test.py", "finalize_test.py", - "initrd_modules.toml", - "kmod_test.py", "layer.py", "layer_test.py", "manifest.py", "manifest_test.py", + "repart_test.py", "substitute.py", "substitute_test.py", "uki.py", @@ -90,11 +138,12 @@ box_python_test( ], deps = [ ":finalize", - ":kmod", + ":repart", ":version", "//:specs", "//:util", "//archive:cpio", + "//kmod:kmod", "//package:installer", "//rootfs:rootfs", ], @@ -130,7 +179,6 @@ tine_python_library( "//:specs", "//:util", ], - visibility = ["//image_format:"], ) tine_python_library( @@ -169,10 +217,10 @@ tine_python_binary( deps = [ ":artifacts-lib", ":finalize", - ":kmod", ":version", "//:specs", "//:util", + "//kmod:kmod", ], visibility = ["PUBLIC"], ) diff --git a/image_format/archive.bzl b/image/archive.bzl similarity index 99% rename from image_format/archive.bzl rename to image/archive.bzl index 271b4ed4..4846fbc3 100644 --- a/image_format/archive.bzl +++ b/image/archive.bzl @@ -5,7 +5,7 @@ load("//distribution:defs.bzl", "distribution") load( - "//image:image.bzl", + ":image.bzl", "IMAGE_TOOLS_ATTR", "ImageInfo", "ImageToolsInfo", diff --git a/image_format/archive.py b/image/archive.py similarity index 100% rename from image_format/archive.py rename to image/archive.py diff --git a/image_format/uki.bzl b/image/boot.bzl similarity index 71% rename from image_format/uki.bzl rename to image/boot.bzl index 2647da76..2dced3b9 100644 --- a/image_format/uki.bzl +++ b/image/boot.bzl @@ -1,10 +1,18 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Unified kernel images built from a logical image's kernel, modules, and stub.""" +"""Unified kernel images, and the boot artifacts extracted from a completed logical image.""" +load("//kmod:initrd_modules.toml", _INITRD_MODULES = "value") +load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") +load(":archive.bzl", "ImageArchiveInfo") +load( + ":ddi.bzl", + "DdiInfo", + "RootHashInfo", # @unused Used as a function argument type. +) load( - "//image:image.bzl", + ":image.bzl", "FILENAME_PATTERN", "IMAGE_TOOLS_ATTR", "ImageInfo", @@ -14,22 +22,7 @@ load( "declare_out", "terminal_image_command", ) -load("//image:initrd_modules.toml", _INITRD_MODULES = "value") -load( - "//image:sign.bzl", - "SigningKeyInfo", - "external_signing_execution", - "merge_signing_access", - "resolve_signing_key", - "signing_key_spec", -) -load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") -load(":archive.bzl", "ImageArchiveInfo") -load( - ":disk.bzl", - "RepartInfo", - "RootHashInfo", # @unused Used as a function argument type. -) +load(":sign.bzl", "SigningKeyInfo", "external_signing_execution", "merge_signing_access", "resolve_signing_key", "signing_key_spec") UkiInfo = provider( doc = "Unified kernel images built for one logical image.", @@ -163,9 +156,9 @@ def uki_subtargets(info: UkiInfo) -> dict[str, list[Provider]]: def _uki_impl(ctx: AnalysisContext) -> list[Provider]: root_hash = None if ctx.attrs.root_hash != None: - root_hash = ctx.attrs.root_hash[RepartInfo].root_hash + root_hash = ctx.attrs.root_hash[DdiInfo].root_hash if root_hash == None: - fail("uki: root_hash RepartInfo does not contain a verity root hash") + fail("uki: root_hash DdiInfo does not contain a verity root hash") info = declare_uki( ctx, tools = ctx.attrs._tools[ImageToolsInfo], @@ -239,9 +232,9 @@ _uki = rule( doc = "cpio archives to prepend to the per-kernel modules archive", ), "root_hash": attrs.option( - attrs.dep(providers = [RepartInfo]), + attrs.dep(providers = [DdiInfo]), default = None, - doc = "repart result whose verity hash is added to the embedded kernel command line", + doc = "DDI whose verity hash is added to the embedded kernel command line", ), "version": attrs.string(default = "0", doc = "image version in the UKI name"), } @@ -257,3 +250,95 @@ def uki(name: str, profiles: list[UkiProfile] = [], **kwargs) -> None: that does not opt out. """ _uki(name = name, profiles = encode_profiles(profiles), **kwargs) + +_ARTIFACTS = { + "initrd": "initrd", + "kernel": "vmlinuz", + "uki": "image.efi", +} + +BootArtifactsInfo = record( + initrd = Artifact, + kernel = Artifact, + uki = Artifact, +) + +def declare_boot_artifacts( + ctx: AnalysisContext, + *, + tools: ImageToolsInfo, + image: ImageInfo, + identifier: str | None = None, +) -> BootArtifactsInfo: + """Declare extraction of what a completed image boots: its kernel, initrd and UKI. + + The driver picks the newest kernel the image carries and reads the initrd and kernel out of + the UKI's PE sections when it boots one, so the results are what a direct kernel boot needs. + """ + selection = declare_out(ctx, identifier, "boot-artifacts.json") + select = terminal_image_command( + ctx, + driver = "boot", + exe = tools.boot, + identifier = identifier, + image = image, + spec = {"out": selection.as_output()}, + ) + ctx.actions.run(select, category = "boot_artifact_select", identifier = identifier) + + artifacts = {} + for kind in sorted(_ARTIFACTS): + out = declare_out(ctx, identifier, _ARTIFACTS[kind]) + extract = terminal_image_command( + ctx, + # The extracted artifacts are named after their kind, so the specs of one composition + # step are told apart by driver name; scoping them by kind would name a directory + # after a file already declared beside it. + driver = "artifacts-" + kind, + exe = tools.artifacts, + identifier = identifier, + image = image, + spec = { + "artifact": kind, + "manifest": selection, + "out": out.as_output(), + }, + ) + ctx.actions.run(extract, category = "boot_artifact_" + kind, identifier = identifier) + artifacts[kind] = out + return BootArtifactsInfo( + initrd = artifacts["initrd"], + kernel = artifacts["kernel"], + uki = artifacts["uki"], + ) + +def boot_subtargets(artifacts: BootArtifactsInfo) -> dict[str, list[Provider]]: + """The extracted artifacts, exposed wherever the image they come from is.""" + return { + "initrd": [DefaultInfo(default_output = artifacts.initrd)], + "kernel": [DefaultInfo(default_output = artifacts.kernel)], + "uki": [DefaultInfo(default_output = artifacts.uki)], + } + +def _bootable_impl(ctx: AnalysisContext) -> list[Provider]: + artifacts = declare_boot_artifacts(ctx, tools = ctx.attrs._tools[ImageToolsInfo], image = ctx.attrs.image[ImageInfo]) + + # A UKI is not guaranteed to exist (kernels can ship as plain files), so the default + # outputs stay limited to the artifacts selection always yields; [uki] extracts on demand. + return [ + DefaultInfo( + default_outputs = [artifacts.kernel, artifacts.initrd], + sub_targets = boot_subtargets(artifacts), + ), + ] + +bootable = rule( + impl = _bootable_impl, + attrs = { + "image": attrs.dep( + providers = [ImageInfo], + doc = "the completed logical image from which to select boot artifacts", + ), + } + | IMAGE_TOOLS_ATTR, +) diff --git a/image/compose.bzl b/image/compose.bzl index 600f5198..6893ae65 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -4,40 +4,36 @@ """Convenience compositions for image products.""" load("//distribution:defs.bzl", "distribution") +load("//package:manager.bzl", "PackageManagerInfo") +load("//platforms:architecture.bzl", "ARCHITECTURES") load( - "//image_format:archive.bzl", + ":archive.bzl", "ARCHIVE_ATTRS", + "COMPRESSIONS", + "ImageArchiveInfo", "declare_image_archive", "declare_image_directory", ) -load("//image_format:boot.bzl", "boot_subtargets", "declare_boot_artifacts") load( - "//image_format:disk.bzl", + ":boot.bzl", + "UKI_ATTRS", + "UkiProfile", # @unused Used as a type. + "boot_subtargets", + "declare_boot_artifacts", + "declare_uki", + "encode_profiles", + "uki_subtargets", +) +load( + ":ddi.bzl", "DISK_FORMATS", "Partition", # @unused Used as a type. + "declare_ddi", "declare_disk_conversion", - "declare_repart", "encode_definitions", "format_partition_labels", "manifest_sub_targets", ) -load( - "//image_format:sysext.bzl", - "SYSEXT_ATTRS", - "declare_image_sysext", - "sysext_published", - "sysext_subtargets", -) -load( - "//image_format:uki.bzl", - "UKI_ATTRS", - "UkiProfile", # @unused Used as a type. - "declare_uki", - "encode_profiles", - "uki_subtargets", -) -load("//package:manager.bzl", "PackageManagerInfo") -load("//platforms:architecture.bzl", "ARCHITECTURES") load( ":image.bzl", "IMAGE_ATTRS", @@ -54,9 +50,10 @@ load( "image_providers", "install_systemd_boot", "merge_os_release", + "remove", "sign_systemd_boot", + "symlink", ) -load(":initrd.bzl", "InitrdInfo", "initrd_image") load(":publish.bzl", "PublishedInfo", "published_metadata_subtargets") load( ":sign.bzl", @@ -64,8 +61,109 @@ load( "VERITY_KEY_ATTR", "resolve_signing_key", ) +load( + ":sysext.bzl", + "SYSEXT_ATTRS", + "declare_image_sysext", + "sysext_published", + "sysext_subtargets", +) load(":version.bzl", "resolve_version") +InitrdInfo = provider( + doc = "A logical initrd image and its derived cpio archive.", + fields = { + "cpio": provider_field(ImageArchiveInfo), + "image": provider_field(ImageInfo), + }, +) + +def _initrd_image_impl(ctx: AnalysisContext) -> list[Provider]: + tools = ctx.attrs._tools[ImageToolsInfo] + image = declare_image( + ctx, + tools = tools, + install_docs = ctx.attrs.install_docs, + install_langs = ctx.attrs.install_langs, + ops = generated( + [ + symlink("/usr/lib/systemd/systemd", "/init"), + symlink("/etc/os-release", "/etc/initrd-release"), + ] + + ctx.attrs.ops, + ["initrd"], + ) + # The removals run after the generators, because the hwdb generator reads hwdb.d. At run time + # udev reads only the compiled hardware database. Nothing in the initrd shows catalog + # messages or resolves service names, so the initrd does not need the catalogs or + # /etc/services. + + [ + remove("/usr/lib/udev/hwdb.d"), + remove("/usr/lib/systemd/catalog"), + remove("/var/lib/systemd/catalog"), + remove("/etc/services"), + ], + package_manager = ctx.attrs.package_manager, + # Each release lists the packages of its "initrd" package set, so the initrd names the set + # instead of packages. + package_sets = ["initrd"] + ctx.attrs.package_sets, + packages = ctx.attrs.packages, + tmpfiles = ctx.attrs.tmpfiles, + version = ctx.attrs.version, + ) + cpio = declare_image_archive( + ctx, + tools = tools, + compression = ctx.attrs.compression, + format = "cpio", + image = image, + strip_pkgdb = ctx.attrs.strip_pkgdb, + ) + return image_providers( + default_outputs = [cpio.archive], + extra = [InitrdInfo(cpio = cpio, image = image), cpio], + image = image, + ) + +_initrd_image = rule( + impl = _initrd_image_impl, + supports_incoming_transition = True, + attrs = IMAGE_ATTRS + | { + "compression": attrs.enum(COMPRESSIONS, default = "zstd"), + # An initrd is small, short-lived, and read once, so the defaults an OS image wants are the + # wrong way round here: documentation and translations only cost boot memory. + "install_docs": attrs.bool(default = False), + "install_langs": attrs.list(attrs.string(), default = ["C.UTF-8"]), + "package_manager": attrs.dep(providers = [PackageManagerInfo]), + "strip_pkgdb": attrs.bool( + default = True, + doc = "leave the package database out of the cpio; the image still carries it", + ), + }, +) + +def initrd_image( + name: str, + ops: list[LayerOperationTree] = [], + version: str | Select | None = None, + distro: str | None = None, + visibility: list[str] | None = None, + **kwargs, +) -> None: + """Declare the conventional initrd, extended by `packages`, `package_sets`, and `ops`. + + The release's `initrd` package set and the operations every initrd needs come first; what this + target declares is added to them rather than replacing them. `bootable_disk_image` declares one + of these for itself when it is given no `initrd`. + """ + _initrd_image( + name = name, + ops = flatten_operations(ops), + version = resolve_version("initrd_image {}".format(name), version), + **(distribution.distributed(name, distro, visibility) | kwargs), + ) + def _composed_image( ctx: AnalysisContext, *, @@ -289,7 +387,7 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: # and every split partition, which repart names after the image it writes them out of. basename = "{}_{}_{}".format(image_id, version, ARCHITECTURES[ctx.attrs.arch].systemd) - system = declare_repart( + system = declare_ddi( ctx, tools = tools, systemd_arch = ARCHITECTURES[ctx.attrs.arch].systemd, @@ -332,7 +430,7 @@ def _bootable_disk_image_impl(ctx: AnalysisContext) -> list[Provider]: version = version, ) - disk = declare_repart( + disk = declare_ddi( ctx, tools = tools, systemd_arch = ARCHITECTURES[ctx.attrs.arch].systemd, diff --git a/image_format/convert.py b/image/convert.py similarity index 100% rename from image_format/convert.py rename to image/convert.py diff --git a/image_format/disk.bzl b/image/ddi.bzl similarity index 89% rename from image_format/disk.bzl rename to image/ddi.bzl index f40f789c..6717a2d2 100644 --- a/image_format/disk.bzl +++ b/image/ddi.bzl @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Partition and raw-disk assembly with systemd-repart.""" +"""Build a DDI, a GPT disk image, from partition definitions with systemd-repart.""" load("//:specs.bzl", "spec_args") load("//box:runtime.bzl", "BoxInfo", "box_run") +load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") load( - "//image:image.bzl", + ":image.bzl", "GPT_LABEL_LIMIT", "IMAGE_TOOLS_ATTR", "ImageInfo", @@ -18,7 +19,7 @@ load( "terminal_image_command", ) load( - "//image:sign.bzl", + ":sign.bzl", "SigningKeyInfo", "VERITY_KEY_ATTR", "external_signing_execution", @@ -26,7 +27,6 @@ load( "resolve_signing_key", "signing_key_spec", ) -load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") Partition = dict[str, typing.Any] @@ -101,7 +101,7 @@ DEFAULT_ROOT_PARTITIONS = [ # systemd-sysupdate A/B slots match partitions by exact label, so the default layouts carry the # image identity: `bootable_disk_image()` renders the placeholders from its image_id and version -# attributes (raw `repart()` users render them with `format_partition_labels()`). +# attributes (raw `ddi()` users render them with `format_partition_labels()`). def _usr_verity_partitions(signed: bool) -> list[Partition]: label = "{image_id}_{version}" definitions = [ @@ -180,20 +180,20 @@ def encode_definitions( `rendered = False` accepts label placeholders for a caller that renders them itself. """ if not definitions and (split or not imports): - fail("repart: definitions may be empty only when assembling partition inputs") + fail("ddi: definitions may be empty only when assembling partition inputs") if not disk and not split: - fail("repart: at least one of disk or split must be enabled") + fail("ddi: at least one of disk or split must be enabled") names = {} for definition in definitions: if definition["name"] in names: - fail("repart: definition names must be unique") + fail("ddi: definition names must be unique") names[definition["name"]] = True label = definition["label"] if rendered and label != None and "{" in label: - fail("repart: unrendered placeholder in label {!r}; render with format_partition_labels()".format(label)) + fail("ddi: unrendered placeholder in label {!r}; render with format_partition_labels()".format(label)) signatures = [it for it in definitions if it["verity"] == "signature"] if bool(signatures) != signed: - fail("repart: Verity=signature and signing credentials must be specified together") + fail("ddi: Verity=signature and signing credentials must be specified together") return [json.encode(definition) for definition in definitions] WrittenPartitionInfo = record( @@ -219,8 +219,8 @@ RootHashInfo = provider( }, ) -RepartInfo = provider( - doc = "A repart result containing an optional disk and any independent partition artifacts.", +DdiInfo = provider( + doc = "A DDI: an optional composed disk and any independent partition artifacts.", fields = { "disk": provider_field(Artifact | None, default = None), # What the composed disk carries, its own partitions and its imported ones taken together. @@ -233,8 +233,8 @@ RepartInfo = provider( }, ) -RepartOutput = record( - info = RepartInfo, +DdiOutput = record( + info = DdiInfo, outputs = list[Artifact], sub_targets = dict[str, list[Provider]], ) @@ -245,13 +245,13 @@ def _verity_kind(definitions: list[Partition]) -> str | None: if definition["verity"] != "data": continue if kind != None: - fail("repart: one invocation cannot produce more than one verity root hash") + fail("ddi: one invocation cannot produce more than one verity root hash") if definition["type"].startswith("usr"): kind = "usr" elif definition["type"].startswith("root"): kind = "root" else: - fail("repart: Verity=data requires a usr or root partition type") + fail("ddi: Verity=data requires a usr or root partition type") return kind def _partition_sub_targets(partitions: list[PartitionInfo]) -> dict[str, list[Provider]]: @@ -279,7 +279,7 @@ def manifest_sub_targets(written: list[WrittenPartitionInfo]) -> dict[str, list[ ], } -def declare_repart( +def declare_ddi( ctx: AnalysisContext, *, tools: ImageToolsInfo, @@ -288,7 +288,7 @@ def declare_repart( definitions: list[str], disk: bool = True, split: bool = False, - imported: list[RepartInfo] = [], + imported: list[DdiInfo] = [], imported_root_hash: RootHashInfo | None = None, seed: str | None = None, verity_key: SigningKeyInfo | None = None, @@ -297,8 +297,8 @@ def declare_repart( mkfs_options: dict[str, list[str]] = {}, basename: str = "image", identifier: str | None = None, -) -> RepartOutput: - """Declare repart actions from resolved image and partition providers. +) -> DdiOutput: + """Declare the systemd-repart action that builds a DDI from an image and partition definitions. With strip_pkgdb, the partitions omit the package database wherever the image's package system keeps it, for a system nothing ever resolves packages in. The image's `[pkgdb]` and `[sbom]` @@ -315,9 +315,9 @@ def declare_repart( if output_size != None: if not disk: - fail("repart: output_size applies to a composed disk, which this invocation does not emit") + fail("ddi: output_size applies to a composed disk, which this invocation does not emit") if not regex_match(SIZE_PATTERN, output_size): - fail("repart: invalid output_size {!r}".format(output_size)) + fail("ddi: invalid output_size {!r}".format(output_size)) # repart hands mkfs one variable per filesystem and splits it on whitespace, so an option # carrying any would silently arrive as two. @@ -325,7 +325,7 @@ def declare_repart( for option in options: if " " in option or "\t" in option: fail( - "repart: {} mkfs option {!r} holds whitespace, which repart splits on".format( + "ddi: {} mkfs option {!r} holds whitespace, which repart splits on".format( filesystem, option, ) @@ -337,7 +337,7 @@ def declare_repart( names = [definition["name"] for definition in decoded] + [partition.definition["name"] for partition in imported_partitions] if len(names) != len({name: True for name in names}): - fail("repart: new and imported partition names must be unique") + fail("ddi: new and imported partition names must be unique") definition_specs = [] outputs = [] @@ -382,6 +382,8 @@ def declare_repart( "architecture": systemd_arch, "basename": basename, "definitions": definition_specs, + # The default seed derives from the identity, so changing "repart" here changes the partition + # UUIDs of every target that declares no identifier. "identity": "{}[{}]".format(ctx.label, identifier or "repart"), "manifest": None, "mkfs_options": mkfs_options, @@ -417,7 +419,7 @@ def declare_repart( kind = _verity_kind(decoded) if kind != None and imported_root_hash != None: - fail("repart: cannot combine multiple verity root hashes") + fail("ddi: cannot combine multiple verity root hashes") root_hash = imported_root_hash if kind != None: hash_output = declare_out(ctx, identifier, "{}hash".format(kind)) @@ -425,7 +427,7 @@ def declare_repart( root_hash = RootHashInfo(hash = hash_output, kind = kind) available_partitions = imported_partitions + new_partitions - info = RepartInfo( + info = DdiInfo( disk = out, manifest = manifest, partitions = available_partitions, @@ -458,30 +460,30 @@ def declare_repart( terminal_image_command( ctx, signing_access = signing_access, - driver = "repart", - exe = tools.disk, + driver = "ddi", + exe = tools.ddi, identifier = identifier, image = image, spec = spec, ), - category = "repart_split" if split else "repart", - identifier = identifier or "repart", + category = "ddi_split" if split else "ddi", + identifier = identifier or "ddi", **external_signing_execution(signing_access), ) - return RepartOutput(info = info, outputs = outputs, sub_targets = sub_targets) + return DdiOutput(info = info, outputs = outputs, sub_targets = sub_targets) -def _repart_impl(ctx: AnalysisContext) -> list[Provider]: +def _ddi_impl(ctx: AnalysisContext) -> list[Provider]: imported = [] imported_root_hash = None for dep in ctx.attrs.partitions: - info = dep[RepartInfo] + info = dep[DdiInfo] imported.append(info) root_hash = info.root_hash if root_hash != None: if imported_root_hash != None: - fail("repart: cannot combine multiple verity root hashes") + fail("ddi: cannot combine multiple verity root hashes") imported_root_hash = root_hash - result = declare_repart( + result = declare_ddi( ctx, tools = ctx.attrs._tools[ImageToolsInfo], systemd_arch = ARCHITECTURES[ctx.attrs._arch].systemd, @@ -503,7 +505,7 @@ def _repart_impl(ctx: AnalysisContext) -> list[Provider]: result.info, ] -REPART_ATTRS = { +DDI_ATTRS = { "basename": attrs.string( default = "image", doc = "what the composed disk and its split partitions are named after, without extension", @@ -532,16 +534,16 @@ REPART_ATTRS = { "verity_key": VERITY_KEY_ATTR, } -_repart = rule( - impl = _repart_impl, - attrs = REPART_ATTRS +_ddi = rule( + impl = _ddi_impl, + attrs = DDI_ATTRS | { "disk": attrs.bool(default = True, doc = "emit a composed raw disk image"), "image": attrs.dep(providers = [ImageInfo], doc = "the logical image used to populate new partitions"), "partitions": attrs.list( - attrs.dep(providers = [RepartInfo]), + attrs.dep(providers = [DdiInfo]), default = [], - doc = "split repart targets whose partitions should be copied into the result", + doc = "split ddi targets whose partitions should be copied into the result", ), "split": attrs.bool(default = False, doc = "also emit independent artifacts for new partitions"), # `_arch` is the architecture the image runs on. repart picks the partition types for it. @@ -550,7 +552,7 @@ _repart = rule( | IMAGE_TOOLS_ATTR, ) -def repart( +def ddi( name: str, definitions: list[Partition], partitions: list[str] = [], @@ -560,7 +562,7 @@ def repart( **kwargs, ) -> None: """Create a disk, independent partitions, or both.""" - _repart( + _ddi( name = name, definitions = encode_definitions( definitions, @@ -591,7 +593,7 @@ def declare_disk_conversion( ctx: AnalysisContext, *, tools: ImageToolsInfo, - disk: RepartInfo, + disk: DdiInfo, box: Dependency, format: str, basename: str = "image", @@ -599,7 +601,7 @@ def declare_disk_conversion( ) -> DiskConversionInfo: """Declare a disk format conversion from a resolved raw disk provider.""" if disk.disk == None: - fail("disk_convert: RepartInfo does not contain a composed disk") + fail("disk_convert: DdiInfo does not contain a composed disk") out = declare_out(ctx, identifier, basename + "." + format) cmd = cmd_args( box_run(box = box[BoxInfo], exe = tools.convert), @@ -621,7 +623,7 @@ def _disk_convert_impl(ctx: AnalysisContext) -> list[Provider]: ctx, tools = ctx.attrs._tools[ImageToolsInfo], basename = ctx.attrs.basename, - disk = ctx.attrs.disk[RepartInfo], + disk = ctx.attrs.disk[DdiInfo], box = ctx.attrs.box, format = ctx.attrs.format, ) @@ -632,7 +634,7 @@ disk_convert = rule( attrs = { "basename": attrs.string(default = "image", doc = "file name of the re-encoded disk, without extension"), "box": attrs.exec_dep(providers = [BoxInfo], doc = "execution environment supplying conversion tools"), - "disk": attrs.dep(providers = [RepartInfo], doc = "the composed raw disk to re-encode"), + "disk": attrs.dep(providers = [DdiInfo], doc = "the composed raw disk to re-encode"), "format": attrs.enum(DISK_FORMATS), } | IMAGE_TOOLS_ATTR, diff --git a/image_format/disk.py b/image/ddi.py similarity index 95% rename from image_format/disk.py rename to image/ddi.py index 1ea512bb..8d13a364 100755 --- a/image_format/disk.py +++ b/image/ddi.py @@ -177,7 +177,7 @@ def _derived_seed( """Derive UUIDs from logical configuration, not action-local CopyBlocks paths. A definition is re-encoded exactly as the rule spelled it, key order included (see - disk.bzl:partition): any other encoding re-identifies every partition of every + ddi.bzl:partition): any other encoding re-identifies every partition of every unchanged image. """ digest = hashlib.sha256(identity.encode()) @@ -215,14 +215,14 @@ def _write_definitions( def _partition_row(rows: list[dict[str, Any]], filename: str) -> dict[str, Any]: matches = [row for row in rows if Path(row.get("file", "")).name == filename] if len(matches) != 1: - util.fail(f"repart: expected one result for {filename}, found {len(matches)}") + util.fail(f"ddi: expected one result for {filename}, found {len(matches)}") return matches[0] def _copy_partition(row: dict[str, Any], name: str, blocks: Path, metadata: Path) -> None: source_value = row.get("split_path") if not source_value or source_value == "-": - util.fail(f"repart: no split artifact was produced for {name}") + util.fail(f"ddi: no split artifact was produced for {name}") source = Path(source_value) blocks.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, blocks) @@ -231,7 +231,7 @@ def _copy_partition(row: dict[str, Any], name: str, blocks: Path, metadata: Path # sector-aligned artifact, so restore the partition's declared raw size. raw_size = int(row["raw_size"]) if blocks.stat().st_size > raw_size: - util.fail(f"repart: split artifact for {name} exceeds its partition") + util.fail(f"ddi: split artifact for {name} exceeds its partition") with blocks.open("r+b") as f: f.truncate(raw_size) @@ -261,16 +261,16 @@ def _grow(disk: Path, size: str) -> None: ) composed = disk.stat().st_size if composed > target: - util.fail(f"repart: the partitions need {composed} bytes, more than the requested {size}") + util.fail(f"ddi: the partitions need {composed} bytes, more than the requested {size}") with disk.open("r+b") as f: f.truncate(target) def main(argv: list[str] | None = None) -> None: - spec = specs.parse(Spec, "repart", argv) + spec = specs.parse(Spec, "ddi", argv) if not spec["definitions"] and not spec["partitions"]: - util.fail("repart: specify at least one definition or imported partition") + util.fail("ddi: specify at least one definition or imported partition") raw_definitions = [entry["definition"] for entry in spec["definitions"]] written = [ WrittenPartition( @@ -298,14 +298,14 @@ def main(argv: list[str] | None = None) -> None: if partition.blocks is not None and partition.metadata is not None ] if not spec["out"] and not splits: - util.fail("repart: specify a disk output, split outputs, or both") + util.fail("ddi: specify a disk output, split outputs, or both") binds: list[tuple[str | Path, str | Path]] = [ (partition.blocks, f"/run/tine/repart/{index}.raw") for index, partition in enumerate(partitions, start=len(written)) ] with ( - finalize.image(spec, program="repart", binds=binds) as tree, + finalize.image(spec, program="ddi", binds=binds) as tree, tempfile.TemporaryDirectory(prefix="repart.") as scratch_dir, ): # The package database is a supply-chain artifact that the image's `[pkgdb]` and `[sbom]` @@ -387,7 +387,7 @@ def main(argv: list[str] | None = None) -> None: if splits: artifacts.append(f"{len(splits)} partitions") mode = " and ".join(artifacts) - print(f"repart: wrote {mode} (seed={seed})", file=sys.stderr) + print(f"ddi: wrote {mode} (seed={seed})", file=sys.stderr) if __name__ == "__main__": diff --git a/image_format/disk_test.py b/image/ddi_test.py similarity index 83% rename from image_format/disk_test.py rename to image/ddi_test.py index 936fc7a1..6acb9a6e 100644 --- a/image_format/disk_test.py +++ b/image/ddi_test.py @@ -3,7 +3,7 @@ """Tests for the repart driver's split-artifact handling. - buck test tine//image_format:test + buck test tine//image:test repart itself is not run here: what matters is the definition the driver hands it and what the driver records about the artifact that comes back, both of which decide how a partition is @@ -16,7 +16,7 @@ from pathlib import Path from typing import cast -import disk +import ddi UUID = "fb1bb90c-9de1-b90c-8b89-e04b87c721bd" PUBLISHED = f"AmutableOS_9_x86-64.usr-x86-64.{UUID}.raw" @@ -41,16 +41,16 @@ class TestSplitName(unittest.TestCase): def test_split_names_by_type_and_uuid(self) -> None: # systemd-sysupdate reads the UUID back out of the name as @u, so the artifact repart # writes is already named the way it is published. - self.assertIn("SplitName=%t.%U", disk.Definition.parse(DEFINITION).render(split=True).splitlines()) + self.assertIn("SplitName=%t.%U", ddi.Definition.parse(DEFINITION).render(split=True).splitlines()) def test_no_split_name_without_split(self) -> None: - rendered = disk.Definition.parse(DEFINITION).render(split=False) + rendered = ddi.Definition.parse(DEFINITION).render(split=False) self.assertNotIn("SplitName", rendered) class TestCopyPartition(unittest.TestCase): def _copy(self, *, contents: bytes, raw_size: int) -> dict[str, object]: - with tempfile.TemporaryDirectory(prefix="disk_test.") as scratch_dir: + with tempfile.TemporaryDirectory(prefix="ddi_test.") as scratch_dir: scratch = Path(scratch_dir) source = scratch / PUBLISHED source.write_bytes(contents) @@ -63,7 +63,7 @@ def _copy(self, *, contents: bytes, raw_size: int) -> dict[str, object]: "type": "usr-x86-64", "uuid": UUID, } - disk._copy_partition(row, "usr", blocks, metadata) + ddi._copy_partition(row, "usr", blocks, metadata) self.assertEqual(blocks.stat().st_size, raw_size) return cast(dict[str, object], json.loads(metadata.read_text())) @@ -82,10 +82,10 @@ def test_refuses_an_artifact_larger_than_its_partition(self) -> None: self._copy(contents=b"x" * 1024, raw_size=512) def test_refuses_a_partition_repart_did_not_split(self) -> None: - with tempfile.TemporaryDirectory(prefix="disk_test.") as scratch_dir: + with tempfile.TemporaryDirectory(prefix="ddi_test.") as scratch_dir: scratch = Path(scratch_dir) with self.assertRaises(SystemExit): - disk._copy_partition({"split_path": "-"}, "usr", scratch / "usr.raw", scratch / "usr.json") + ddi._copy_partition({"split_path": "-"}, "usr", scratch / "usr.raw", scratch / "usr.json") if __name__ == "__main__": diff --git a/image/defs.bzl b/image/defs.bzl index 8c92fdc2..b8248e51 100644 --- a/image/defs.bzl +++ b/image/defs.bzl @@ -7,45 +7,37 @@ Load the `image` namespace from here; the modules behind it are implementation s rearranged. """ +load("//kmod:initrd_modules.toml", _INITRD_MODULES = "value") load( - "//image_format:archive.bzl", + ":archive.bzl", "COMPRESSIONS", "ImageArchiveInfo", "ImageDirectoryInfo", "image_archive", "image_directory", ) -load("//image_format:boot.bzl", "bootable") +load(":boot.bzl", "UkiInfo", "bootable", "uki", "uki_profile") load( - "//image_format:disk.bzl", + ":compose.bzl", + "InitrdInfo", + "bootable_disk_image", + "initrd_image", + "rootfs_archive", + "sysext_image", +) +load( + ":ddi.bzl", "DEFAULT_ROOT_PARTITIONS", "DEFAULT_SIGNED_USR_VERITY_PARTITIONS", "DEFAULT_USR_VERITY_PARTITIONS", "DISK_FORMATS", + "DdiInfo", "DiskConversionInfo", - "RepartInfo", "RootHashInfo", + "ddi", "disk_convert", "format_partition_labels", "partition", - "repart", -) -load( - "//image_format:sysext.bzl", - "SysextImageInfo", - "image_sysext", -) -load( - "//image_format:uki.bzl", - "UkiInfo", - "uki", - "uki_profile", -) -load( - ":compose.bzl", - "bootable_disk_image", - "rootfs_archive", - "sysext_image", ) load( ":image.bzl", @@ -55,6 +47,7 @@ load( "copy", "depmod", "hwdb", + "image_install", "install_from", "install_systemd_boot", "layer", @@ -67,9 +60,6 @@ load( "symlink", "write_file", ) -load(":initrd.bzl", "InitrdInfo", "initrd_image") -load(":initrd_modules.toml", _INITRD_MODULES = "value") -load(":install.bzl", "image_install") load(":publish.bzl", "PublishedInfo", "image_artifacts") load( ":sign.bzl", @@ -79,25 +69,32 @@ load( "pkcs11_signing_key", ) load(":substitute.bzl", "substitute") +load( + ":sysext.bzl", + "SysextImageInfo", + "image_sysext", +) load(":version.bzl", "resolve_version") load(":vm.bzl", "image_vm") image = struct( - # Logical images and their operations. + # Logical images and the operations that build them. layer = layer, - run = run, - python = python, - install_from = install_from, - mkdir = mkdir, - symlink = symlink, - write_file = write_file, - remove = remove, - copy = copy, - merge_os_release = merge_os_release, - install_systemd_boot = install_systemd_boot, - depmod = depmod, - hwdb = hwdb, - locale_gen = locale_gen, + ops = struct( + run = run, + python = python, + install_from = install_from, + mkdir = mkdir, + symlink = symlink, + write_file = write_file, + remove = remove, + copy = copy, + merge_os_release = merge_os_release, + install_systemd_boot = install_systemd_boot, + depmod = depmod, + hwdb = hwdb, + locale_gen = locale_gen, + ), # Terminal outputs. archive = image_archive, directory = image_directory, @@ -106,7 +103,7 @@ image = struct( uki_profile = uki_profile, resolve_version = resolve_version, bootable = bootable, - repart = repart, + ddi = ddi, partition = partition, format_partition_labels = format_partition_labels, disk_convert = disk_convert, @@ -139,7 +136,7 @@ image = struct( ImageDirectoryInfo = ImageDirectoryInfo, SysextImageInfo = SysextImageInfo, UkiInfo = UkiInfo, - RepartInfo = RepartInfo, + DdiInfo = DdiInfo, RootHashInfo = RootHashInfo, DiskConversionInfo = DiskConversionInfo, InitrdInfo = InitrdInfo, diff --git a/image/image.bzl b/image/image.bzl index 8ff5ff44..6f36f8be 100644 --- a/image/image.bzl +++ b/image/image.bzl @@ -19,7 +19,7 @@ ImageToolsInfo = provider( "artifacts": provider_field(Dependency), "boot": provider_field(Dependency), "convert": provider_field(Dependency), - "disk": provider_field(Dependency), + "ddi": provider_field(Dependency), "layer": provider_field(Dependency), "manifest": provider_field(Dependency), "python": provider_field(Dependency), @@ -35,7 +35,7 @@ _TOOLS = [ "artifacts", "boot", "convert", - "disk", + "ddi", "layer", "manifest", "python", @@ -854,3 +854,37 @@ def layer( ) -> None: """Create an initial image or apply one delta to a parent image.""" _image(name = name, ops = flatten_operations(ops), **(distribution.distributed(name, distro, visibility) | kwargs)) + +def _image_install_impl(ctx: AnalysisContext) -> list[Provider]: + from_targets, operations = expand_install_from(ctx.attrs.ops) + + # A convenience view: building this target builds what it would copy into an image. + copies = {} + for operation in operations: + if operation[0] == "copy": + copies[operation[1].short_path] = operation[1] + return [ + DefaultInfo(default_outputs = [copies[path] for path in sorted(copies)]), + ImageInstallInfo( + operations = operations, + packages = sorted({package: None for package in ctx.attrs.packages + from_targets}), + ), + ] + +_image_install = rule( + impl = _image_install_impl, + attrs = { + "ops": attrs.list(IMAGE_OPERATION_ATTR, default = [], doc = "operations an image applies to install this target"), + "packages": attrs.list(attrs.string(), default = [], doc = "native packages this target needs installed"), + }, +) + +def image_install(name: str, ops: list[LayerOperationTree] = [], packages: list[str] = [], **kwargs) -> None: + """Attach packages and install operations to a target, applied by `install_from()` in an image. + + An image installing this target adds its packages to its own install request, so a reusable + target declares what it needs rather than leaving it to every caller. + """ + if not ops and not packages: + fail("image_install {}: declare the packages or operations that install it".format(name)) + _image_install(name = name, ops = flatten_operations(ops), packages = packages, **kwargs) diff --git a/image/initrd.bzl b/image/initrd.bzl deleted file mode 100644 index 9a32f35d..00000000 --- a/image/initrd.bzl +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: Amutable GmbH -# SPDX-License-Identifier: MPL-2.0 - -"""The conventional initrd a bootable image boots, as a target of its own.""" - -load("//distribution:defs.bzl", "distribution") -load( - "//image_format:archive.bzl", - "COMPRESSIONS", - "ImageArchiveInfo", - "declare_image_archive", -) -load("//package:manager.bzl", "PackageManagerInfo") -load( - ":image.bzl", - "IMAGE_ATTRS", - "ImageInfo", - "ImageToolsInfo", # @unused Used as a function argument type. - "LayerOperationTree", # @unused Used as a type. - "declare_image", - "flatten_operations", - "generated", - "image_providers", - "remove", - "symlink", -) -load(":version.bzl", "resolve_version") - -InitrdInfo = provider( - doc = "A logical initrd image and its derived cpio archive.", - fields = { - "cpio": provider_field(ImageArchiveInfo), - "image": provider_field(ImageInfo), - }, -) - -# The release names what an initrd of its own family needs, so this stays symbolic. -_DEFAULT_PACKAGE_SETS = ["initrd"] - -_DEFAULT_OPS = [ - symlink("/usr/lib/systemd/systemd", "/init"), - symlink("/etc/os-release", "/etc/initrd-release"), -] - -# Pruned once the generators have read what they need: udev reads its binary hardware database at run -# time, not the sources it was compiled from, while these databases serve humans and service-name -# resolution, neither of which happens in the initrd. -_PRUNED = [ - remove("/usr/lib/udev/hwdb.d"), - remove("/usr/lib/systemd/catalog"), - remove("/var/lib/systemd/catalog"), - remove("/etc/services"), -] - -def _initrd_image_impl(ctx: AnalysisContext) -> list[Provider]: - tools = ctx.attrs._tools[ImageToolsInfo] - image = declare_image( - ctx, - tools = tools, - install_docs = ctx.attrs.install_docs, - install_langs = ctx.attrs.install_langs, - ops = generated(_DEFAULT_OPS + ctx.attrs.ops, _DEFAULT_PACKAGE_SETS) + _PRUNED, - package_manager = ctx.attrs.package_manager, - package_sets = _DEFAULT_PACKAGE_SETS + ctx.attrs.package_sets, - packages = ctx.attrs.packages, - tmpfiles = ctx.attrs.tmpfiles, - version = ctx.attrs.version, - ) - cpio = declare_image_archive( - ctx, - tools = tools, - compression = ctx.attrs.compression, - format = "cpio", - image = image, - strip_pkgdb = ctx.attrs.strip_pkgdb, - ) - return image_providers( - default_outputs = [cpio.archive], - extra = [InitrdInfo(cpio = cpio, image = image), cpio], - image = image, - ) - -_initrd_image = rule( - impl = _initrd_image_impl, - supports_incoming_transition = True, - attrs = IMAGE_ATTRS - | { - "compression": attrs.enum(COMPRESSIONS, default = "zstd"), - # An initrd is small, short-lived, and read once, so the defaults an OS image wants are the - # wrong way round here: documentation and translations only cost boot memory. - "install_docs": attrs.bool(default = False), - "install_langs": attrs.list(attrs.string(), default = ["C.UTF-8"]), - "package_manager": attrs.dep(providers = [PackageManagerInfo]), - "strip_pkgdb": attrs.bool( - default = True, - doc = "leave the package database out of the cpio; the image still carries it", - ), - }, -) - -def initrd_image( - name: str, - ops: list[LayerOperationTree] = [], - version: str | Select | None = None, - distro: str | None = None, - visibility: list[str] | None = None, - **kwargs, -) -> None: - """Declare the conventional initrd, extended by `packages`, `package_sets`, and `ops`. - - The release's `initrd` package set and the operations every initrd needs come first; what this - target declares is added to them rather than replacing them. `bootable_disk_image` declares one - of these for itself when it is given no `initrd`. - """ - _initrd_image( - name = name, - ops = flatten_operations(ops), - version = resolve_version("initrd_image {}".format(name), version), - **(distribution.distributed(name, distro, visibility) | kwargs), - ) diff --git a/image/install.bzl b/image/install.bzl deleted file mode 100644 index 6de04775..00000000 --- a/image/install.bzl +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-FileCopyrightText: Amutable GmbH -# SPDX-License-Identifier: MPL-2.0 - -"""Install operations a target attaches to itself, for any image to apply.""" - -load( - ":image.bzl", - "IMAGE_OPERATION_ATTR", - "ImageInstallInfo", - "LayerOperationTree", # @unused Used as a type. - "expand_install_from", - "flatten_operations", -) - -def _image_install_impl(ctx: AnalysisContext) -> list[Provider]: - from_targets, operations = expand_install_from(ctx.attrs.ops) - - # A convenience view: building this target builds what it would copy into an image. - copies = {} - for operation in operations: - if operation[0] == "copy": - copies[operation[1].short_path] = operation[1] - return [ - DefaultInfo(default_outputs = [copies[path] for path in sorted(copies)]), - ImageInstallInfo( - operations = operations, - packages = sorted({package: None for package in ctx.attrs.packages + from_targets}), - ), - ] - -_image_install = rule( - impl = _image_install_impl, - attrs = { - "ops": attrs.list(IMAGE_OPERATION_ATTR, default = [], doc = "operations an image applies to install this target"), - "packages": attrs.list(attrs.string(), default = [], doc = "native packages this target needs installed"), - }, -) - -def image_install(name: str, ops: list[LayerOperationTree] = [], packages: list[str] = [], **kwargs) -> None: - """Attach packages and install operations to a target, applied by `install_from()` in an image. - - An image installing this target adds its packages to its own install request, so a reusable - target declares what it needs rather than leaving it to every caller. - """ - if not ops and not packages: - fail("image_install {}: declare the packages or operations that install it".format(name)) - _image_install(name = name, ops = flatten_operations(ops), packages = packages, **kwargs) diff --git a/image/publish.bzl b/image/publish.bzl index ab1f0ef7..0603df31 100644 --- a/image/publish.bzl +++ b/image/publish.bzl @@ -14,8 +14,8 @@ the metadata written beside it. That is the whole reason `image_artifacts` assem from a dynamic action: everything else in it is known while the graph is still being built. """ -load("//image:image.bzl", "ImageInfo", "image_metadata_subtargets") -load("//image_format:disk.bzl", "PartitionInfo") +load(":ddi.bzl", "PartitionInfo") +load(":image.bzl", "ImageInfo", "image_metadata_subtargets") PublishedInfo = provider( doc = "The artifacts one target contributes to a release, keyed by published name.", diff --git a/image_format/repart.py b/image/repart.py similarity index 100% rename from image_format/repart.py rename to image/repart.py diff --git a/image_format/repart_test.py b/image/repart_test.py similarity index 98% rename from image_format/repart_test.py rename to image/repart_test.py index 03dd10b8..4a0595d4 100644 --- a/image_format/repart_test.py +++ b/image/repart_test.py @@ -3,7 +3,7 @@ """Tests for the mkfs environment and for reading repart's partition report back. - buck test tine//image_format:test + buck test tine//image:test repart itself is not run here. """ diff --git a/image_format/sysext.bzl b/image/sysext.bzl similarity index 98% rename from image_format/sysext.bzl rename to image/sysext.bzl index 61eef255..a62dccf4 100644 --- a/image_format/sysext.bzl +++ b/image/sysext.bzl @@ -11,8 +11,9 @@ before merging. With `base`, only the delta layered above that image is packaged extension-release pins the base's ID/VERSION_ID. """ +load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") load( - "//image:image.bzl", + ":image.bzl", "IMAGE_TOOLS_ATTR", "ImageInfo", "ImageToolsInfo", @@ -20,9 +21,9 @@ load( "pkgdb_paths", "terminal_image_command", ) -load("//image:publish.bzl", "PublishedInfo") +load(":publish.bzl", "PublishedInfo") load( - "//image:sign.bzl", + ":sign.bzl", "SigningKeyInfo", "VERITY_KEY_ATTR", "external_signing_execution", @@ -30,7 +31,6 @@ load( "resolve_signing_key", "signing_key_spec", ) -load("//platforms:architecture.bzl", "ARCHITECTURES", "architecture") SysextImageInfo = provider( doc = "A systemd system-extension DDI generated from a logical image.", diff --git a/image_format/sysext.py b/image/sysext.py similarity index 100% rename from image_format/sysext.py rename to image/sysext.py diff --git a/image/uki_test.py b/image/uki_test.py index 86b2b9dd..d5b337a5 100644 --- a/image/uki_test.py +++ b/image/uki_test.py @@ -72,7 +72,7 @@ def test_secure_boot_alone(self) -> None: self.assertEqual(uki._signing_arguments(SECURE_BOOT, None, [], OPTIONS), SECURE_BOOT_ARGUMENTS) def test_expected_pcr_alone(self) -> None: - """The driver keeps the roles separate; only uki.bzl insists on signing what it seals.""" + """The driver accepts an expected-PCR key alone. declare_uki() requires a Secure Boot key with it.""" self.assertEqual(uki._signing_arguments(None, PCR, [], OPTIONS), PCR_ARGUMENTS) def test_the_initrd_policy_rides_along_with_the_expected_pcr_key(self) -> None: diff --git a/image/vm.bzl b/image/vm.bzl index c89a141e..800bd455 100644 --- a/image/vm.bzl +++ b/image/vm.bzl @@ -6,13 +6,13 @@ load("@prelude//platforms:defs.bzl", "host_configuration") load("//box:runtime.bzl", "BoxInfo", "box_run") load("//distribution:defs.bzl", "distribution") -load("//image_format:disk.bzl", "RepartInfo", "SIZE_PATTERN") -load("//image_format:sysext.bzl", "SysextImageInfo") +load(":ddi.bzl", "DdiInfo", "SIZE_PATTERN") +load(":sysext.bzl", "SysextImageInfo") def _image_vm_impl(ctx: AnalysisContext) -> list[Provider]: - disk = ctx.attrs.image[RepartInfo] + disk = ctx.attrs.image[DdiInfo] if disk.disk == None: - fail("image_vm: RepartInfo does not contain a composed disk") + fail("image_vm: DdiInfo does not contain a composed disk") run = cmd_args( box_run( box = ctx.attrs.box[BoxInfo], @@ -131,7 +131,7 @@ _image_vm = rule( default = None, doc = 'size to grow the disk file to before booting, e.g. "8G"; the built image itself grows', ), - "image": attrs.dep(providers = [RepartInfo], doc = "the raw disk image to boot ephemerally"), + "image": attrs.dep(providers = [DdiInfo], doc = "the raw disk image to boot ephemerally"), "ram": attrs.option( attrs.string(), default = None, diff --git a/image_format/BUCK b/image_format/BUCK deleted file mode 100644 index a6b71116..00000000 --- a/image_format/BUCK +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-FileCopyrightText: Amutable GmbH -# SPDX-License-Identifier: MPL-2.0 - -"""Terminal image-output format drivers.""" - -load("//box:test.bzl", "box_python_test") -load("//python:defs.bzl", "tine_python_binary", "tine_python_library") - -tine_python_binary( - name = "archive", - main = "archive.py", - deps = [ - "//:specs", - "//:util", - "//archive:cpio", - "//archive:tar", - "//image:finalize", - ], - visibility = ["PUBLIC"], -) - -tine_python_binary( - name = "disk", - main = "disk.py", - deps = [ - ":repart", - "//:specs", - "//:util", - "//image:finalize", - "//image:manifest-lib", - ], - visibility = ["PUBLIC"], -) - -tine_python_library( - name = "repart", - srcs = ["repart.py"], - deps = ["//:util"], -) - -tine_python_binary( - name = "convert", - main = "convert.py", - deps = ["//:specs", "//:util"], - visibility = ["PUBLIC"], -) - -tine_python_binary( - name = "sysext", - main = "sysext.py", - deps = [ - ":repart", - "//:specs", - "//:util", - "//image:finalize", - "//image:manifest-lib", - "//rootfs:rootfs", - ], - visibility = ["PUBLIC"], -) - -box_python_test( - name = "test", - box = "//tools:dev.box", - srcs = [ - "disk.py", - "disk_test.py", - "repart_test.py", - ], - deps = [ - ":repart", - "//:specs", - "//:util", - "//image:finalize", - "//image:manifest-lib", - ], -) diff --git a/image_format/boot.bzl b/image_format/boot.bzl deleted file mode 100644 index 825e5e92..00000000 --- a/image_format/boot.bzl +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: Amutable GmbH -# SPDX-License-Identifier: MPL-2.0 - -"""Semantic boot-artifact extraction from a completed logical image.""" - -load( - "//image:image.bzl", - "IMAGE_TOOLS_ATTR", - "ImageInfo", - "ImageToolsInfo", - "declare_out", - "terminal_image_command", -) - -_ARTIFACTS = { - "initrd": "initrd", - "kernel": "vmlinuz", - "uki": "image.efi", -} - -BootArtifactsInfo = record( - initrd = Artifact, - kernel = Artifact, - uki = Artifact, -) - -def declare_boot_artifacts( - ctx: AnalysisContext, - *, - tools: ImageToolsInfo, - image: ImageInfo, - identifier: str | None = None, -) -> BootArtifactsInfo: - """Declare extraction of what a completed image boots: its kernel, initrd and UKI. - - The driver picks the newest kernel the image carries and reads the initrd and kernel out of - the UKI's PE sections when it boots one, so the results are what a direct kernel boot needs. - """ - selection = declare_out(ctx, identifier, "boot-artifacts.json") - select = terminal_image_command( - ctx, - driver = "boot", - exe = tools.boot, - identifier = identifier, - image = image, - spec = {"out": selection.as_output()}, - ) - ctx.actions.run(select, category = "boot_artifact_select", identifier = identifier) - - artifacts = {} - for kind in sorted(_ARTIFACTS): - out = declare_out(ctx, identifier, _ARTIFACTS[kind]) - extract = terminal_image_command( - ctx, - # The extracted artifacts are named after their kind, so the specs of one composition - # step are told apart by driver name; scoping them by kind would name a directory - # after a file already declared beside it. - driver = "artifacts-" + kind, - exe = tools.artifacts, - identifier = identifier, - image = image, - spec = { - "artifact": kind, - "manifest": selection, - "out": out.as_output(), - }, - ) - ctx.actions.run(extract, category = "boot_artifact_" + kind, identifier = identifier) - artifacts[kind] = out - return BootArtifactsInfo( - initrd = artifacts["initrd"], - kernel = artifacts["kernel"], - uki = artifacts["uki"], - ) - -def boot_subtargets(artifacts: BootArtifactsInfo) -> dict[str, list[Provider]]: - """The extracted artifacts, exposed wherever the image they come from is.""" - return { - "initrd": [DefaultInfo(default_output = artifacts.initrd)], - "kernel": [DefaultInfo(default_output = artifacts.kernel)], - "uki": [DefaultInfo(default_output = artifacts.uki)], - } - -def _bootable_impl(ctx: AnalysisContext) -> list[Provider]: - artifacts = declare_boot_artifacts(ctx, tools = ctx.attrs._tools[ImageToolsInfo], image = ctx.attrs.image[ImageInfo]) - - # A UKI is not guaranteed to exist (kernels can ship as plain files), so the default - # outputs stay limited to the artifacts selection always yields; [uki] extracts on demand. - return [ - DefaultInfo( - default_outputs = [artifacts.kernel, artifacts.initrd], - sub_targets = boot_subtargets(artifacts), - ), - ] - -bootable = rule( - impl = _bootable_impl, - attrs = { - "image": attrs.dep( - providers = [ImageInfo], - doc = "the completed logical image from which to select boot artifacts", - ), - } - | IMAGE_TOOLS_ATTR, -) diff --git a/kmod/BUCK b/kmod/BUCK new file mode 100644 index 00000000..9e9b83e7 --- /dev/null +++ b/kmod/BUCK @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Kernel module selection for the initrd of a UKI.""" + +load("//box:test.bzl", "box_python_test") +load("//python:defs.bzl", "tine_python_library") + +tine_python_library( + name = "kmod", + srcs = ["kmod.py"], + deps = ["//:util"], + visibility = ["tine//..."], +) + +box_python_test( + name = "test", + box = "tine//tools:dev.box", + srcs = [ + "initrd_modules.toml", + "kmod_test.py", + ], + deps = [ + ":kmod", + "//archive:cpio", + ], +) diff --git a/image/initrd_modules.toml b/kmod/initrd_modules.toml similarity index 100% rename from image/initrd_modules.toml rename to kmod/initrd_modules.toml diff --git a/image/kmod.py b/kmod/kmod.py similarity index 100% rename from image/kmod.py rename to kmod/kmod.py diff --git a/image/kmod_test.py b/kmod/kmod_test.py similarity index 99% rename from image/kmod_test.py rename to kmod/kmod_test.py index 3eba0252..42b05c01 100644 --- a/image/kmod_test.py +++ b/kmod/kmod_test.py @@ -3,7 +3,7 @@ """Tests for kernel module selection and the explicit-path cpio packer. - buck test tine//image:test + buck test tine//kmod:test The pattern box, the firmware walk and the packing decide what a UKI carries, and none of them need libkmod or a real kernel, so they run against synthetic module trees here. The dependency diff --git a/image_format/pyproject.toml b/kmod/pyproject.toml similarity index 100% rename from image_format/pyproject.toml rename to kmod/pyproject.toml diff --git a/ruff.toml b/ruff.toml index 75b00690..2625f062 100644 --- a/ruff.toml +++ b/ruff.toml @@ -15,7 +15,7 @@ src = [ "cargo", "box", "image", - "image_format", + "kmod", "package", "package_system/pacman", "package_system/rpm",