From 7cd737de6aea51acbd2e2cd45944fecd1dd60c27 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 6 Nov 2026 11:02:19 +0000 Subject: [PATCH] http: generalize tool download to arbitrary files This allows projects to pin and build on a published file which is not a tool, such as a disk image. `http.tool` writes `-download` which hides the file's suffix, and marks it executable. Add `http.file` for the generalized "any pinned file download" part, and make `http.tool` a special case for it for a single executable program. Signed-off-by: Martin Pitt --- README.md | 2 +- docs/user/http.md | 37 +++++++++++++++++++++--------- http/BUCK | 2 +- http/defs.bzl | 57 ++++++++++++++++++++++++++--------------------- 4 files changed, 59 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index f307b1f3..610b7134 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ User guides: - [Maintaining packages](docs/user/importer.md): importing and updating packages from upstream distributions, local modifications, branch curation - [Development boxes](docs/user/box.md): pinned interactive development environments -- [Pinning prebuilt tools](docs/user/http.md): downloading a tool release pinned in `tine.lock.json` +- [Pinning downloads](docs/user/http.md): downloading a file or tool release pinned in `tine.lock.json` - [Signing with external keys](docs/user/signing-pkcs11.md): Secure Boot, PCR policy and verity signing over PKCS#11, with the keys outside the build - [Shared build cache](docs/user/remote-cache.md): configuring it, the keys, the bucket, and reading the diff --git a/docs/user/http.md b/docs/user/http.md index de307fbc..90385a5f 100644 --- a/docs/user/http.md +++ b/docs/user/http.md @@ -3,23 +3,34 @@ SPDX-FileCopyrightText: Amutable GmbH SPDX-License-Identifier: MPL-2.0 --> -# Pinning prebuilt tools +# Pinning downloads -`http.tool()` downloads a prebuilt tool for each CPU architecture that tine builds for. The project's -`tine.lock.json` pins every download by sha256 and size, and Buck checks the download against both. `bump` -updates the pins to the newest upstream release. +`http.file()` downloads a file for each CPU architecture that tine builds for, such as a disk image that +a project builds on. `http.tool()` does the same for a prebuilt program (possibly shipped as an archive), +and makes it runnable. The project's `tine.lock.json` pins every download by sha256 and size, and Buck +checks the download against both. `bump` updates the pins to the newest upstream release. -## Declaring a tool +## Declaring a download -Add an entry for the tool to `tine.lock.json` at the root of the project. The `ruff` entry in tine's own -[`tine.lock.json`](../../tine.lock.json) pins a GitHub release. +Add an entry for the download to `tine.lock.json` at the root of the project. The `ruff` entry in tine's +own [`tine.lock.json`](../../tine.lock.json) pins a GitHub release. -Then load the entry in a `BUCK` file and declare the tool: +Then load the entry in a `BUCK` file and declare the download. A file is the target's output, named +`name`: ```starlark load("//:tine.lock.json", pins = "value") load("@tine//http:defs.bzl", "http") +http.file( + name = "os.raw.zst", + spec = pins["os-disk"], +) +``` + +A program is a tool: + +```starlark http.tool( name = "ruff", path = "ruff", @@ -32,8 +43,8 @@ http.tool( ## The pin - `repository` and `release` name a GitHub release, and each `artifact` is a file attached to it. -- `url` replaces `repository` for a tool that is not published on GitHub. Each artifact is downloaded from - `/`, and `release` is the version. +- `url` replaces `repository` for a download that is not published on GitHub. Each artifact is downloaded + from `/`, and `release` is the version. - `releases` names the index that `bump` reads for a pin with a `url`. Its `url` is a page that lists the releases, and every match of its `pattern`, a Python regular expression, is a release. `bump` compares the numbers in each match and takes the highest. A prerelease such as `go1.28rc1` ranks by its numbers @@ -48,6 +59,10 @@ http.tool( ## The arguments - `spec` is the pin, as loaded from `tine.lock.json`. +- `executable = True` makes the file of `http.file()` executable. + +The arguments of `http.tool()` beyond `spec`: + - Without `path` or `directory`, the download is the program itself. - `path` names the program inside an archive. The program is the target's output. - `directory = True` makes the whole unpacked archive the target's output, for a tool that needs the files @@ -58,7 +73,7 @@ http.tool( ## Updating the pins -Run `tine bump` with the tools to update. It updates the `tine.lock.json` of the project that it runs in: +Run `tine bump` with the pins to update. It updates the `tine.lock.json` of the project that it runs in: ```sh tine bump --tool ruff diff --git a/http/BUCK b/http/BUCK index 806fbca8..c210142a 100644 --- a/http/BUCK +++ b/http/BUCK @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Prebuilt tools downloaded and verified by Buck.""" +"""Pinned files and prebuilt tools, downloaded and verified by Buck.""" load("//python:defs.bzl", "python") diff --git a/http/defs.bzl b/http/defs.bzl index 6133c92a..8fe416b4 100644 --- a/http/defs.bzl +++ b/http/defs.bzl @@ -41,6 +41,30 @@ _tool = rule( }, ) +def _download(spec: dict) -> dict[str, Select]: + """The `urls`, `sha256` and `size_bytes` of Buck's http rules for a tine.lock.json entry. + + The artifact is downloaded from the GitHub release of the entry's `repository`, or from the entry's + `url`. + """ + base = spec["url"] if "url" in spec else "https://github.com/{}/releases/download/{}".format(spec["repository"], spec["release"]) + platforms = spec["platforms"] + return { + "sha256": architecture.select({cpu: e["sha256"] for cpu, e in platforms.items()}), + # Without a size buck asks the server for one with a HEAD request before every download, which is + # a second chance for a release host to fail a build that has the bytes pinned already. + "size_bytes": architecture.select({cpu: e["size"] for cpu, e in platforms.items()}), + "urls": architecture.select({cpu: ["{}/{}".format(base, e["artifact"])] for cpu, e in platforms.items()}), + } + +def http_pinned_file(name: str, spec: dict, executable: bool = False, visibility: list[str] | None = None) -> None: + """Pin a file for each supported CPU, downloaded and sha256-verified by Buck. + + `spec` is a tine.lock.json entry: a `release`, and per-CPU `artifact`, `sha256` and `size`. The + target's output is the download, named `name`. + """ + http_file(name = name, executable = executable, visibility = visibility, **_download(spec)) + def http_tool( name: str, spec: dict, @@ -50,41 +74,21 @@ def http_tool( directory: bool = False, visibility: list[str] | None = None, ) -> None: - """Pin a prebuilt tool for each supported CPU, downloaded and sha256-verified by Buck. + """Pin a prebuilt program as http.file does, for `buck run` and as an executable dependency. - `spec` is a tine.lock.json entry: a `release`, and per-CPU `artifact`, `sha256` and `size` (+ optional - `strip_prefix`). The artifact is downloaded from the GitHub release of the entry's `repository`, or from - the entry's `url`. `path` selects a member of an archive; `compressed` decompresses a bare + The download is the program itself, or an archive that contains it, whose entry in `spec` may have a + `strip_prefix`. `path` selects a member of an archive; `compressed` decompresses a bare zstd-compressed binary, which Buck's http rules cannot unpack themselves. `directory` makes the whole unpacked archive the default output, and `path` then names the program that `buck run` runs. """ if directory and compressed: fail("http.tool {}: a compressed download is a single binary, not a directory".format(name)) - base = spec["url"] if "url" in spec else "https://github.com/{}/releases/download/{}".format(spec["repository"], spec["release"]) - platforms = spec["platforms"] - urls = architecture.select({cpu: ["{}/{}".format(base, e["artifact"])] for cpu, e in platforms.items()}) - sha256 = architecture.select({cpu: e["sha256"] for cpu, e in platforms.items()}) - - # Without a size buck asks the server for one with a HEAD request before every download, which is - # a second chance for a release host to fail a build that has the bytes pinned already. - size_bytes = architecture.select({cpu: e["size"] for cpu, e in platforms.items()}) if path == None and not directory: - http_file( - name = name + "-download", - executable = True, - sha256 = sha256, - size_bytes = size_bytes, - urls = urls, - ) + http_pinned_file(name = name + "-download", spec = spec, executable = True) strip_prefix = None else: - http_archive( - name = name + "-download", - sha256 = sha256, - size_bytes = size_bytes, - urls = urls, - ) - strip_prefix = architecture.select({cpu: e.get("strip_prefix") for cpu, e in platforms.items()}) + http_archive(name = name + "-download", **_download(spec)) + strip_prefix = architecture.select({cpu: e.get("strip_prefix") for cpu, e in spec["platforms"].items()}) _tool( name = name, args = args or [], @@ -97,5 +101,6 @@ def http_tool( ) http = struct( + file = http_pinned_file, tool = http_tool, )