From aef97a6b6e140ebc909766651cc7ce2d20afe641 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 18:34:16 +0000 Subject: [PATCH 01/15] Link a cached prebuilt AWS-LC in CI instead of compiling it on every run `install-tools` now restores an AWS-LC install prefix from the Actions cache, building it from the workspace's `aws-lc-sys` on a miss, and points `aws-lc-sys` at it through `AWS_LC_SYS_SYSTEM_DIR_` and `AWS_LC_SYS_STATIC_`. The prefix holds the library, headers and bindings of an ordinary `dev` source build, so the linked result matches compiling it. Bench and CodSpeed jobs build with optimized profiles and keep compiling AWS-LC. --- .github/actions/install-aws-lc/action.yml | 75 +++++++++++++++++++ .../actions/install-aws-lc/build-prefix.sh | 54 +++++++++++++ .github/actions/install-tools/action.yml | 7 ++ .github/workflows/bench.yml | 4 + .github/workflows/codspeed.yml | 1 + turbo.json | 1 + 6 files changed, 142 insertions(+) create mode 100644 .github/actions/install-aws-lc/action.yml create mode 100755 .github/actions/install-aws-lc/build-prefix.sh diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml new file mode 100644 index 00000000000..f0c6ebaea29 --- /dev/null +++ b/.github/actions/install-aws-lc/action.yml @@ -0,0 +1,75 @@ +name: Install AWS-LC +description: Restores a prebuilt AWS-LC from the Actions cache, building it on a miss, and makes `aws-lc-sys` link it instead of compiling AWS-LC. + +inputs: + profile: + description: Cargo profile the prebuilt library is compiled with. Every build in the job links it, whatever profile that build uses. + default: dev + +runs: + using: composite + steps: + - name: Compute AWS-LC cache key + id: key + shell: bash + env: + PROFILE: ${{ inputs.profile }} + ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', 'rust-toolchain.toml', '.cargo/config.toml') }} + run: | + locked=$(awk '/^\[\[package\]\]/ { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block; block = "" } { block = block "\n" $0 } END { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block }' Cargo.lock) + version=$(sed -n '/name = "aws-lc-sys"/{n;s/^version = "\(.*\)"$/\1/p;}' <<<"$locked") + if [[ -z $version ]]; then + echo "::error::aws-lc-sys is not in Cargo.lock" + exit 1 + fi + inputs=$( + echo "$locked" + echo "profile=$PROFILE" + echo "files=$ACTION_FILES" + rustc -vV + cc --version + ar --version | head -n 1 + env | grep -E '^(TARGET_|HOST_)?(CC|CFLAGS|AR|ARFLAGS|CRATE_CC_NO_DEFAULTS|AWS_LC_SYS_[A-Z_]+)(_[A-Za-z0-9_.-]+)?=' | grep -vE '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' | sort || true + ) + echo "$inputs" + { + echo "version=$version" + echo "key=aws-lc-sys-$version-${{ runner.os }}-${{ runner.arch }}-$PROFILE-$(sha256sum <<<"$inputs" | cut -c 1-16)" + echo "prefix=${{ runner.temp }}/aws-lc" + } >>"$GITHUB_OUTPUT" + + - name: Restore AWS-LC + id: restore + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.key.outputs.prefix }} + key: ${{ steps.key.outputs.key }} + + - name: Build AWS-LC + if: steps.restore.outputs.cache-hit != 'true' + shell: bash + run: ${{ github.action_path }}/build-prefix.sh "${{ steps.key.outputs.prefix }}" "${{ inputs.profile }}" + + - name: Save AWS-LC + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.key.outputs.prefix }} + key: ${{ steps.key.outputs.key }} + + - name: Link AWS-LC from the prefix + shell: bash + env: + PREFIX: ${{ steps.key.outputs.prefix }} + VERSION: ${{ steps.key.outputs.version }} + run: | + built=$(cat "$PREFIX/aws-lc-sys.version") + if [[ $built != "$VERSION" ]]; then + echo "::error::the AWS-LC prefix was built for aws-lc-sys $built, but Cargo.lock has $VERSION" + exit 1 + fi + target=$(rustc -vV | sed -n 's/^host: //p' | tr '-' '_') + { + echo "AWS_LC_SYS_SYSTEM_DIR_${target}=$PREFIX" + echo "AWS_LC_SYS_STATIC_${target}=1" + } >>"$GITHUB_ENV" diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh new file mode 100755 index 00000000000..db6831150e1 --- /dev/null +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Usage: build-prefix.sh [cargo profile] +# +# Builds `aws-lc-sys` from source and lays its library, headers and bindings out as an install +# prefix, which `aws-lc-sys` links when given `AWS_LC_SYS_SYSTEM_DIR` and `AWS_LC_SYS_STATIC=1`. +set -euo pipefail + +prefix=$1 +profile=${2:-dev} + +target_dir=$(mktemp -d) +trap 'rm -rf "$target_dir"' EXIT + +# Compiling AWS-LC locally is faster than fetching each object from the remote compiler cache. +unset RUSTC_WRAPPER +while read -r name; do unset "$name"; done < <(compgen -e | grep -E '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' || true) + +package=$(cargo metadata --locked --format-version 1 | + jq --compact-output --exit-status '[.packages[] | select(.name == "aws-lc-sys")] | if length == 1 then .[0] else error("expected one aws-lc-sys package, found \(length)") end') +manifest_dir=$(dirname "$(jq --raw-output '.manifest_path' <<<"$package")") +version=$(jq --raw-output '.version' <<<"$package") + +build_script=$(cargo build --locked --package aws-lc-sys --profile "$profile" --target-dir "$target_dir" --message-format json | + jq --compact-output 'select(.reason == "build-script-executed" and (.package_id | test("aws-lc-sys")))') + +out_dir=$(jq --raw-output --exit-status '.out_dir' <<<"$build_script") +library=$(jq --raw-output --exit-status '[.linked_libs[] | select(startswith("static=") and endswith("_crypto"))] | if length == 1 then .[0] | ltrimstr("static=") else error("expected one static crypto library, found \(length)") end' <<<"$build_script") + +if jq --exit-status '.cfgs | index("use_bindgen_pregenerated")' <<<"$build_script" >/dev/null; then + bindings=$out_dir/bindings.rs +else + bindings=$(jq --raw-output --exit-status --arg src "$manifest_dir/src" '[.cfgs[] | $src + "/" + . + "_crypto.rs"]' <<<"$build_script" | + jq --raw-output '.[]' | while read -r candidate; do [[ -f $candidate ]] && echo "$candidate"; done) + if [[ $(wc -l <<<"$bindings") -ne 1 || -z $bindings ]]; then + echo "::error::expected one pregenerated bindings file for the emitted cfgs, found: ${bindings:-none}" + exit 1 + fi +fi + +rm -rf "$prefix" +mkdir -p "$prefix/lib" "$prefix/share/rust" +cp "$out_dir/lib${library}.a" "$prefix/lib/libcrypto-awslc.a" +cp -R "$out_dir/include" "$prefix/include" + +# `aws-lc-sys` includes these bindings inside a module that carries the same `allow` list, and an +# included file cannot start with an inner attribute. +sed '/^#!\[allow($/,/^)\]$/d' "$bindings" >"$prefix/share/rust/aws_lc_bindings.rs" +if grep --quiet '^#!' "$prefix/share/rust/aws_lc_bindings.rs"; then + echo "::error::could not remove the inner attributes from $bindings" + exit 1 +fi + +echo "$version" >"$prefix/aws-lc-sys.version" +echo "Built the AWS-LC prefix for aws-lc-sys $version at $prefix" diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index 763e12d5c50..27a244a05e5 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -17,6 +17,9 @@ inputs: sccache: description: Should sccache be installed? Can either be `"true"` or `true` default: "true" + aws_lc: + description: Should `aws-lc-sys` link a prebuilt AWS-LC compiled with the `dev` profile? Can either be `"true"` or `true`. Disable it in jobs that build with an optimized profile. + default: "true" runs: using: composite @@ -78,6 +81,10 @@ runs: retry_wait_seconds: 60 command: ${{ github.action_path }}/install-rust.sh + - name: "Install AWS-LC" + if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.aws_lc == true || inputs.aws_lc == 'true') }} + uses: $/.github/actions/install-aws-lc + - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} continue-on-error: true diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0cedef3293d..e2e780e95be 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -110,6 +110,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + aws_lc: false - name: Prune repository if: github.event_name == 'pull_request' @@ -148,6 +149,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + aws_lc: false - name: Prune repository uses: $/.github/actions/prune-repository @@ -231,6 +233,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + aws_lc: false - name: Prune repository if: github.event_name == 'pull_request' @@ -327,6 +330,7 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} sccache: false # sccache is already running in the background + aws_lc: false - name: Prune repository uses: $/.github/actions/prune-repository diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 3c4f0caaa45..d17018f849a 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -89,6 +89,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + aws_lc: false - name: Prune repository uses: $/.github/actions/prune-repository diff --git a/turbo.json b/turbo.json index 8e5b1d9af17..ee684a7a468 100644 --- a/turbo.json +++ b/turbo.json @@ -154,6 +154,7 @@ "yarn*" ], "globalPassThroughEnv": [ + "AWS_LC_SYS_*", "CARGO_TERM_PROGRESS_WHEN", "RUSTC_WRAPPER", "CARGO_INCREMENTAL" From 1622773f3efa90eee803005338a40099e3d1910a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 18:34:16 +0000 Subject: [PATCH 02/15] Add a temporary workflow measuring the hash-graph compile with and without the prebuilt AWS-LC --- .../workflows/hash-graph-compile-timings.yml | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/hash-graph-compile-timings.yml diff --git a/.github/workflows/hash-graph-compile-timings.yml b/.github/workflows/hash-graph-compile-timings.yml new file mode 100644 index 00000000000..e17983787ec --- /dev/null +++ b/.github/workflows/hash-graph-compile-timings.yml @@ -0,0 +1,72 @@ +name: hash-graph compile timings + +on: + pull_request: + paths: + - .github/workflows/hash-graph-compile-timings.yml + - .github/actions/install-aws-lc/** + +jobs: + sccache-credentials: + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + outputs: + encrypted-credentials: ${{ steps.mint.outputs.encrypted_credentials }} + steps: + - name: Checkout source code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Mint sccache credentials + id: mint + uses: $/.github/actions/mint-sccache-credentials + with: + vault_address: ${{ vars.VAULT_STAGE_ADDR }} + cf_access_client_id: ${{ vars.CF_ACCESS_STAGE_CLIENT_ID }} + cf_access_client_secret: ${{ secrets.CF_ACCESS_STAGE_CLIENT_SECRET }} + encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + + compile: + name: Compile (aws-lc=${{ matrix.aws_lc }}) + needs: [sccache-credentials] + runs-on: ubuntu-24.04 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + aws_lc: [prebuilt, source] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Clean up disk + uses: $/.github/actions/clean-up-disk + + - name: Install tools + id: install-tools + uses: $/.github/actions/install-tools + with: + token: ${{ secrets.GITHUB_TOKEN }} + encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} + sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} + aws_lc: ${{ matrix.aws_lc == 'prebuilt' }} + + - name: Compile hash-graph with timings + run: | + "$RUSTC_WRAPPER" --zero-stats + SECONDS=0 + cargo build --bin hash-graph --all-features --timings + echo "::notice::Full compile took ${SECONDS}s" + stats=$("$RUSTC_WRAPPER" --show-stats) + echo "$stats" + echo "::notice::sccache: $(grep -E '^Cache (hits|misses) ' <<<"$stats" | tr -s ' ' | paste -sd ';')" + python3 - target/cargo-timings/cargo-timing.html <<'EOF' + import json, re, sys + units = json.loads(re.search(r"UNIT_DATA = (\[.*?\]);", open(sys.argv[1]).read(), re.S).group(1)) + for unit in units: + if unit["name"] == "aws-lc-sys": + print(f"::notice::aws-lc-sys {unit.get('target') or 'lib'}: start {unit['start']:.1f}s, {unit['duration']:.1f}s") + EOF + grep -h 'Building with' target/debug/build/aws-lc-sys*/*/output target/debug/build/aws-lc-sys/*/run/stdout 2>/dev/null | sed 's/^cargo:warning=/::notice::aws-lc-sys /' || true From 3e527454c20b3816b07a296e19c5af8d18bff93d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 18:45:40 +0000 Subject: [PATCH 03/15] Build the prebuilt AWS-LC per profile and pass action values through the environment Every job that installs Rust now links a prebuilt AWS-LC. Jobs building with an optimized profile select a `release` prefix instead of compiling AWS-LC, and each profile has its own cache entry. The composite action reads expressions through `env` rather than interpolating them into scripts. --- .github/actions/install-aws-lc/action.yml | 20 +++++++++++++++----- .github/actions/install-tools/action.yml | 10 ++++++---- .github/workflows/bench.yml | 8 ++++---- .github/workflows/codspeed.yml | 2 +- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index f0c6ebaea29..8745e898133 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -3,7 +3,7 @@ description: Restores a prebuilt AWS-LC from the Actions cache, building it on a inputs: profile: - description: Cargo profile the prebuilt library is compiled with. Every build in the job links it, whatever profile that build uses. + description: Cargo profile the job's Rust builds use, `dev` or `release`. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev runs: @@ -15,7 +15,13 @@ runs: env: PROFILE: ${{ inputs.profile }} ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', 'rust-toolchain.toml', '.cargo/config.toml') }} + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} run: | + if [[ $PROFILE != dev && $PROFILE != release ]]; then + echo "::error::the AWS-LC profile must be \`dev\` or \`release\`, not \`$PROFILE\`" + exit 1 + fi locked=$(awk '/^\[\[package\]\]/ { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block; block = "" } { block = block "\n" $0 } END { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block }' Cargo.lock) version=$(sed -n '/name = "aws-lc-sys"/{n;s/^version = "\(.*\)"$/\1/p;}' <<<"$locked") if [[ -z $version ]]; then @@ -29,13 +35,13 @@ runs: rustc -vV cc --version ar --version | head -n 1 - env | grep -E '^(TARGET_|HOST_)?(CC|CFLAGS|AR|ARFLAGS|CRATE_CC_NO_DEFAULTS|AWS_LC_SYS_[A-Z_]+)(_[A-Za-z0-9_.-]+)?=' | grep -vE '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' | sort || true + env | grep -E '^((TARGET_|HOST_)?(CC|CFLAGS|AR|ARFLAGS|CRATE_CC_NO_DEFAULTS)(_[A-Za-z0-9_.-]+)?|AWS_LC_SYS_[A-Za-z0-9_]+|CARGO_PROFILE_[A-Z_]+|(CARGO_ENCODED_)?RUSTFLAGS)=' | grep -vE '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' | sort || true ) echo "$inputs" { echo "version=$version" - echo "key=aws-lc-sys-$version-${{ runner.os }}-${{ runner.arch }}-$PROFILE-$(sha256sum <<<"$inputs" | cut -c 1-16)" - echo "prefix=${{ runner.temp }}/aws-lc" + echo "key=aws-lc-sys-$version-$RUNNER_OS_NAME-$RUNNER_ARCH_NAME-$PROFILE-$(sha256sum <<<"$inputs" | cut -c 1-16)" + echo "prefix=$RUNNER_TEMP/aws-lc-$PROFILE" } >>"$GITHUB_OUTPUT" - name: Restore AWS-LC @@ -48,7 +54,11 @@ runs: - name: Build AWS-LC if: steps.restore.outputs.cache-hit != 'true' shell: bash - run: ${{ github.action_path }}/build-prefix.sh "${{ steps.key.outputs.prefix }}" "${{ inputs.profile }}" + env: + BUILD_PREFIX: ${{ github.action_path }}/build-prefix.sh + PREFIX: ${{ steps.key.outputs.prefix }} + PROFILE: ${{ inputs.profile }} + run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE"' - name: Save AWS-LC if: steps.restore.outputs.cache-hit != 'true' diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index 27a244a05e5..d3e5d5c3f4b 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -17,9 +17,9 @@ inputs: sccache: description: Should sccache be installed? Can either be `"true"` or `true` default: "true" - aws_lc: - description: Should `aws-lc-sys` link a prebuilt AWS-LC compiled with the `dev` profile? Can either be `"true"` or `true`. Disable it in jobs that build with an optimized profile. - default: "true" + aws_lc_profile: + description: Cargo profile, `dev` or `release`, that the prebuilt AWS-LC `aws-lc-sys` links is compiled with. Set it to the profile the job's Rust builds use. + default: dev runs: using: composite @@ -82,8 +82,10 @@ runs: command: ${{ github.action_path }}/install-rust.sh - name: "Install AWS-LC" - if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.aws_lc == true || inputs.aws_lc == 'true') }} + if: ${{ inputs.rust == true || inputs.rust == 'true' }} uses: $/.github/actions/install-aws-lc + with: + profile: ${{ inputs.aws_lc_profile }} - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e2e780e95be..4f0395c6280 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -110,7 +110,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - aws_lc: false + aws_lc_profile: release - name: Prune repository if: github.event_name == 'pull_request' @@ -149,7 +149,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - aws_lc: false + aws_lc_profile: release - name: Prune repository uses: $/.github/actions/prune-repository @@ -233,7 +233,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - aws_lc: false + aws_lc_profile: release - name: Prune repository if: github.event_name == 'pull_request' @@ -330,7 +330,7 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} sccache: false # sccache is already running in the background - aws_lc: false + aws_lc_profile: release - name: Prune repository uses: $/.github/actions/prune-repository diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d17018f849a..382c8c03158 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -89,7 +89,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - aws_lc: false + aws_lc_profile: release - name: Prune repository uses: $/.github/actions/prune-repository From 6142bd4627b6c7f2546219ba28f0c13a3e4e3c27 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 18:45:41 +0000 Subject: [PATCH 04/15] Measure only the prebuilt AWS-LC in the temporary compile timings workflow --- .github/workflows/hash-graph-compile-timings.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/hash-graph-compile-timings.yml b/.github/workflows/hash-graph-compile-timings.yml index e17983787ec..4120169ccc9 100644 --- a/.github/workflows/hash-graph-compile-timings.yml +++ b/.github/workflows/hash-graph-compile-timings.yml @@ -28,15 +28,11 @@ jobs: encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} compile: - name: Compile (aws-lc=${{ matrix.aws_lc }}) + name: Compile needs: [sccache-credentials] runs-on: ubuntu-24.04 permissions: contents: read - strategy: - fail-fast: false - matrix: - aws_lc: [prebuilt, source] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -45,13 +41,11 @@ jobs: uses: $/.github/actions/clean-up-disk - name: Install tools - id: install-tools uses: $/.github/actions/install-tools with: token: ${{ secrets.GITHUB_TOKEN }} encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - aws_lc: ${{ matrix.aws_lc == 'prebuilt' }} - name: Compile hash-graph with timings run: | From a1d7572ba909f2060b885abae222712def879cde Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:14:10 +0000 Subject: [PATCH 05/15] Build the AWS-LC prefix with AWS-LC's own CMake install `build-prefix.sh` now downloads the locked `aws-lc-sys` crate, checks it against the Cargo.lock checksum, and builds and installs its bundled AWS-LC with CMake and the Rust bindings that build generates, keeping the crate's symbol prefix. The cache key moves to `cache-key.py`, which reads Cargo.lock with `tomllib`. --- .github/actions/install-aws-lc/action.yml | 40 ++------ .../actions/install-aws-lc/bindgen-version | 1 + .../actions/install-aws-lc/build-prefix.sh | 91 +++++++++---------- .github/actions/install-aws-lc/cache-key.py | 59 ++++++++++++ .github/actions/install-tools/action.yml | 1 + 5 files changed, 116 insertions(+), 76 deletions(-) create mode 100644 .github/actions/install-aws-lc/bindgen-version create mode 100644 .github/actions/install-aws-lc/cache-key.py diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index 8745e898133..8fffaa5f957 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -5,6 +5,9 @@ inputs: profile: description: Cargo profile the job's Rust builds use, `dev` or `release`. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev + token: + description: GitHub token `cargo-binstall` uses to download `bindgen-cli` when the prefix has to be built. + required: true runs: using: composite @@ -13,36 +16,10 @@ runs: id: key shell: bash env: + CACHE_KEY: ${{ github.action_path }}/cache-key.py PROFILE: ${{ inputs.profile }} - ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', 'rust-toolchain.toml', '.cargo/config.toml') }} - RUNNER_OS_NAME: ${{ runner.os }} - RUNNER_ARCH_NAME: ${{ runner.arch }} - run: | - if [[ $PROFILE != dev && $PROFILE != release ]]; then - echo "::error::the AWS-LC profile must be \`dev\` or \`release\`, not \`$PROFILE\`" - exit 1 - fi - locked=$(awk '/^\[\[package\]\]/ { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block; block = "" } { block = block "\n" $0 } END { if (block ~ /\nname = "(aws-lc-sys|cc)"\n/) print block }' Cargo.lock) - version=$(sed -n '/name = "aws-lc-sys"/{n;s/^version = "\(.*\)"$/\1/p;}' <<<"$locked") - if [[ -z $version ]]; then - echo "::error::aws-lc-sys is not in Cargo.lock" - exit 1 - fi - inputs=$( - echo "$locked" - echo "profile=$PROFILE" - echo "files=$ACTION_FILES" - rustc -vV - cc --version - ar --version | head -n 1 - env | grep -E '^((TARGET_|HOST_)?(CC|CFLAGS|AR|ARFLAGS|CRATE_CC_NO_DEFAULTS)(_[A-Za-z0-9_.-]+)?|AWS_LC_SYS_[A-Za-z0-9_]+|CARGO_PROFILE_[A-Z_]+|(CARGO_ENCODED_)?RUSTFLAGS)=' | grep -vE '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' | sort || true - ) - echo "$inputs" - { - echo "version=$version" - echo "key=aws-lc-sys-$version-$RUNNER_OS_NAME-$RUNNER_ARCH_NAME-$PROFILE-$(sha256sum <<<"$inputs" | cut -c 1-16)" - echo "prefix=$RUNNER_TEMP/aws-lc-$PROFILE" - } >>"$GITHUB_OUTPUT" + ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**') }} + run: python3 "$CACHE_KEY" "$PROFILE" - name: Restore AWS-LC id: restore @@ -58,7 +35,10 @@ runs: BUILD_PREFIX: ${{ github.action_path }}/build-prefix.sh PREFIX: ${{ steps.key.outputs.prefix }} PROFILE: ${{ inputs.profile }} - run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE"' + VERSION: ${{ steps.key.outputs.version }} + CHECKSUM: ${{ steps.key.outputs.checksum }} + GITHUB_TOKEN: ${{ inputs.token }} + run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE" "$VERSION" "$CHECKSUM"' - name: Save AWS-LC if: steps.restore.outputs.cache-hit != 'true' diff --git a/.github/actions/install-aws-lc/bindgen-version b/.github/actions/install-aws-lc/bindgen-version new file mode 100644 index 00000000000..fac6d18325a --- /dev/null +++ b/.github/actions/install-aws-lc/bindgen-version @@ -0,0 +1 @@ +0.72.1 diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index db6831150e1..f9089097b94 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -1,54 +1,53 @@ #!/usr/bin/env bash -# Usage: build-prefix.sh [cargo profile] +# Usage: build-prefix.sh # -# Builds `aws-lc-sys` from source and lays its library, headers and bindings out as an install -# prefix, which `aws-lc-sys` links when given `AWS_LC_SYS_SYSTEM_DIR` and `AWS_LC_SYS_STATIC=1`. +# Builds the AWS-LC source bundled in the `aws-lc-sys` crate with AWS-LC's own CMake build, and +# installs the static library, headers and generated Rust bindings into , which +# `aws-lc-sys` links when `AWS_LC_SYS_SYSTEM_DIR` points at it. set -euo pipefail prefix=$1 -profile=${2:-dev} - -target_dir=$(mktemp -d) -trap 'rm -rf "$target_dir"' EXIT - -# Compiling AWS-LC locally is faster than fetching each object from the remote compiler cache. -unset RUSTC_WRAPPER -while read -r name; do unset "$name"; done < <(compgen -e | grep -E '^AWS_LC_SYS_(SYSTEM_DIR|STATIC)' || true) - -package=$(cargo metadata --locked --format-version 1 | - jq --compact-output --exit-status '[.packages[] | select(.name == "aws-lc-sys")] | if length == 1 then .[0] else error("expected one aws-lc-sys package, found \(length)") end') -manifest_dir=$(dirname "$(jq --raw-output '.manifest_path' <<<"$package")") -version=$(jq --raw-output '.version' <<<"$package") - -build_script=$(cargo build --locked --package aws-lc-sys --profile "$profile" --target-dir "$target_dir" --message-format json | - jq --compact-output 'select(.reason == "build-script-executed" and (.package_id | test("aws-lc-sys")))') - -out_dir=$(jq --raw-output --exit-status '.out_dir' <<<"$build_script") -library=$(jq --raw-output --exit-status '[.linked_libs[] | select(startswith("static=") and endswith("_crypto"))] | if length == 1 then .[0] | ltrimstr("static=") else error("expected one static crypto library, found \(length)") end' <<<"$build_script") - -if jq --exit-status '.cfgs | index("use_bindgen_pregenerated")' <<<"$build_script" >/dev/null; then - bindings=$out_dir/bindings.rs -else - bindings=$(jq --raw-output --exit-status --arg src "$manifest_dir/src" '[.cfgs[] | $src + "/" + . + "_crypto.rs"]' <<<"$build_script" | - jq --raw-output '.[]' | while read -r candidate; do [[ -f $candidate ]] && echo "$candidate"; done) - if [[ $(wc -l <<<"$bindings") -ne 1 || -z $bindings ]]; then - echo "::error::expected one pregenerated bindings file for the emitted cfgs, found: ${bindings:-none}" +profile=$2 +version=$3 +checksum=$4 + +case $profile in + dev) build_type=Debug ;; + release) build_type=Release ;; + *) + echo "::error::unknown profile \`$profile\`, expected \`dev\` or \`release\`" exit 1 - fi -fi - -rm -rf "$prefix" -mkdir -p "$prefix/lib" "$prefix/share/rust" -cp "$out_dir/lib${library}.a" "$prefix/lib/libcrypto-awslc.a" -cp -R "$out_dir/include" "$prefix/include" - -# `aws-lc-sys` includes these bindings inside a module that carries the same `allow` list, and an -# included file cannot start with an inner attribute. -sed '/^#!\[allow($/,/^)\]$/d' "$bindings" >"$prefix/share/rust/aws_lc_bindings.rs" -if grep --quiet '^#!' "$prefix/share/rust/aws_lc_bindings.rs"; then - echo "::error::could not remove the inner attributes from $bindings" - exit 1 -fi + ;; +esac + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +curl --fail --silent --show-error --location --retry 3 \ + --output "$work/aws-lc-sys.crate" "https://static.crates.io/crates/aws-lc-sys/aws-lc-sys-$version.crate" +sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" +tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" +crate=$work/aws-lc-sys-$version + +cargo binstall --no-confirm --locked --root "$work/bindgen" "bindgen-cli@$(cat "$(dirname "$0")/bindgen-version")" +export PATH=$work/bindgen/bin:$PATH + +# These options match the CMake build `aws-lc-sys` itself runs, including the crate's symbol +# prefix, so the bindings and library match what `aws-lc-rs` is compiled against. +cmake -S "$crate/aws-lc" -B "$work/build" \ + -DCMAKE_BUILD_TYPE="$build_type" \ + -DCMAKE_INSTALL_PREFIX="$prefix" \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_LIBSSL=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_TOOL=OFF \ + -DDISABLE_GO=ON \ + -DDISABLE_PERL=ON \ + -DENABLE_SOURCE_MODIFICATION=OFF \ + -DBORINGSSL_PREFIX="aws_lc_${version//./_}" \ + -DBORINGSSL_PREFIX_HEADERS="$crate/generated-include" \ + -DGENERATE_RUST_BINDINGS=ON +cmake --build "$work/build" --parallel "$(nproc)" +cmake --install "$work/build" echo "$version" >"$prefix/aws-lc-sys.version" -echo "Built the AWS-LC prefix for aws-lc-sys $version at $prefix" diff --git a/.github/actions/install-aws-lc/cache-key.py b/.github/actions/install-aws-lc/cache-key.py new file mode 100644 index 00000000000..c9a45fbd37a --- /dev/null +++ b/.github/actions/install-aws-lc/cache-key.py @@ -0,0 +1,59 @@ +""" +Usage: cache-key.py + +Writes the locked `aws-lc-sys` version and checksum, the cache key of its AWS-LC prefix and the +prefix path as GitHub Actions step outputs. +""" + +import hashlib +import os +import re +import subprocess +import sys +import tomllib + +profile = sys.argv[1] +if profile not in ("dev", "release"): + sys.exit(f"::error::unknown profile `{profile}`, expected `dev` or `release`") + +with open("Cargo.lock", "rb") as lockfile: + packages = [ + p for p in tomllib.load(lockfile)["package"] if p["name"] == "aws-lc-sys" + ] +if len(packages) != 1: + sys.exit( + f"::error::expected one aws-lc-sys package in Cargo.lock, found {len(packages)}" + ) +version = packages[0]["version"] +checksum = packages[0]["checksum"] + +build_environment = sorted( + f"{name}={value}" + for name, value in os.environ.items() + if re.fullmatch(r"(CC|CFLAGS|ASMFLAGS|LDFLAGS|CMAKE_[A-Z_]+)", name) +) +inputs = "\n".join( + [ + checksum, + profile, + os.environ["ACTION_FILES"], + subprocess.run( + ["cc", "--version"], check=True, capture_output=True, text=True + ).stdout, + subprocess.run( + ["cmake", "--version"], check=True, capture_output=True, text=True + ).stdout, + *build_environment, + ] +) +print(inputs) +digest = hashlib.sha256(inputs.encode()).hexdigest()[:16] + +with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print(f"version={version}", file=output) + print(f"checksum={checksum}", file=output) + print( + f"key=aws-lc-sys-{version}-{os.environ['RUNNER_OS']}-{os.environ['RUNNER_ARCH']}-{profile}-{digest}", + file=output, + ) + print(f"prefix={os.environ['RUNNER_TEMP']}/aws-lc-{profile}", file=output) diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index d3e5d5c3f4b..2a3bbb56914 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -86,6 +86,7 @@ runs: uses: $/.github/actions/install-aws-lc with: profile: ${{ inputs.aws_lc_profile }} + token: ${{ inputs.token }} - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} From 0c149448396b47646951e52980cef312d6ef634e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:33:15 +0000 Subject: [PATCH 06/15] Install `bindgen-cli` through mise for the AWS-LC prefix build The CI mise environment pins `bindgen-cli`, and the cache key includes the version on `PATH`. `build-prefix.sh` reports its build time as a notice, which marks the jobs that missed the cache. --- .config/mise/config.ci.toml | 1 + .config/mise/mise.ci.lock | 4 ++++ .github/actions/install-aws-lc/action.yml | 4 ---- .github/actions/install-aws-lc/bindgen-version | 1 - .github/actions/install-aws-lc/build-prefix.sh | 4 +--- .github/actions/install-aws-lc/cache-key.py | 3 +++ .github/actions/install-tools/action.yml | 1 - 7 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 .github/actions/install-aws-lc/bindgen-version diff --git a/.config/mise/config.ci.toml b/.config/mise/config.ci.toml index 60976042799..62c7dd13d43 100644 --- a/.config/mise/config.ci.toml +++ b/.config/mise/config.ci.toml @@ -1,4 +1,5 @@ [tools] +"cargo:bindgen-cli" = "0.72.1" "cargo:clippy-sarif" = "0.8.0" "cargo:sarif-fmt" = "0.8.0" diff --git a/.config/mise/mise.ci.lock b/.config/mise/mise.ci.lock index 172b11c8f40..5919782e5e2 100644 --- a/.config/mise/mise.ci.lock +++ b/.config/mise/mise.ci.lock @@ -1,5 +1,9 @@ # @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html +[[tools."cargo:bindgen-cli"]] +version = "0.72.1" +backend = "cargo:bindgen-cli" + [[tools."cargo:clippy-sarif"]] version = "0.8.0" backend = "cargo:clippy-sarif" diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index 8fffaa5f957..a4d04909010 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -5,9 +5,6 @@ inputs: profile: description: Cargo profile the job's Rust builds use, `dev` or `release`. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev - token: - description: GitHub token `cargo-binstall` uses to download `bindgen-cli` when the prefix has to be built. - required: true runs: using: composite @@ -37,7 +34,6 @@ runs: PROFILE: ${{ inputs.profile }} VERSION: ${{ steps.key.outputs.version }} CHECKSUM: ${{ steps.key.outputs.checksum }} - GITHUB_TOKEN: ${{ inputs.token }} run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE" "$VERSION" "$CHECKSUM"' - name: Save AWS-LC diff --git a/.github/actions/install-aws-lc/bindgen-version b/.github/actions/install-aws-lc/bindgen-version deleted file mode 100644 index fac6d18325a..00000000000 --- a/.github/actions/install-aws-lc/bindgen-version +++ /dev/null @@ -1 +0,0 @@ -0.72.1 diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index f9089097b94..09b2fb319f7 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -29,9 +29,6 @@ sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" crate=$work/aws-lc-sys-$version -cargo binstall --no-confirm --locked --root "$work/bindgen" "bindgen-cli@$(cat "$(dirname "$0")/bindgen-version")" -export PATH=$work/bindgen/bin:$PATH - # These options match the CMake build `aws-lc-sys` itself runs, including the crate's symbol # prefix, so the bindings and library match what `aws-lc-rs` is compiled against. cmake -S "$crate/aws-lc" -B "$work/build" \ @@ -51,3 +48,4 @@ cmake --build "$work/build" --parallel "$(nproc)" cmake --install "$work/build" echo "$version" >"$prefix/aws-lc-sys.version" +echo "::notice::Built the $profile AWS-LC prefix for aws-lc-sys $version in ${SECONDS}s" diff --git a/.github/actions/install-aws-lc/cache-key.py b/.github/actions/install-aws-lc/cache-key.py index c9a45fbd37a..b13f383cdff 100644 --- a/.github/actions/install-aws-lc/cache-key.py +++ b/.github/actions/install-aws-lc/cache-key.py @@ -43,6 +43,9 @@ subprocess.run( ["cmake", "--version"], check=True, capture_output=True, text=True ).stdout, + subprocess.run( + ["bindgen", "--version"], check=True, capture_output=True, text=True + ).stdout, *build_environment, ] ) diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index 2a3bbb56914..d3e5d5c3f4b 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -86,7 +86,6 @@ runs: uses: $/.github/actions/install-aws-lc with: profile: ${{ inputs.aws_lc_profile }} - token: ${{ inputs.token }} - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} From 2c0023b1a721a759d4238fff016df2aa4914ba82 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:45:58 +0000 Subject: [PATCH 07/15] Install `bindgen-cli` only when the AWS-LC prefix has to be built `bindgen-cli` moves from the `ci` mise environment to an `aws-lc` environment that `build-prefix.sh` installs on a cache miss, so other jobs no longer install it. The cache key covers the pinned version through the environment's config and lockfile, and a failing compiler version probe names the command. The temporary compile timings workflow is removed. --- .config/mise/config.aws-lc.toml | 3 + .config/mise/config.ci.toml | 1 - .config/mise/mise.aws-lc.lock | 5 ++ .config/mise/mise.ci.lock | 4 -- .github/actions/install-aws-lc/action.yml | 6 +- .../actions/install-aws-lc/build-prefix.sh | 4 ++ .github/actions/install-aws-lc/cache-key.py | 20 +++--- .github/actions/install-tools/action.yml | 1 + .../workflows/hash-graph-compile-timings.yml | 66 ------------------- 9 files changed, 29 insertions(+), 81 deletions(-) create mode 100644 .config/mise/config.aws-lc.toml create mode 100644 .config/mise/mise.aws-lc.lock delete mode 100644 .github/workflows/hash-graph-compile-timings.yml diff --git a/.config/mise/config.aws-lc.toml b/.config/mise/config.aws-lc.toml new file mode 100644 index 00000000000..53a7abd89cc --- /dev/null +++ b/.config/mise/config.aws-lc.toml @@ -0,0 +1,3 @@ +# Tools the `install-aws-lc` action installs when it has to build the AWS-LC prefix. +[tools] +"cargo:bindgen-cli" = "0.72.1" diff --git a/.config/mise/config.ci.toml b/.config/mise/config.ci.toml index 62c7dd13d43..60976042799 100644 --- a/.config/mise/config.ci.toml +++ b/.config/mise/config.ci.toml @@ -1,5 +1,4 @@ [tools] -"cargo:bindgen-cli" = "0.72.1" "cargo:clippy-sarif" = "0.8.0" "cargo:sarif-fmt" = "0.8.0" diff --git a/.config/mise/mise.aws-lc.lock b/.config/mise/mise.aws-lc.lock new file mode 100644 index 00000000000..d511c9143f4 --- /dev/null +++ b/.config/mise/mise.aws-lc.lock @@ -0,0 +1,5 @@ +# @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html + +[[tools."cargo:bindgen-cli"]] +version = "0.72.1" +backend = "cargo:bindgen-cli" diff --git a/.config/mise/mise.ci.lock b/.config/mise/mise.ci.lock index 5919782e5e2..172b11c8f40 100644 --- a/.config/mise/mise.ci.lock +++ b/.config/mise/mise.ci.lock @@ -1,9 +1,5 @@ # @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html -[[tools."cargo:bindgen-cli"]] -version = "0.72.1" -backend = "cargo:bindgen-cli" - [[tools."cargo:clippy-sarif"]] version = "0.8.0" backend = "cargo:clippy-sarif" diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index a4d04909010..1f9b900eb69 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -5,6 +5,9 @@ inputs: profile: description: Cargo profile the job's Rust builds use, `dev` or `release`. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev + token: + description: GitHub token mise uses to install `bindgen-cli` when the prefix has to be built. + required: true runs: using: composite @@ -15,7 +18,7 @@ runs: env: CACHE_KEY: ${{ github.action_path }}/cache-key.py PROFILE: ${{ inputs.profile }} - ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**') }} + ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', '.config/mise/config.aws-lc.toml', '.config/mise/mise.aws-lc.lock') }} run: python3 "$CACHE_KEY" "$PROFILE" - name: Restore AWS-LC @@ -34,6 +37,7 @@ runs: PROFILE: ${{ inputs.profile }} VERSION: ${{ steps.key.outputs.version }} CHECKSUM: ${{ steps.key.outputs.checksum }} + GITHUB_TOKEN: ${{ inputs.token }} run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE" "$VERSION" "$CHECKSUM"' - name: Save AWS-LC diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index 09b2fb319f7..3c6cc76730a 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -29,6 +29,10 @@ sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" crate=$work/aws-lc-sys-$version +export MISE_ENV=${MISE_ENV:+$MISE_ENV,}aws-lc +mise install --locked cargo:bindgen-cli +PATH=$(mise where cargo:bindgen-cli)/bin:$PATH + # These options match the CMake build `aws-lc-sys` itself runs, including the crate's symbol # prefix, so the bindings and library match what `aws-lc-rs` is compiled against. cmake -S "$crate/aws-lc" -B "$work/build" \ diff --git a/.github/actions/install-aws-lc/cache-key.py b/.github/actions/install-aws-lc/cache-key.py index b13f383cdff..f2d153a1aa3 100644 --- a/.github/actions/install-aws-lc/cache-key.py +++ b/.github/actions/install-aws-lc/cache-key.py @@ -32,20 +32,22 @@ for name, value in os.environ.items() if re.fullmatch(r"(CC|CFLAGS|ASMFLAGS|LDFLAGS|CMAKE_[A-Z_]+)", name) ) + + +def version_of(tool): + result = subprocess.run([tool, "--version"], capture_output=True, text=True) + if result.returncode != 0: + sys.exit(f"::error::`{tool} --version` failed: {result.stderr.strip()}") + return result.stdout + + inputs = "\n".join( [ checksum, profile, os.environ["ACTION_FILES"], - subprocess.run( - ["cc", "--version"], check=True, capture_output=True, text=True - ).stdout, - subprocess.run( - ["cmake", "--version"], check=True, capture_output=True, text=True - ).stdout, - subprocess.run( - ["bindgen", "--version"], check=True, capture_output=True, text=True - ).stdout, + version_of("cc"), + version_of("cmake"), *build_environment, ] ) diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index d3e5d5c3f4b..2a3bbb56914 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -86,6 +86,7 @@ runs: uses: $/.github/actions/install-aws-lc with: profile: ${{ inputs.aws_lc_profile }} + token: ${{ inputs.token }} - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} diff --git a/.github/workflows/hash-graph-compile-timings.yml b/.github/workflows/hash-graph-compile-timings.yml deleted file mode 100644 index 4120169ccc9..00000000000 --- a/.github/workflows/hash-graph-compile-timings.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: hash-graph compile timings - -on: - pull_request: - paths: - - .github/workflows/hash-graph-compile-timings.yml - - .github/actions/install-aws-lc/** - -jobs: - sccache-credentials: - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write - outputs: - encrypted-credentials: ${{ steps.mint.outputs.encrypted_credentials }} - steps: - - name: Checkout source code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Mint sccache credentials - id: mint - uses: $/.github/actions/mint-sccache-credentials - with: - vault_address: ${{ vars.VAULT_STAGE_ADDR }} - cf_access_client_id: ${{ vars.CF_ACCESS_STAGE_CLIENT_ID }} - cf_access_client_secret: ${{ secrets.CF_ACCESS_STAGE_CLIENT_SECRET }} - encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - - compile: - name: Compile - needs: [sccache-credentials] - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Clean up disk - uses: $/.github/actions/clean-up-disk - - - name: Install tools - uses: $/.github/actions/install-tools - with: - token: ${{ secrets.GITHUB_TOKEN }} - encrypted_sccache_credentials: ${{ needs.sccache-credentials.outputs.encrypted-credentials }} - sccache_encryption_key: ${{ secrets.SCCACHE_ENC_KEY }} - - - name: Compile hash-graph with timings - run: | - "$RUSTC_WRAPPER" --zero-stats - SECONDS=0 - cargo build --bin hash-graph --all-features --timings - echo "::notice::Full compile took ${SECONDS}s" - stats=$("$RUSTC_WRAPPER" --show-stats) - echo "$stats" - echo "::notice::sccache: $(grep -E '^Cache (hits|misses) ' <<<"$stats" | tr -s ' ' | paste -sd ';')" - python3 - target/cargo-timings/cargo-timing.html <<'EOF' - import json, re, sys - units = json.loads(re.search(r"UNIT_DATA = (\[.*?\]);", open(sys.argv[1]).read(), re.S).group(1)) - for unit in units: - if unit["name"] == "aws-lc-sys": - print(f"::notice::aws-lc-sys {unit.get('target') or 'lib'}: start {unit['start']:.1f}s, {unit['duration']:.1f}s") - EOF - grep -h 'Building with' target/debug/build/aws-lc-sys*/*/output target/debug/build/aws-lc-sys/*/run/stdout 2>/dev/null | sed 's/^cargo:warning=/::notice::aws-lc-sys /' || true From dfecf99cb38ed157cba20f421c899363caeab827 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:59:34 +0000 Subject: [PATCH 08/15] Install `bindgen-cli` with the CI tools and trim the AWS-LC CMake options `bindgen-cli` is a CI mise tool again, and `build-prefix.sh` uses the one on `PATH`, failing with a clear error when it is missing. The cache key covers its pinned version through the CI mise config and lockfile instead of running it. The CMake build keeps only the options this build needs, and `cache-key.py` names its loop variable. --- .config/mise/config.aws-lc.toml | 3 --- .config/mise/config.ci.toml | 1 + .config/mise/mise.aws-lc.lock | 5 ----- .config/mise/mise.ci.lock | 4 ++++ .github/actions/install-aws-lc/action.yml | 6 +----- .github/actions/install-aws-lc/build-prefix.sh | 14 +++++++------- .github/actions/install-aws-lc/cache-key.py | 4 +++- .github/actions/install-tools/action.yml | 1 - 8 files changed, 16 insertions(+), 22 deletions(-) delete mode 100644 .config/mise/config.aws-lc.toml delete mode 100644 .config/mise/mise.aws-lc.lock diff --git a/.config/mise/config.aws-lc.toml b/.config/mise/config.aws-lc.toml deleted file mode 100644 index 53a7abd89cc..00000000000 --- a/.config/mise/config.aws-lc.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Tools the `install-aws-lc` action installs when it has to build the AWS-LC prefix. -[tools] -"cargo:bindgen-cli" = "0.72.1" diff --git a/.config/mise/config.ci.toml b/.config/mise/config.ci.toml index 60976042799..62c7dd13d43 100644 --- a/.config/mise/config.ci.toml +++ b/.config/mise/config.ci.toml @@ -1,4 +1,5 @@ [tools] +"cargo:bindgen-cli" = "0.72.1" "cargo:clippy-sarif" = "0.8.0" "cargo:sarif-fmt" = "0.8.0" diff --git a/.config/mise/mise.aws-lc.lock b/.config/mise/mise.aws-lc.lock deleted file mode 100644 index d511c9143f4..00000000000 --- a/.config/mise/mise.aws-lc.lock +++ /dev/null @@ -1,5 +0,0 @@ -# @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html - -[[tools."cargo:bindgen-cli"]] -version = "0.72.1" -backend = "cargo:bindgen-cli" diff --git a/.config/mise/mise.ci.lock b/.config/mise/mise.ci.lock index 172b11c8f40..5919782e5e2 100644 --- a/.config/mise/mise.ci.lock +++ b/.config/mise/mise.ci.lock @@ -1,5 +1,9 @@ # @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html +[[tools."cargo:bindgen-cli"]] +version = "0.72.1" +backend = "cargo:bindgen-cli" + [[tools."cargo:clippy-sarif"]] version = "0.8.0" backend = "cargo:clippy-sarif" diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index 1f9b900eb69..b00486b8d96 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -5,9 +5,6 @@ inputs: profile: description: Cargo profile the job's Rust builds use, `dev` or `release`. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev - token: - description: GitHub token mise uses to install `bindgen-cli` when the prefix has to be built. - required: true runs: using: composite @@ -18,7 +15,7 @@ runs: env: CACHE_KEY: ${{ github.action_path }}/cache-key.py PROFILE: ${{ inputs.profile }} - ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', '.config/mise/config.aws-lc.toml', '.config/mise/mise.aws-lc.lock') }} + ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', '.config/mise/config.ci.toml', '.config/mise/mise.ci.lock') }} run: python3 "$CACHE_KEY" "$PROFILE" - name: Restore AWS-LC @@ -37,7 +34,6 @@ runs: PROFILE: ${{ inputs.profile }} VERSION: ${{ steps.key.outputs.version }} CHECKSUM: ${{ steps.key.outputs.checksum }} - GITHUB_TOKEN: ${{ inputs.token }} run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE" "$VERSION" "$CHECKSUM"' - name: Save AWS-LC diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index 3c6cc76730a..984698e5b7c 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -29,22 +29,22 @@ sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" crate=$work/aws-lc-sys-$version -export MISE_ENV=${MISE_ENV:+$MISE_ENV,}aws-lc -mise install --locked cargo:bindgen-cli -PATH=$(mise where cargo:bindgen-cli)/bin:$PATH +if ! bindgen_version=$(bindgen --version 2>&1); then + echo "::error::\`bindgen --version\` failed, \`mise install --env ci\` should provide \`bindgen-cli\`: $bindgen_version" + exit 1 +fi +echo "Using $bindgen_version from $(command -v bindgen)" -# These options match the CMake build `aws-lc-sys` itself runs, including the crate's symbol -# prefix, so the bindings and library match what `aws-lc-rs` is compiled against. +# The crate ships AWS-LC with pregenerated assembly and its symbol-prefix headers, so the build +# needs neither Go nor Perl, and keeps the prefix the crate's own bindings use. cmake -S "$crate/aws-lc" -B "$work/build" \ -DCMAKE_BUILD_TYPE="$build_type" \ -DCMAKE_INSTALL_PREFIX="$prefix" \ - -DBUILD_SHARED_LIBS=OFF \ -DBUILD_LIBSSL=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_TOOL=OFF \ -DDISABLE_GO=ON \ -DDISABLE_PERL=ON \ - -DENABLE_SOURCE_MODIFICATION=OFF \ -DBORINGSSL_PREFIX="aws_lc_${version//./_}" \ -DBORINGSSL_PREFIX_HEADERS="$crate/generated-include" \ -DGENERATE_RUST_BINDINGS=ON diff --git a/.github/actions/install-aws-lc/cache-key.py b/.github/actions/install-aws-lc/cache-key.py index f2d153a1aa3..45a85befc98 100644 --- a/.github/actions/install-aws-lc/cache-key.py +++ b/.github/actions/install-aws-lc/cache-key.py @@ -18,7 +18,9 @@ with open("Cargo.lock", "rb") as lockfile: packages = [ - p for p in tomllib.load(lockfile)["package"] if p["name"] == "aws-lc-sys" + package + for package in tomllib.load(lockfile)["package"] + if package["name"] == "aws-lc-sys" ] if len(packages) != 1: sys.exit( diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index 2a3bbb56914..d3e5d5c3f4b 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -86,7 +86,6 @@ runs: uses: $/.github/actions/install-aws-lc with: profile: ${{ inputs.aws_lc_profile }} - token: ${{ inputs.token }} - name: "Install sccache" if: ${{ (inputs.rust == true || inputs.rust == 'true') && (inputs.sccache == true || inputs.sccache == 'true') && inputs.encrypted_sccache_credentials != '' }} From e295b368ccb4c5917c6ea1c736bc17e4d3823ba5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:10:06 +0000 Subject: [PATCH 09/15] Add a temporary workflow reporting the install-tools state --- .github/workflows/debug-install-tools.yml | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/debug-install-tools.yml diff --git a/.github/workflows/debug-install-tools.yml b/.github/workflows/debug-install-tools.yml new file mode 100644 index 00000000000..491b4af588f --- /dev/null +++ b/.github/workflows/debug-install-tools.yml @@ -0,0 +1,44 @@ +name: Debug install-tools + +on: + pull_request: + paths: + - .github/workflows/debug-install-tools.yml + +jobs: + debug: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install tools + id: install-tools + continue-on-error: true + uses: $/.github/actions/install-tools + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Report + if: always() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + report() { + local name=$1 + shift + local output + output=$("$@" 2>&1 | tail -n 25 | tr '\n' '|' | cut -c 1-3000) || true + echo "::warning title=${name}::${output}" + } + report "mise install" mise install --env ci --jobs 1 --locked + report "which bindgen" which bindgen + report "bindgen --version" bindgen --version + report "mise ls" mise ls --env ci --current + report "mise which protoc" mise which protoc + report "python3" python3 --version + report "cache key" env ACTION_FILES=x PROFILE=dev python3 .github/actions/install-aws-lc/cache-key.py dev + report "env" bash -c 'env | grep -E "^(AWS_LC|MISE|RUSTC_WRAPPER)" | sort' + report "prefix" bash -c 'ls -la "$RUNNER_TEMP"/aws-lc-* "$RUNNER_TEMP"/aws-lc-*/lib 2>&1' From 5bb45048c9b1ddccd5691357d901ee7a54759ac9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:12:01 +0000 Subject: [PATCH 10/15] Report the AWS-LC prefix build output from the temporary debug workflow --- .github/workflows/debug-install-tools.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/debug-install-tools.yml b/.github/workflows/debug-install-tools.yml index 491b4af588f..b3af18bc69d 100644 --- a/.github/workflows/debug-install-tools.yml +++ b/.github/workflows/debug-install-tools.yml @@ -33,12 +33,7 @@ jobs: output=$("$@" 2>&1 | tail -n 25 | tr '\n' '|' | cut -c 1-3000) || true echo "::warning title=${name}::${output}" } - report "mise install" mise install --env ci --jobs 1 --locked - report "which bindgen" which bindgen - report "bindgen --version" bindgen --version - report "mise ls" mise ls --env ci --current - report "mise which protoc" mise which protoc - report "python3" python3 --version - report "cache key" env ACTION_FILES=x PROFILE=dev python3 .github/actions/install-aws-lc/cache-key.py dev - report "env" bash -c 'env | grep -E "^(AWS_LC|MISE|RUSTC_WRAPPER)" | sort' - report "prefix" bash -c 'ls -la "$RUNNER_TEMP"/aws-lc-* "$RUNNER_TEMP"/aws-lc-*/lib 2>&1' + report "bindgen header" bash -c "echo '#include ' > /tmp/t.h && bindgen /tmp/t.h | head -n 5" + report "build prefix" .github/actions/install-aws-lc/build-prefix.sh /tmp/dbg-prefix dev 0.45.0 9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27 + report "built prefix" ls -R /tmp/dbg-prefix + report "libclang" bash -c "ls /usr/lib/llvm-*/lib/libclang*.so* /usr/lib/x86_64-linux-gnu/libclang* 2>&1" From dcfac60e0bc993fa5ffbc4ed8ab2f0f0153ef087 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:14:29 +0000 Subject: [PATCH 11/15] Pass the resolved `bindgen` binary to the AWS-LC CMake build CMake runs bindgen from its build directory outside the repository, where the mise shim exits with "No version is set for shim: bindgen", so the prefix build failed and every job missed the cache. `build-prefix.sh` now hands CMake the binary `mise which bindgen` resolves. The temporary debug workflow is removed. --- .github/actions/install-aws-lc/action.yml | 2 +- .../actions/install-aws-lc/build-prefix.sh | 9 +++-- .github/workflows/debug-install-tools.yml | 39 ------------------- 3 files changed, 6 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/debug-install-tools.yml diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index b00486b8d96..97e92ae9c35 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -49,7 +49,7 @@ runs: PREFIX: ${{ steps.key.outputs.prefix }} VERSION: ${{ steps.key.outputs.version }} run: | - built=$(cat "$PREFIX/aws-lc-sys.version") + built=$(cat "$PREFIX/aws-lc-sys.version" 2>/dev/null || echo none) if [[ $built != "$VERSION" ]]; then echo "::error::the AWS-LC prefix was built for aws-lc-sys $built, but Cargo.lock has $VERSION" exit 1 diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index 984698e5b7c..08e240a3353 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -29,11 +29,11 @@ sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" crate=$work/aws-lc-sys-$version -if ! bindgen_version=$(bindgen --version 2>&1); then - echo "::error::\`bindgen --version\` failed, \`mise install --env ci\` should provide \`bindgen-cli\`: $bindgen_version" +# CMake runs bindgen outside the repository, where the mise shim cannot resolve a version. +if ! bindgen=$(mise which bindgen); then + echo "::error::\`mise which bindgen\` failed, \`mise install --env ci\` should provide \`bindgen-cli\`" exit 1 fi -echo "Using $bindgen_version from $(command -v bindgen)" # The crate ships AWS-LC with pregenerated assembly and its symbol-prefix headers, so the build # needs neither Go nor Perl, and keeps the prefix the crate's own bindings use. @@ -47,7 +47,8 @@ cmake -S "$crate/aws-lc" -B "$work/build" \ -DDISABLE_PERL=ON \ -DBORINGSSL_PREFIX="aws_lc_${version//./_}" \ -DBORINGSSL_PREFIX_HEADERS="$crate/generated-include" \ - -DGENERATE_RUST_BINDINGS=ON + -DGENERATE_RUST_BINDINGS=ON \ + -DBINDGEN_EXECUTABLE="$bindgen" cmake --build "$work/build" --parallel "$(nproc)" cmake --install "$work/build" diff --git a/.github/workflows/debug-install-tools.yml b/.github/workflows/debug-install-tools.yml deleted file mode 100644 index b3af18bc69d..00000000000 --- a/.github/workflows/debug-install-tools.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Debug install-tools - -on: - pull_request: - paths: - - .github/workflows/debug-install-tools.yml - -jobs: - debug: - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install tools - id: install-tools - continue-on-error: true - uses: $/.github/actions/install-tools - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Report - if: always() - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - report() { - local name=$1 - shift - local output - output=$("$@" 2>&1 | tail -n 25 | tr '\n' '|' | cut -c 1-3000) || true - echo "::warning title=${name}::${output}" - } - report "bindgen header" bash -c "echo '#include ' > /tmp/t.h && bindgen /tmp/t.h | head -n 5" - report "build prefix" .github/actions/install-aws-lc/build-prefix.sh /tmp/dbg-prefix dev 0.45.0 9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27 - report "built prefix" ls -R /tmp/dbg-prefix - report "libclang" bash -c "ls /usr/lib/llvm-*/lib/libclang*.so* /usr/lib/x86_64-linux-gnu/libclang* 2>&1" From d84296d4e90e6aece7496b8b9e51986bddca4e54 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:41:02 +0000 Subject: [PATCH 12/15] Skip the prebuilt AWS-LC on checkouts that lack its action The integration bench checks out its base branch and installs tools there, but a base revision without `install-aws-lc` has no `bindgen-cli` in its CI mise tools, so the prefix build failed. Such a checkout now compiles AWS-LC from source, as that revision does on its own. --- .github/actions/install-tools/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index d3e5d5c3f4b..c1650218509 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -81,8 +81,10 @@ runs: retry_wait_seconds: 60 command: ${{ github.action_path }}/install-rust.sh + # The action builds AWS-LC with tools from the checked-out revision's mise config, which a + # revision without the action, such as the base branch a bench compares against, lacks. - name: "Install AWS-LC" - if: ${{ inputs.rust == true || inputs.rust == 'true' }} + if: ${{ (inputs.rust == true || inputs.rust == 'true') && hashFiles('.github/actions/install-aws-lc/action.yml') != '' }} uses: $/.github/actions/install-aws-lc with: profile: ${{ inputs.aws_lc_profile }} From 413f3783609106bc7bd0f05293870d75ec0b09bf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 00:05:57 +0000 Subject: [PATCH 13/15] Key the AWS-LC cache on `cargo pkgid` and build from cargo's crate source The cache key is the `aws-lc-sys` version from `cargo pkgid`, the runner, the profile and the action's files, so `cache-key.py` and its Cargo.lock parsing go. On a miss, `build-prefix.sh` builds the crate source cargo fetched and checksummed for `cargo metadata`, rather than downloading the crate itself, and keeps CMake from modifying that copy. The prefix no longer records its version: the key already pins it. --- .github/actions/install-aws-lc/action.yml | 18 ++++------ .../actions/install-aws-lc/build-prefix.sh | 33 +++++++++---------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index 97e92ae9c35..4a04721478f 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -13,10 +13,14 @@ runs: id: key shell: bash env: - CACHE_KEY: ${{ github.action_path }}/cache-key.py PROFILE: ${{ inputs.profile }} ACTION_FILES: ${{ hashFiles('.github/actions/install-aws-lc/**', '.config/mise/config.ci.toml', '.config/mise/mise.ci.lock') }} - run: python3 "$CACHE_KEY" "$PROFILE" + run: | + package=$(cargo pkgid --locked aws-lc-sys) + { + echo "key=aws-lc-sys-${package##*@}-$RUNNER_OS-$RUNNER_ARCH-$PROFILE-$ACTION_FILES" + echo "prefix=$RUNNER_TEMP/aws-lc-$PROFILE" + } >>"$GITHUB_OUTPUT" - name: Restore AWS-LC id: restore @@ -32,9 +36,7 @@ runs: BUILD_PREFIX: ${{ github.action_path }}/build-prefix.sh PREFIX: ${{ steps.key.outputs.prefix }} PROFILE: ${{ inputs.profile }} - VERSION: ${{ steps.key.outputs.version }} - CHECKSUM: ${{ steps.key.outputs.checksum }} - run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE" "$VERSION" "$CHECKSUM"' + run: '"$BUILD_PREFIX" "$PREFIX" "$PROFILE"' - name: Save AWS-LC if: steps.restore.outputs.cache-hit != 'true' @@ -47,13 +49,7 @@ runs: shell: bash env: PREFIX: ${{ steps.key.outputs.prefix }} - VERSION: ${{ steps.key.outputs.version }} run: | - built=$(cat "$PREFIX/aws-lc-sys.version" 2>/dev/null || echo none) - if [[ $built != "$VERSION" ]]; then - echo "::error::the AWS-LC prefix was built for aws-lc-sys $built, but Cargo.lock has $VERSION" - exit 1 - fi target=$(rustc -vV | sed -n 's/^host: //p' | tr '-' '_') { echo "AWS_LC_SYS_SYSTEM_DIR_${target}=$PREFIX" diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index 08e240a3353..2edfcad3f18 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash -# Usage: build-prefix.sh +# Usage: build-prefix.sh # -# Builds the AWS-LC source bundled in the `aws-lc-sys` crate with AWS-LC's own CMake build, and -# installs the static library, headers and generated Rust bindings into , which +# Builds the AWS-LC source bundled in the locked `aws-lc-sys` crate with AWS-LC's own CMake build, +# and installs the static library, headers and generated Rust bindings into , which # `aws-lc-sys` links when `AWS_LC_SYS_SYSTEM_DIR` points at it. set -euo pipefail prefix=$1 profile=$2 -version=$3 -checksum=$4 case $profile in dev) build_type=Debug ;; @@ -20,14 +18,9 @@ case $profile in ;; esac -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT - -curl --fail --silent --show-error --location --retry 3 \ - --output "$work/aws-lc-sys.crate" "https://static.crates.io/crates/aws-lc-sys/aws-lc-sys-$version.crate" -sha256sum --check --quiet <<<"$checksum $work/aws-lc-sys.crate" -tar --extract --gzip --file "$work/aws-lc-sys.crate" --directory "$work" -crate=$work/aws-lc-sys-$version +crate=$(dirname "$(cargo metadata --locked --format-version 1 | jq --raw-output '.packages[] | select(.name == "aws-lc-sys") | .manifest_path')") +version=$(cargo pkgid --locked aws-lc-sys) +version=${version##*@} # CMake runs bindgen outside the repository, where the mise shim cannot resolve a version. if ! bindgen=$(mise which bindgen); then @@ -35,9 +28,13 @@ if ! bindgen=$(mise which bindgen); then exit 1 fi +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + # The crate ships AWS-LC with pregenerated assembly and its symbol-prefix headers, so the build -# needs neither Go nor Perl, and keeps the prefix the crate's own bindings use. -cmake -S "$crate/aws-lc" -B "$work/build" \ +# needs neither Go nor Perl, and keeps the prefix the crate's own bindings use. The source is +# cargo's registry copy, which the build must leave unmodified. +cmake -S "$crate/aws-lc" -B "$work" \ -DCMAKE_BUILD_TYPE="$build_type" \ -DCMAKE_INSTALL_PREFIX="$prefix" \ -DBUILD_LIBSSL=OFF \ @@ -45,12 +42,12 @@ cmake -S "$crate/aws-lc" -B "$work/build" \ -DBUILD_TOOL=OFF \ -DDISABLE_GO=ON \ -DDISABLE_PERL=ON \ + -DENABLE_SOURCE_MODIFICATION=OFF \ -DBORINGSSL_PREFIX="aws_lc_${version//./_}" \ -DBORINGSSL_PREFIX_HEADERS="$crate/generated-include" \ -DGENERATE_RUST_BINDINGS=ON \ -DBINDGEN_EXECUTABLE="$bindgen" -cmake --build "$work/build" --parallel "$(nproc)" -cmake --install "$work/build" +cmake --build "$work" --parallel "$(nproc)" +cmake --install "$work" -echo "$version" >"$prefix/aws-lc-sys.version" echo "::notice::Built the $profile AWS-LC prefix for aws-lc-sys $version in ${SECONDS}s" From 200a7b0b1b9cc06607b9c0a4ae8ce464e3f9004b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 00:06:09 +0000 Subject: [PATCH 14/15] Remove the unused AWS-LC cache key script --- .github/actions/install-aws-lc/cache-key.py | 66 --------------------- 1 file changed, 66 deletions(-) delete mode 100644 .github/actions/install-aws-lc/cache-key.py diff --git a/.github/actions/install-aws-lc/cache-key.py b/.github/actions/install-aws-lc/cache-key.py deleted file mode 100644 index 45a85befc98..00000000000 --- a/.github/actions/install-aws-lc/cache-key.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Usage: cache-key.py - -Writes the locked `aws-lc-sys` version and checksum, the cache key of its AWS-LC prefix and the -prefix path as GitHub Actions step outputs. -""" - -import hashlib -import os -import re -import subprocess -import sys -import tomllib - -profile = sys.argv[1] -if profile not in ("dev", "release"): - sys.exit(f"::error::unknown profile `{profile}`, expected `dev` or `release`") - -with open("Cargo.lock", "rb") as lockfile: - packages = [ - package - for package in tomllib.load(lockfile)["package"] - if package["name"] == "aws-lc-sys" - ] -if len(packages) != 1: - sys.exit( - f"::error::expected one aws-lc-sys package in Cargo.lock, found {len(packages)}" - ) -version = packages[0]["version"] -checksum = packages[0]["checksum"] - -build_environment = sorted( - f"{name}={value}" - for name, value in os.environ.items() - if re.fullmatch(r"(CC|CFLAGS|ASMFLAGS|LDFLAGS|CMAKE_[A-Z_]+)", name) -) - - -def version_of(tool): - result = subprocess.run([tool, "--version"], capture_output=True, text=True) - if result.returncode != 0: - sys.exit(f"::error::`{tool} --version` failed: {result.stderr.strip()}") - return result.stdout - - -inputs = "\n".join( - [ - checksum, - profile, - os.environ["ACTION_FILES"], - version_of("cc"), - version_of("cmake"), - *build_environment, - ] -) -print(inputs) -digest = hashlib.sha256(inputs.encode()).hexdigest()[:16] - -with open(os.environ["GITHUB_OUTPUT"], "a") as output: - print(f"version={version}", file=output) - print(f"checksum={checksum}", file=output) - print( - f"key=aws-lc-sys-{version}-{os.environ['RUNNER_OS']}-{os.environ['RUNNER_ARCH']}-{profile}-{digest}", - file=output, - ) - print(f"prefix={os.environ['RUNNER_TEMP']}/aws-lc-{profile}", file=output) From 14a049002f26f326a66b230c6ca3c41d85f22193 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 08:38:42 +0000 Subject: [PATCH 15/15] Key the AWS-LC cache on the runner image and harden the prefix build The key uses `ImageOS` instead of `RUNNER_OS`, so jobs on `ubuntu-latest` stop sharing an install with `ubuntu-24.04` jobs once the label moves to a newer image. `build-prefix.sh` fails with its own error when `cargo metadata` fails, removes a previous install before building, and names the image in its notice. The `aws_lc_profile` description reuses the `profile` wording. --- .github/actions/install-aws-lc/action.yml | 2 +- .github/actions/install-aws-lc/build-prefix.sh | 13 +++++++++++-- .github/actions/install-tools/action.yml | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/actions/install-aws-lc/action.yml b/.github/actions/install-aws-lc/action.yml index 4a04721478f..470dc423d2c 100644 --- a/.github/actions/install-aws-lc/action.yml +++ b/.github/actions/install-aws-lc/action.yml @@ -18,7 +18,7 @@ runs: run: | package=$(cargo pkgid --locked aws-lc-sys) { - echo "key=aws-lc-sys-${package##*@}-$RUNNER_OS-$RUNNER_ARCH-$PROFILE-$ACTION_FILES" + echo "key=aws-lc-sys-${package##*@}-${ImageOS:?the runner image sets no ImageOS}-$RUNNER_ARCH-$PROFILE-$ACTION_FILES" echo "prefix=$RUNNER_TEMP/aws-lc-$PROFILE" } >>"$GITHUB_OUTPUT" diff --git a/.github/actions/install-aws-lc/build-prefix.sh b/.github/actions/install-aws-lc/build-prefix.sh index 2edfcad3f18..507191868ed 100755 --- a/.github/actions/install-aws-lc/build-prefix.sh +++ b/.github/actions/install-aws-lc/build-prefix.sh @@ -18,7 +18,15 @@ case $profile in ;; esac -crate=$(dirname "$(cargo metadata --locked --format-version 1 | jq --raw-output '.packages[] | select(.name == "aws-lc-sys") | .manifest_path')") +if ! metadata=$(cargo metadata --locked --format-version 1); then + echo "::error::\`cargo metadata --locked\` failed" + exit 1 +fi +if ! manifest=$(jq --raw-output --exit-status '.packages[] | select(.name == "aws-lc-sys") | .manifest_path' <<<"$metadata"); then + echo "::error::\`cargo metadata\` lists no aws-lc-sys package" + exit 1 +fi +crate=$(dirname "$manifest") version=$(cargo pkgid --locked aws-lc-sys) version=${version##*@} @@ -30,6 +38,7 @@ fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT +rm -rf "$prefix" # The crate ships AWS-LC with pregenerated assembly and its symbol-prefix headers, so the build # needs neither Go nor Perl, and keeps the prefix the crate's own bindings use. The source is @@ -50,4 +59,4 @@ cmake -S "$crate/aws-lc" -B "$work" \ cmake --build "$work" --parallel "$(nproc)" cmake --install "$work" -echo "::notice::Built the $profile AWS-LC prefix for aws-lc-sys $version in ${SECONDS}s" +echo "::notice::Built the $profile AWS-LC prefix for aws-lc-sys $version on ${ImageOS:-an unknown image} in ${SECONDS}s" diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index c1650218509..b479ea25372 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -18,7 +18,7 @@ inputs: description: Should sccache be installed? Can either be `"true"` or `true` default: "true" aws_lc_profile: - description: Cargo profile, `dev` or `release`, that the prebuilt AWS-LC `aws-lc-sys` links is compiled with. Set it to the profile the job's Rust builds use. + description: Cargo profile the job's Rust builds use, `dev` or `release`, which the prebuilt AWS-LC is compiled with. Profiles inheriting from one of them, such as `test` or `bench`, use its value. default: dev runs: