From ae412dfeb09e082b8ffb2ab9bbe5ce834c8dcda7 Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 13:44:03 +0100 Subject: [PATCH 1/7] ADFA-2602: Generate asset zips per ABI from any ref generate_assets.yml hardcoded `ref: stage` and could only source debug assets from the live site, so it could not build the toolchain-upgrade branch at all: the 9.6.1 assets are unpublished, so assetsDownloadDebug 404s against appdevforall.org. - `ref` input replaces the hardcoded stage checkout. - `asset_source` input: `site` keeps today's behaviour, `release` pulls the ingredients from a draft release using the built-in token, which needs no new secret. - Split into `prepare` plus a v7/v8 `zip` matrix so the two zips build in parallel. Staging stays in one job deliberately: two concurrent legs would both write ~/.ssh/id_rsa on the same self-hosted runner, and one leg's cleanup would pull the key out from under the other. - Gates: every ingredient present and non-empty, documentation.db carries Content.templateId, and the built zip contains all seven expected entries. Wrong-variant staging passes size and checksum checks, so it needs a content gate. - assets/*.zip, documentation.db and core.cgt are removed before staging. createPluginArtifactsZip and createPluginMavenRepoZip write into the source tree, so their output survives between runs on a persistent workspace. - Cloudflare R2 replaces the fixed Drive file IDs. cloudflare-r2-upload.py gains R2_BUCKET and R2_KEY_PREFIX overrides that default to today's values, so release.yml and weekly-release.yml are unaffected. - Fix a guard that tested "DB_FILE_ID" instead of "$DB_FILE_ID" and so never fired on a missing secret. - Drop the zip job's heap to 6g now that two legs can share one runner. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 279 ++++++++++++++++++-------- scripts/cloudflare-r2-upload.py | 9 +- 2 files changed, 204 insertions(+), 84 deletions(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index 4741fa56f4..bcb4a79392 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -1,4 +1,4 @@ -name: Generate Assets Zips and Upload to Google Drive +name: Generate Assets Zips permissions: id-token: write @@ -7,13 +7,30 @@ permissions: on: workflow_dispatch: + inputs: + ref: + description: Branch or tag to build the asset zips from + required: false + default: stage + asset_source: + description: Where the debug asset ingredients come from + required: false + default: site + type: choice + options: + - site + - release + asset_release_tag: + description: 'Release tag holding the debug ingredients (asset_source: release)' + required: false + default: adfa-2602-rc env: SCP_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} jobs: - generate_assets: - name: Generate Assets Zips + prepare: + name: Stage debug assets runs-on: self-hosted timeout-minutes: 90 @@ -21,7 +38,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: stage + ref: ${{ inputs.ref }} + + - name: Remove stale staged assets + run: | + rm -f assets/*.zip assets/documentation.db assets/core.cgt + ls -la assets/ 2>/dev/null || true - name: Check if Nix is installed id: check_nix @@ -55,6 +77,7 @@ jobs: access_token_scopes: 'https://www.googleapis.com/auth/drive' - name: Set up SSH key + if: inputs.asset_source == 'site' env: GREENGEEKS_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} GREENGEEKS_KEY: ${{ secrets.GREENGEEKS_SSH_PRIVATE_KEY }} @@ -109,7 +132,8 @@ jobs: rm -f ~/.ssh/id_ed25519 ~/.ssh/id_ecdsa ~/.ssh/id_dsa ~/.ssh/id_rsa.pub 2>/dev/null ssh-keyscan -H "$GREENGEEKS_HOST" >> ~/.ssh/known_hosts 2>/dev/null - - name: Download debug assets + - name: Download debug assets from the site + if: inputs.asset_source == 'site' run: | flox activate -d flox/base -- ./gradlew :app:assetsDownloadDebug --no-daemon \ -Dorg.gradle.jvmargs="-Xmx10g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ @@ -117,12 +141,28 @@ jobs: -Dorg.gradle.workers.max=1 \ -Dorg.gradle.parallel=false + - name: Download debug assets from a release + if: inputs.asset_source == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET_TAG: ${{ inputs.asset_release_tag }} + run: | + mkdir -p assets + gh release download "$ASSET_TAG" --dir assets --clobber \ + --pattern 'android-sdk-*.zip' \ + --pattern 'bootstrap-*.zip' \ + --pattern 'gradle-*-bin.zip' \ + --pattern 'gradle-api-*.jar.zip' \ + --pattern 'localMvnRepository.zip' \ + --pattern 'core.cgt' + ls -la assets/ + - name: Download latest documentation.db from Google Drive run: | DB_FILE_ID="${{ secrets.DOCUMENTATION_DB_FILE_ID }}" ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" - if [ -z "DB_FILE_ID" ]; then + if [ -z "$DB_FILE_ID" ]; then echo "ERROR: DOCUMENTATION_DB_FILE_ID secret not set" echo "Please set the DOCUMENTATION_DB_FILE_ID secret in repository settings" exit 1 @@ -151,84 +191,170 @@ jobs: echo "Successfully downloaded documentation.db ($FILE_SIZE_HUMAN)" - - name: Assemble Assets + - name: Verify staged ingredients run: | - flox activate -d flox/base -- ./gradlew :app:assembleAssets --no-daemon \ - -Dorg.gradle.jvmargs="-Xmx10g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ - -Dandroid.aapt2.daemonHeapSize=4096M \ - -Dorg.gradle.workers.max=1 \ - -Dorg.gradle.parallel=false + missing=0 + for f in localMvnRepository.zip documentation.db core.cgt \ + android-sdk-arm64-v8a.zip android-sdk-armeabi-v7a.zip \ + bootstrap-arm64-v8a.zip bootstrap-armeabi-v7a.zip; do + if [ ! -s "assets/$f" ]; then + echo "ERROR: missing or empty assets/$f" + missing=1 + fi + done + if ! ls assets/gradle-*-bin.zip >/dev/null 2>&1; then + echo "ERROR: no assets/gradle-*-bin.zip staged" + missing=1 + fi + if ! ls assets/gradle-api-*.jar.zip >/dev/null 2>&1; then + echo "ERROR: no assets/gradle-api-*.jar.zip staged" + missing=1 + fi + [ "$missing" -eq 0 ] || exit 1 + command -v sqlite3 >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq sqlite3; } + sqlite3 assets/documentation.db \ + "SELECT 1 FROM pragma_table_info('Content') WHERE name='templateId';" | grep -q 1 \ + || { echo "ERROR: documentation.db predates the templateId column; docs requests will 500"; exit 1; } + echo "All ingredients staged" + + - name: Upload staged assets + uses: actions/upload-artifact@v4 + with: + name: debug-assets + path: assets/ + retention-days: 1 + if-no-files-found: error - - name: V8 Assets Path - id: assets_v8 + - name: Cleanup google-services.json + if: always() run: | - assets_path="app/build/outputs/assets/assets-arm64-v8a.zip" - echo "ASSETS_PATH=$assets_path" >> $GITHUB_OUTPUT + rm -f app/google-services.json + echo "google-services.json cleaned up successfully" - - name: V7 Assets Path - id: assets_v7 + - name: Cleanup ssh + if: always() && inputs.asset_source == 'site' run: | - assets_path="app/build/outputs/assets/assets-armeabi-v7a.zip" - echo "ASSETS_PATH=$assets_path" >> $GITHUB_OUTPUT + # Remove SSH key + rm -f ~/.ssh/id_rsa + # Clean up SSH known_hosts (remove the entry for this host) + if [ -n "$SCP_HOST" ]; then + ssh-keygen -R "$SCP_HOST" 2>/dev/null || true + fi + # Remove entire .ssh directory if empty + rmdir ~/.ssh 2>/dev/null || true - - name: Upload asset zips to Google Drive + zip: + name: Zip assets + runs-on: self-hosted + timeout-minutes: 60 + needs: prepare + strategy: + fail-fast: false + matrix: + include: + - abi: v8 + arch: arm64-v8a + label: 64-bit ARM + - abi: v7 + arch: armeabi-v7a + label: 32-bit ARM + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Remove stale staged assets run: | - echo "Uploading assets v8 and v7 to Google Drive..." + rm -f assets/*.zip assets/documentation.db assets/core.cgt - ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" - V8_FILE_ID="${{ secrets.ASSETS_V8_FILE_ID }}" - V7_FILE_ID="${{ secrets.ASSETS_V7_FILE_ID }}" - - V8_PATH="${{ steps.assets_v8.outputs.ASSETS_PATH }}" - V7_PATH="${{ steps.assets_v7.outputs.ASSETS_PATH }}" - - # Upload v8 - response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ - -H "Authorization: Bearer $ACCESS_TOKEN" \ - -H "Content-Type: application/zip" \ - --upload-file "${V8_PATH}" \ - "https://www.googleapis.com/upload/drive/v3/files/${V8_FILE_ID}?uploadType=media") - - if [[ "$response" -ne 200 ]]; then - echo "Upload of ${V8_PATH} failed with HTTP status $response" - exit 1 + - name: Check if Nix is installed + id: check_nix + run: | + if command -v nix >/dev/null 2>&1; then + echo "nix is installed" + echo "nix_installed=true" >> $GITHUB_ENV + else + echo "nix is not installed" + echo "nix_installed=false" >> $GITHUB_ENV fi - # Upload v7 - response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ - -H "Authorization: Bearer $ACCESS_TOKEN" \ - -H "Content-Type: application/zip" \ - --upload-file "${V7_PATH}" \ - "https://www.googleapis.com/upload/drive/v3/files/${V7_FILE_ID}?uploadType=media") + - name: Install Flox + if: env.nix_installed == 'false' + uses: flox/install-flox-action@v2 + + - name: Create google-services.json + env: + GOOGLE_SERVICES_JSON: ${{ secrets.GOOGLE_SERVICES_JSON }} + run: | + echo "$GOOGLE_SERVICES_JSON" > app/google-services.json - if [[ "$response" -ne 200 ]]; then - echo "Upload of ${V7_PATH} failed with HTTP status $response" + - name: Download staged assets + uses: actions/download-artifact@v4 + with: + name: debug-assets + path: assets + + - name: Assemble assets zip + run: | + variant_upper=$(echo "${{ matrix.abi }}" | tr '[:lower:]' '[:upper:]') + flox activate -d flox/base -- ./gradlew :app:assemble${variant_upper}Assets --no-daemon \ + -Dorg.gradle.jvmargs="-Xmx6g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ + -Dandroid.aapt2.daemonHeapSize=4096M \ + -Dorg.gradle.workers.max=1 \ + -Dorg.gradle.parallel=false + + - name: Verify assets zip + id: assets_zip + run: | + zip_path="app/build/outputs/assets/assets-${{ matrix.arch }}.zip" + if [ ! -s "$zip_path" ]; then + echo "ERROR: $zip_path was not produced" exit 1 fi + unzip -tq "$zip_path" + for entry in android-sdk.zip bootstrap.zip localMvnRepository.zip documentation.db \ + core.cgt plugin-artifacts.zip plugin-maven-repo.zip; do + unzip -l "$zip_path" | grep -qE "[[:space:]]$entry$" \ + || { echo "ERROR: $zip_path is missing entry $entry"; exit 1; } + done + echo "ASSETS_PATH=$zip_path" >> $GITHUB_OUTPUT + echo "$(du -h "$zip_path" | cut -f1) $zip_path" - echo "Upload complete." + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Upload assets zip to Cloudflare R2 + id: r2 + env: + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_KEY_ID: ${{ vars.CLOUDFLARE_KEY_ID }} + CLOUDFLARE_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.R2_ASSETS_BUCKET || 'apk-repo' }} + R2_KEY_PREFIX: assets/ + run: | + uv run --with boto3 scripts/cloudflare-r2-upload.py "${{ steps.assets_zip.outputs.ASSETS_PATH }}" + echo "DOWNLOAD_URL=https://download.appdevforall.org/assets/assets-${{ matrix.arch }}.zip" >> $GITHUB_OUTPUT - - name: Send Rich Slack Notification + - name: Send Slack notification env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + ASSET_LABEL: ${{ matrix.label }} + DOWNLOAD_URL: ${{ steps.r2.outputs.DOWNLOAD_URL }} + BUILD_REF: ${{ inputs.ref }} run: | - - V8_FILE_ID="${{ secrets.ASSETS_V8_FILE_ID }}" - V7_FILE_ID="${{ secrets.ASSETS_V7_FILE_ID }}" - - GDRIVE_V8_LINK="" - GDRIVE_V7_LINK="" - jq -n \ - --arg v8_link "$GDRIVE_V8_LINK" \ - --arg v7_link "$GDRIVE_V7_LINK" \ + --arg label "$ASSET_LABEL" \ + --arg url "$DOWNLOAD_URL" \ + --arg ref "$BUILD_REF" \ '{ blocks: [ { type: "header", text: { type: "plain_text", - text: ":rocket: [Updated] New Assets Zips Available", + text: ":rocket: [Updated] New Assets Zip Available", emoji: true } }, @@ -236,36 +362,27 @@ jobs: type: "section", text: { type: "mrkdwn", - text: "*Assets 64-bit ARM Link:* \($v8_link)" + text: "*\($label):* <\($url)|Download assets zip>" } }, { - type: "section", - text: { - type: "mrkdwn", - text: "*Assets 32-bit ARM Link:* \($v7_link)" - } - } + type: "context", + elements: [ + { + type: "mrkdwn", + text: "Built from `\($ref)`. Push to `/sdcard/Download/` on the device, then `adb shell touch` it." + } + ] + } ] }' > payload.json - + curl -X POST -H "Content-type: application/json" --data @payload.json "$SLACK_WEBHOOK" rm -f payload.json - - name: Cleanup google-services.json - if: always() - run: | - rm -f app/google-services.json - echo "google-services.json cleaned up successfully" - - name: Cleanup ssh + - name: Clean up outputs if: always() run: | - # Remove SSH key - rm -f ~/.ssh/id_rsa - # Clean up SSH known_hosts (remove the entry for this host) - if [ -n "$SCP_HOST" ]; then - ssh-keygen -R "$SCP_HOST" 2>/dev/null || true - fi - # Remove entire .ssh directory if empty - rmdir ~/.ssh 2>/dev/null || true + rm -f app/google-services.json + rm -rf app/build/outputs/assets/ diff --git a/scripts/cloudflare-r2-upload.py b/scripts/cloudflare-r2-upload.py index 1943d76de0..a38b8ce251 100644 --- a/scripts/cloudflare-r2-upload.py +++ b/scripts/cloudflare-r2-upload.py @@ -19,7 +19,8 @@ CLOUDFLARE_ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"] CLOUDFLARE_KEY_ID = os.environ["CLOUDFLARE_KEY_ID"] CLOUDFLARE_SECRET_ACCESS_KEY = os.environ["CLOUDFLARE_SECRET_ACCESS_KEY"] -BUCKET_NAME = "apk-repo" +BUCKET_NAME = os.environ.get("R2_BUCKET") or "apk-repo" +KEY_PREFIX = os.environ.get("R2_KEY_PREFIX", "") R2_ENDPOINT_URL = f"https://{CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" @@ -70,6 +71,8 @@ def progress_callback(bytes_amount): if extra_args: upload_kwargs["ExtraArgs"] = extra_args -print(f"Uploading {file_name} ({file_size / (1024*1024):.1f} MB) to R2...", flush=True) -s3.upload_file(file_path, BUCKET_NAME, file_name, **upload_kwargs) +object_key = f"{KEY_PREFIX}{file_name}" + +print(f"Uploading {file_name} ({file_size / (1024*1024):.1f} MB) to R2 {BUCKET_NAME}/{object_key}...", flush=True) +s3.upload_file(file_path, BUCKET_NAME, object_key, **upload_kwargs) print("Upload complete.", flush=True) From 78495c7c81906ceef6b1b9b978ab2ff548a5212f Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 14:27:13 +0100 Subject: [PATCH 2/7] ADFA-2602: Take debug ingredients from the GreenGeeks candidate staging area Reuse the transit the project already has: dev-assets can scp to GreenGeeks and this workflow already sets up the same key for the site path, so an unpublished asset set needs no new credential and no manual upload. asset_source: candidate scps from candidate_path (printed in the dev-assets asset-set run summary) instead of the live dev-assets directory. Candidates land under TMP_ASSETS_PATH, which is separate from public_html/dev-assets, so nothing overwrites the assets the nightly release depends on -- important because localMvnRepository.zip, core.cgt, documentation.db, android-sdk-* and bootstrap-* have identical filenames across toolchains. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index bcb4a79392..505b1c06b0 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -19,7 +19,12 @@ on: type: choice options: - site + - candidate - release + candidate_path: + description: 'GreenGeeks candidate staging path (asset_source: candidate), e.g. tmp/assets/candidates/42' + required: false + default: '' asset_release_tag: description: 'Release tag holding the debug ingredients (asset_source: release)' required: false @@ -77,7 +82,7 @@ jobs: access_token_scopes: 'https://www.googleapis.com/auth/drive' - name: Set up SSH key - if: inputs.asset_source == 'site' + if: inputs.asset_source == 'site' || inputs.asset_source == 'candidate' env: GREENGEEKS_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} GREENGEEKS_KEY: ${{ secrets.GREENGEEKS_SSH_PRIVATE_KEY }} @@ -141,6 +146,23 @@ jobs: -Dorg.gradle.workers.max=1 \ -Dorg.gradle.parallel=false + - name: Download debug assets from the candidate staging area + if: inputs.asset_source == 'candidate' + env: + GREENGEEKS_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} + CANDIDATE_PATH: ${{ inputs.candidate_path }} + run: | + set -euo pipefail + if [ -z "$CANDIDATE_PATH" ]; then + echo "ERROR: asset_source is 'candidate' but candidate_path is empty." + echo " Take the path from the dev-assets asset-set run summary." + exit 1 + fi + mkdir -p assets + scp "$GREENGEEKS_HOST:$CANDIDATE_PATH/debug/*" assets/ + rm -f assets/*.md5 + ls -la assets/ + - name: Download debug assets from a release if: inputs.asset_source == 'release' env: @@ -232,7 +254,7 @@ jobs: echo "google-services.json cleaned up successfully" - name: Cleanup ssh - if: always() && inputs.asset_source == 'site' + if: always() && (inputs.asset_source == 'site' || inputs.asset_source == 'candidate') run: | # Remove SSH key rm -f ~/.ssh/id_rsa From 35d3769380816759aa4bd3f86d26baf3c52f5ed6 Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 16:14:37 +0100 Subject: [PATCH 3/7] ADFA-2602: Qualify the asset zip URL by toolchain version Two runs from different toolchains wrote the same R2 key, so the second silently replaced the first at one stable URL and the Slack links were indistinguishable -- a tester who downloaded from the earlier message got 8.14.3 assets while believing they had 9.6.1. Read GRADLE_DISTRIBUTION_VERSION from the checked-out org.adfa.constants and key the upload as assets//assets-.zip, so an 8.14.3 and a 9.6.1 set can coexist and the URL says which is which. The Slack message now names the Gradle version alongside the ref. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index 505b1c06b0..df937f2611 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -327,6 +327,16 @@ jobs: -Dorg.gradle.workers.max=1 \ -Dorg.gradle.parallel=false + - name: Read the toolchain version + id: toolchain + run: | + set -euo pipefail + CONSTANTS=composite-builds/build-deps-common/constants/src/main/java/org/adfa/constants/constants.kt + ver=$(grep 'GRADLE_DISTRIBUTION_VERSION' "$CONSTANTS" | grep -o '"[0-9.]*"' | tr -d '"') + [ -n "$ver" ] || { echo "ERROR: could not read GRADLE_DISTRIBUTION_VERSION"; exit 1; } + echo "GRADLE_VERSION=$ver" >> $GITHUB_OUTPUT + echo "toolchain: gradle $ver" + - name: Verify assets zip id: assets_zip run: | @@ -354,10 +364,10 @@ jobs: CLOUDFLARE_KEY_ID: ${{ vars.CLOUDFLARE_KEY_ID }} CLOUDFLARE_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }} R2_BUCKET: ${{ vars.R2_ASSETS_BUCKET || 'apk-repo' }} - R2_KEY_PREFIX: assets/ + R2_KEY_PREFIX: assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/ run: | uv run --with boto3 scripts/cloudflare-r2-upload.py "${{ steps.assets_zip.outputs.ASSETS_PATH }}" - echo "DOWNLOAD_URL=https://download.appdevforall.org/assets/assets-${{ matrix.arch }}.zip" >> $GITHUB_OUTPUT + echo "DOWNLOAD_URL=https://download.appdevforall.org/assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/assets-${{ matrix.arch }}.zip" >> $GITHUB_OUTPUT - name: Send Slack notification env: @@ -365,11 +375,13 @@ jobs: ASSET_LABEL: ${{ matrix.label }} DOWNLOAD_URL: ${{ steps.r2.outputs.DOWNLOAD_URL }} BUILD_REF: ${{ inputs.ref }} + GRADLE_VERSION: ${{ steps.toolchain.outputs.GRADLE_VERSION }} run: | jq -n \ --arg label "$ASSET_LABEL" \ --arg url "$DOWNLOAD_URL" \ --arg ref "$BUILD_REF" \ + --arg gradle "$GRADLE_VERSION" \ '{ blocks: [ { @@ -392,7 +404,7 @@ jobs: elements: [ { type: "mrkdwn", - text: "Built from `\($ref)`. Push to `/sdcard/Download/` on the device, then `adb shell touch` it." + text: "Gradle \($gradle), built from `\($ref)`. Push to `/sdcard/Download/` on the device, then `adb shell touch` it." } ] } From af7824a3c2b0f7c34606149dc64793355f24604b Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 16:25:09 +0100 Subject: [PATCH 4/7] ADFA-2602: Run the uploader from this workflow's own checkout, and mute Slack by default The 9.6.1 runs wrote their zips to the bucket root instead of the versioned prefix, so the versioned URL 404'd while the run reported success. Cause: the workflow file comes from the dispatched ref, but the repository content -- scripts included -- comes from inputs.ref. Building the toolchain branch therefore ran that branch's cloudflare-r2-upload.py, which predates R2_BUCKET/R2_KEY_PREFIX and ignored both. The tell is that the modified script's 'to R2 /' line never appears in those logs. Check this workflow's own ref out to .workflow-tools and run the uploader from there, so the workflow no longer depends on the branch under build carrying its tooling. Slack is now behind notify_slack (default false): test runs were posting links to the team channel, and with the bug above those links were wrong. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index df937f2611..2fb5035dae 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -21,6 +21,11 @@ on: - site - candidate - release + notify_slack: + description: Post the download links to Slack + required: false + type: boolean + default: false candidate_path: description: 'GreenGeeks candidate staging path (asset_source: candidate), e.g. tmp/assets/candidates/42' required: false @@ -354,6 +359,12 @@ jobs: echo "ASSETS_PATH=$zip_path" >> $GITHUB_OUTPUT echo "$(du -h "$zip_path" | cut -f1) $zip_path" + - name: Checkout this workflow's tooling + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + path: .workflow-tools + - name: Install uv uses: astral-sh/setup-uv@v6 @@ -366,10 +377,11 @@ jobs: R2_BUCKET: ${{ vars.R2_ASSETS_BUCKET || 'apk-repo' }} R2_KEY_PREFIX: assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/ run: | - uv run --with boto3 scripts/cloudflare-r2-upload.py "${{ steps.assets_zip.outputs.ASSETS_PATH }}" + uv run --with boto3 .workflow-tools/scripts/cloudflare-r2-upload.py "${{ steps.assets_zip.outputs.ASSETS_PATH }}" echo "DOWNLOAD_URL=https://download.appdevforall.org/assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/assets-${{ matrix.arch }}.zip" >> $GITHUB_OUTPUT - name: Send Slack notification + if: inputs.notify_slack env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} ASSET_LABEL: ${{ matrix.label }} From 6ee65e9a53e1fa14f045e489df8bcda7b2bfd1ab Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 17:03:37 +0100 Subject: [PATCH 5/7] style: spotless reformat, no functional change The ratchet is file-level, so adding the R2_BUCKET/R2_KEY_PREFIX overrides pulled the whole file under it and required reindenting the pre-existing 4-space Python to tabs. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- scripts/cloudflare-r2-upload.py | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/scripts/cloudflare-r2-upload.py b/scripts/cloudflare-r2-upload.py index a38b8ce251..07c35f4fc9 100644 --- a/scripts/cloudflare-r2-upload.py +++ b/scripts/cloudflare-r2-upload.py @@ -6,15 +6,15 @@ from botocore.config import Config REQUIRED_ENV = ( - "CLOUDFLARE_ACCOUNT_ID", - "CLOUDFLARE_KEY_ID", - "CLOUDFLARE_SECRET_ACCESS_KEY", + "CLOUDFLARE_ACCOUNT_ID", + "CLOUDFLARE_KEY_ID", + "CLOUDFLARE_SECRET_ACCESS_KEY", ) for name in REQUIRED_ENV: - if not os.environ.get(name): - print(f"ERROR: {name} environment variable is not set.", file=sys.stderr) - sys.exit(1) + if not os.environ.get(name): + print(f"ERROR: {name} environment variable is not set.", file=sys.stderr) + sys.exit(1) CLOUDFLARE_ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"] CLOUDFLARE_KEY_ID = os.environ["CLOUDFLARE_KEY_ID"] @@ -25,23 +25,23 @@ R2_ENDPOINT_URL = f"https://{CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" config = Config( - read_timeout=300, - connect_timeout=60, - retries={"max_attempts": 10}, + read_timeout=300, + connect_timeout=60, + retries={"max_attempts": 10}, ) s3 = boto3.client( - service_name="s3", - endpoint_url=R2_ENDPOINT_URL, - aws_access_key_id=CLOUDFLARE_KEY_ID, - aws_secret_access_key=CLOUDFLARE_SECRET_ACCESS_KEY, - region_name="auto", - config=config, + service_name="s3", + endpoint_url=R2_ENDPOINT_URL, + aws_access_key_id=CLOUDFLARE_KEY_ID, + aws_secret_access_key=CLOUDFLARE_SECRET_ACCESS_KEY, + region_name="auto", + config=config, ) if len(sys.argv) < 2: - print("Usage: cloudflare-r2-upload.py ", file=sys.stderr) - sys.exit(1) + print("Usage: cloudflare-r2-upload.py ", file=sys.stderr) + sys.exit(1) file_path = sys.argv[1] file_size = os.path.getsize(file_path) @@ -49,27 +49,27 @@ extra_args = {} if file_name.lower().endswith(".apk"): - extra_args["ContentType"] = "application/vnd.android.package-archive" + extra_args["ContentType"] = "application/vnd.android.package-archive" # Progress callback: print new lines at 10% intervals for CI-friendly logs (bytes_amount is incremental per call) _seen_so_far = [0] _last_printed_pct = [-1] def progress_callback(bytes_amount): - _seen_so_far[0] += bytes_amount - if file_size <= 0: - return - pct = int(100 * _seen_so_far[0] / file_size) - if pct >= _last_printed_pct[0] + 10 or pct == 100: - _last_printed_pct[0] = pct - mb = _seen_so_far[0] / (1024 * 1024) - total_mb = file_size / (1024 * 1024) - print(f"Upload progress: {pct}% ({mb:.1f} MB / {total_mb:.1f} MB)", flush=True) + _seen_so_far[0] += bytes_amount + if file_size <= 0: + return + pct = int(100 * _seen_so_far[0] / file_size) + if pct >= _last_printed_pct[0] + 10 or pct == 100: + _last_printed_pct[0] = pct + mb = _seen_so_far[0] / (1024 * 1024) + total_mb = file_size / (1024 * 1024) + print(f"Upload progress: {pct}% ({mb:.1f} MB / {total_mb:.1f} MB)", flush=True) upload_kwargs = {"Callback": progress_callback} if extra_args: - upload_kwargs["ExtraArgs"] = extra_args + upload_kwargs["ExtraArgs"] = extra_args object_key = f"{KEY_PREFIX}{file_name}" From bad69e2308e24a21432980d286990269ce0230eb Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Wed, 9 Sep 2026 18:49:23 +0100 Subject: [PATCH 6/7] ADFA-2602: Address review on the asset-zip workflow Three findings from @jatezzz on #1815: - The content gate checked seven entries and skipped exactly the two whose filenames encode the Gradle version, so a constants.kt bump without the matching createAssetsZip/Asset() rename would publish an 8.14.3 payload to assets/9.6.1/ with a Slack line claiming 9.6.1 -- the mislabel this PR exists to prevent. Gate on the interpolated names as well. - The ~/.ssh/id_rsa hazard the PR body describes spans runs, not just matrix legs: two overlapping dispatches let one run's Cleanup ssh fire during the other's scp. Added a top-level concurrency group with cancel-in-progress: false, since cancelling mid-scp would leave the key and partial staging on the shared runner. - The artifact round-trip re-compressed an already-compressed payload and shipped each leg the other ABI's assets. Measured: the artifact was 1,484,450,451 bytes against ~1,590 MB of input, so compression bought ~6.7% -- documentation.db does not compress either, because its content blobs are already brotli-encoded. Now compression-level: 0, and the staging splits into a common artifact plus one per ABI, so each leg no longer downloads the other ABI's ~431 MB. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 51 +++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index 2fb5035dae..9848786ed9 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -35,6 +35,10 @@ on: required: false default: adfa-2602-rc +concurrency: + group: generate-assets + cancel-in-progress: false + env: SCP_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} @@ -244,11 +248,37 @@ jobs: || { echo "ERROR: documentation.db predates the templateId column; docs requests will 500"; exit 1; } echo "All ingredients staged" - - name: Upload staged assets + - name: Upload staged assets (ABI-independent) uses: actions/upload-artifact@v4 with: - name: debug-assets - path: assets/ + name: debug-assets-common + path: | + assets/ + !assets/android-sdk-*.zip + !assets/bootstrap-*.zip + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + - name: Upload staged assets (v8) + uses: actions/upload-artifact@v4 + with: + name: debug-assets-v8 + path: | + assets/android-sdk-arm64-v8a.zip + assets/bootstrap-arm64-v8a.zip + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + - name: Upload staged assets (v7) + uses: actions/upload-artifact@v4 + with: + name: debug-assets-v7 + path: | + assets/android-sdk-armeabi-v7a.zip + assets/bootstrap-armeabi-v7a.zip + compression-level: 0 retention-days: 1 if-no-files-found: error @@ -317,10 +347,16 @@ jobs: run: | echo "$GOOGLE_SERVICES_JSON" > app/google-services.json - - name: Download staged assets + - name: Download staged assets (ABI-independent) + uses: actions/download-artifact@v4 + with: + name: debug-assets-common + path: assets + + - name: Download staged assets (this ABI) uses: actions/download-artifact@v4 with: - name: debug-assets + name: debug-assets-${{ matrix.abi }} path: assets - name: Assemble assets zip @@ -344,6 +380,8 @@ jobs: - name: Verify assets zip id: assets_zip + env: + GRADLE_VERSION: ${{ steps.toolchain.outputs.GRADLE_VERSION }} run: | zip_path="app/build/outputs/assets/assets-${{ matrix.arch }}.zip" if [ ! -s "$zip_path" ]; then @@ -352,7 +390,8 @@ jobs: fi unzip -tq "$zip_path" for entry in android-sdk.zip bootstrap.zip localMvnRepository.zip documentation.db \ - core.cgt plugin-artifacts.zip plugin-maven-repo.zip; do + core.cgt plugin-artifacts.zip plugin-maven-repo.zip \ + "gradle-$GRADLE_VERSION-bin.zip" "gradle-api-$GRADLE_VERSION.jar.zip"; do unzip -l "$zip_path" | grep -qE "[[:space:]]$entry$" \ || { echo "ERROR: $zip_path is missing entry $entry"; exit 1; } done From 4921639145d1592da75a3918859667865e2a99d7 Mon Sep 17 00:00:00 2001 From: Daniel Alome Date: Thu, 10 Sep 2026 19:01:09 +0100 Subject: [PATCH 7/7] ADFA-2602: Keep the asset zips on Google Drive, not R2 Reverts the storage change. Moving to R2 was not asked for and was not part of fixing what this PR exists to fix - the workflow needed a ref input, an ingredient source and per-ABI parallelism, none of which depend on where the zip lands. The two fixed Drive file IDs and their stable drive.google.com/file/d//view links are the behaviour the team already has, so they stay. - The zip job PATCHes its zip to the Drive file for its ABI, exactly as the single-job version did, and gets its own Drive access token: the matrix leg cannot reuse the token minted in prepare. - The file id comes from the matrix as a secret name, indexed as secrets[matrix.drive_file_id_secret], rather than a matrix.abi == 'v8' && ... || ... ternary, which would silently fall through to the v7 file if the v8 secret were ever empty. An explicit guard fails the step when the secret is missing. - No set -euo pipefail in that step, matching the original: with curl --fail, set -e aborts the assignment before the HTTP-status check can report the status, so the explicit error message would be unreachable. - Slack goes back to the original wording ("*Assets 64-bit ARM Link:* <...>"). It keeps the "Gradle , built from " context line, which is new - the Drive URL is stable by design, so nothing in the notification otherwise says which toolchain the zip carries. That is how an 8.14.3 zip got downloaded as 9.6.1. Say so and I will drop the line. - Drops the .workflow-tools checkout and the uv install: both existed only to run cloudflare-r2-upload.py. curl is on the runner, so this workflow no longer calls a repo script at all, which removes the dispatched-ref-vs-inputs.ref mismatch that made those steps necessary. - scripts/cloudflare-r2-upload.py is restored byte-identical to stage. Its R2_BUCKET/R2_KEY_PREFIX overrides had no other caller, and reverting it also drops the Spotless reformat that only happened because the edit pulled the file under the file-level ratchet. Verified: the workflow parses, every step body passes bash -n, and the Slack jq program renders with the Drive link. Claude-Session: https://claude.ai/code/session_017HGpMsUzZ5wxCfMtDZ2HGP --- .github/workflows/generate_assets.yml | 57 ++++++++++++++--------- scripts/cloudflare-r2-upload.py | 65 +++++++++++++-------------- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index 9848786ed9..b115b02dac 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -312,9 +312,11 @@ jobs: - abi: v8 arch: arm64-v8a label: 64-bit ARM + drive_file_id_secret: ASSETS_V8_FILE_ID - abi: v7 arch: armeabi-v7a label: 32-bit ARM + drive_file_id_secret: ASSETS_V7_FILE_ID steps: - name: Checkout repository @@ -347,6 +349,15 @@ jobs: run: | echo "$GOOGLE_SERVICES_JSON" > app/google-services.json + - name: Authenticate to Google Cloud for Drive access + id: auth_drive + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.WIF_PROVIDER }} + service_account: ${{ secrets.IDENTITY_EMAIL }} + token_format: 'access_token' + access_token_scopes: 'https://www.googleapis.com/auth/drive' + - name: Download staged assets (ABI-independent) uses: actions/download-artifact@v4 with: @@ -398,33 +409,39 @@ jobs: echo "ASSETS_PATH=$zip_path" >> $GITHUB_OUTPUT echo "$(du -h "$zip_path" | cut -f1) $zip_path" - - name: Checkout this workflow's tooling - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - path: .workflow-tools - - - name: Install uv - uses: astral-sh/setup-uv@v6 - - - name: Upload assets zip to Cloudflare R2 - id: r2 + - name: Upload assets zip to Google Drive + id: drive env: - CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_KEY_ID: ${{ vars.CLOUDFLARE_KEY_ID }} - CLOUDFLARE_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }} - R2_BUCKET: ${{ vars.R2_ASSETS_BUCKET || 'apk-repo' }} - R2_KEY_PREFIX: assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/ + ACCESS_TOKEN: ${{ steps.auth_drive.outputs.access_token }} + DRIVE_FILE_ID: ${{ secrets[matrix.drive_file_id_secret] }} + ASSETS_PATH: ${{ steps.assets_zip.outputs.ASSETS_PATH }} run: | - uv run --with boto3 .workflow-tools/scripts/cloudflare-r2-upload.py "${{ steps.assets_zip.outputs.ASSETS_PATH }}" - echo "DOWNLOAD_URL=https://download.appdevforall.org/assets/${{ steps.toolchain.outputs.GRADLE_VERSION }}/assets-${{ matrix.arch }}.zip" >> $GITHUB_OUTPUT + if [ -z "$DRIVE_FILE_ID" ]; then + echo "ERROR: ${{ matrix.drive_file_id_secret }} is not set" + exit 1 + fi + + echo "Uploading $ASSETS_PATH to Drive file $DRIVE_FILE_ID..." + response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/zip" \ + --upload-file "$ASSETS_PATH" \ + "https://www.googleapis.com/upload/drive/v3/files/${DRIVE_FILE_ID}?uploadType=media") + + if [[ "$response" -ne 200 ]]; then + echo "Upload of $ASSETS_PATH failed with HTTP status $response" + exit 1 + fi + + echo "Upload complete." + echo "DOWNLOAD_URL=https://drive.google.com/file/d/${DRIVE_FILE_ID}/view" >> $GITHUB_OUTPUT - name: Send Slack notification if: inputs.notify_slack env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} ASSET_LABEL: ${{ matrix.label }} - DOWNLOAD_URL: ${{ steps.r2.outputs.DOWNLOAD_URL }} + DOWNLOAD_URL: ${{ steps.drive.outputs.DOWNLOAD_URL }} BUILD_REF: ${{ inputs.ref }} GRADLE_VERSION: ${{ steps.toolchain.outputs.GRADLE_VERSION }} run: | @@ -447,7 +464,7 @@ jobs: type: "section", text: { type: "mrkdwn", - text: "*\($label):* <\($url)|Download assets zip>" + text: "*Assets \($label) Link:* <\($url)|Download Assets Zip \($label)>" } }, { diff --git a/scripts/cloudflare-r2-upload.py b/scripts/cloudflare-r2-upload.py index 07c35f4fc9..1943d76de0 100644 --- a/scripts/cloudflare-r2-upload.py +++ b/scripts/cloudflare-r2-upload.py @@ -6,42 +6,41 @@ from botocore.config import Config REQUIRED_ENV = ( - "CLOUDFLARE_ACCOUNT_ID", - "CLOUDFLARE_KEY_ID", - "CLOUDFLARE_SECRET_ACCESS_KEY", + "CLOUDFLARE_ACCOUNT_ID", + "CLOUDFLARE_KEY_ID", + "CLOUDFLARE_SECRET_ACCESS_KEY", ) for name in REQUIRED_ENV: - if not os.environ.get(name): - print(f"ERROR: {name} environment variable is not set.", file=sys.stderr) - sys.exit(1) + if not os.environ.get(name): + print(f"ERROR: {name} environment variable is not set.", file=sys.stderr) + sys.exit(1) CLOUDFLARE_ACCOUNT_ID = os.environ["CLOUDFLARE_ACCOUNT_ID"] CLOUDFLARE_KEY_ID = os.environ["CLOUDFLARE_KEY_ID"] CLOUDFLARE_SECRET_ACCESS_KEY = os.environ["CLOUDFLARE_SECRET_ACCESS_KEY"] -BUCKET_NAME = os.environ.get("R2_BUCKET") or "apk-repo" -KEY_PREFIX = os.environ.get("R2_KEY_PREFIX", "") +BUCKET_NAME = "apk-repo" R2_ENDPOINT_URL = f"https://{CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" config = Config( - read_timeout=300, - connect_timeout=60, - retries={"max_attempts": 10}, + read_timeout=300, + connect_timeout=60, + retries={"max_attempts": 10}, ) s3 = boto3.client( - service_name="s3", - endpoint_url=R2_ENDPOINT_URL, - aws_access_key_id=CLOUDFLARE_KEY_ID, - aws_secret_access_key=CLOUDFLARE_SECRET_ACCESS_KEY, - region_name="auto", - config=config, + service_name="s3", + endpoint_url=R2_ENDPOINT_URL, + aws_access_key_id=CLOUDFLARE_KEY_ID, + aws_secret_access_key=CLOUDFLARE_SECRET_ACCESS_KEY, + region_name="auto", + config=config, ) if len(sys.argv) < 2: - print("Usage: cloudflare-r2-upload.py ", file=sys.stderr) - sys.exit(1) + print("Usage: cloudflare-r2-upload.py ", file=sys.stderr) + sys.exit(1) file_path = sys.argv[1] file_size = os.path.getsize(file_path) @@ -49,30 +48,28 @@ extra_args = {} if file_name.lower().endswith(".apk"): - extra_args["ContentType"] = "application/vnd.android.package-archive" + extra_args["ContentType"] = "application/vnd.android.package-archive" # Progress callback: print new lines at 10% intervals for CI-friendly logs (bytes_amount is incremental per call) _seen_so_far = [0] _last_printed_pct = [-1] def progress_callback(bytes_amount): - _seen_so_far[0] += bytes_amount - if file_size <= 0: - return - pct = int(100 * _seen_so_far[0] / file_size) - if pct >= _last_printed_pct[0] + 10 or pct == 100: - _last_printed_pct[0] = pct - mb = _seen_so_far[0] / (1024 * 1024) - total_mb = file_size / (1024 * 1024) - print(f"Upload progress: {pct}% ({mb:.1f} MB / {total_mb:.1f} MB)", flush=True) + _seen_so_far[0] += bytes_amount + if file_size <= 0: + return + pct = int(100 * _seen_so_far[0] / file_size) + if pct >= _last_printed_pct[0] + 10 or pct == 100: + _last_printed_pct[0] = pct + mb = _seen_so_far[0] / (1024 * 1024) + total_mb = file_size / (1024 * 1024) + print(f"Upload progress: {pct}% ({mb:.1f} MB / {total_mb:.1f} MB)", flush=True) upload_kwargs = {"Callback": progress_callback} if extra_args: - upload_kwargs["ExtraArgs"] = extra_args + upload_kwargs["ExtraArgs"] = extra_args -object_key = f"{KEY_PREFIX}{file_name}" - -print(f"Uploading {file_name} ({file_size / (1024*1024):.1f} MB) to R2 {BUCKET_NAME}/{object_key}...", flush=True) -s3.upload_file(file_path, BUCKET_NAME, object_key, **upload_kwargs) +print(f"Uploading {file_name} ({file_size / (1024*1024):.1f} MB) to R2...", flush=True) +s3.upload_file(file_path, BUCKET_NAME, file_name, **upload_kwargs) print("Upload complete.", flush=True)