diff --git a/.github/workflows/generate_assets.yml b/.github/workflows/generate_assets.yml index 4741fa56f4..b115b02dac 100644 --- a/.github/workflows/generate_assets.yml +++ b/.github/workflows/generate_assets.yml @@ -1,4 +1,4 @@ -name: Generate Assets Zips and Upload to Google Drive +name: Generate Assets Zips permissions: id-token: write @@ -7,13 +7,44 @@ permissions: on: workflow_dispatch: + inputs: + ref: + description: Branch or tag to build the asset zips from + required: false + default: stage + asset_source: + description: Where the debug asset ingredients come from + required: false + default: site + type: choice + options: + - site + - candidate + - release + notify_slack: + description: Post the download links to Slack + required: false + type: boolean + default: false + candidate_path: + description: 'GreenGeeks candidate staging path (asset_source: candidate), e.g. tmp/assets/candidates/42' + required: false + default: '' + asset_release_tag: + description: 'Release tag holding the debug ingredients (asset_source: release)' + required: false + default: adfa-2602-rc + +concurrency: + group: generate-assets + cancel-in-progress: false env: SCP_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} jobs: - generate_assets: - name: Generate Assets Zips + prepare: + name: Stage debug assets runs-on: self-hosted timeout-minutes: 90 @@ -21,7 +52,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: stage + ref: ${{ inputs.ref }} + + - name: Remove stale staged assets + run: | + rm -f assets/*.zip assets/documentation.db assets/core.cgt + ls -la assets/ 2>/dev/null || true - name: Check if Nix is installed id: check_nix @@ -55,6 +91,7 @@ jobs: access_token_scopes: 'https://www.googleapis.com/auth/drive' - name: Set up SSH key + if: inputs.asset_source == 'site' || inputs.asset_source == 'candidate' env: GREENGEEKS_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} GREENGEEKS_KEY: ${{ secrets.GREENGEEKS_SSH_PRIVATE_KEY }} @@ -109,7 +146,8 @@ jobs: rm -f ~/.ssh/id_ed25519 ~/.ssh/id_ecdsa ~/.ssh/id_dsa ~/.ssh/id_rsa.pub 2>/dev/null ssh-keyscan -H "$GREENGEEKS_HOST" >> ~/.ssh/known_hosts 2>/dev/null - - name: Download debug assets + - name: Download debug assets from the site + if: inputs.asset_source == 'site' run: | flox activate -d flox/base -- ./gradlew :app:assetsDownloadDebug --no-daemon \ -Dorg.gradle.jvmargs="-Xmx10g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ @@ -117,12 +155,45 @@ jobs: -Dorg.gradle.workers.max=1 \ -Dorg.gradle.parallel=false + - name: Download debug assets from the candidate staging area + if: inputs.asset_source == 'candidate' + env: + GREENGEEKS_HOST: ${{ vars.GREENGEEKS_SSH_HOST }} + CANDIDATE_PATH: ${{ inputs.candidate_path }} + run: | + set -euo pipefail + if [ -z "$CANDIDATE_PATH" ]; then + echo "ERROR: asset_source is 'candidate' but candidate_path is empty." + echo " Take the path from the dev-assets asset-set run summary." + exit 1 + fi + mkdir -p assets + scp "$GREENGEEKS_HOST:$CANDIDATE_PATH/debug/*" assets/ + rm -f assets/*.md5 + ls -la assets/ + + - name: Download debug assets from a release + if: inputs.asset_source == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ASSET_TAG: ${{ inputs.asset_release_tag }} + run: | + mkdir -p assets + gh release download "$ASSET_TAG" --dir assets --clobber \ + --pattern 'android-sdk-*.zip' \ + --pattern 'bootstrap-*.zip' \ + --pattern 'gradle-*-bin.zip' \ + --pattern 'gradle-api-*.jar.zip' \ + --pattern 'localMvnRepository.zip' \ + --pattern 'core.cgt' + ls -la assets/ + - name: Download latest documentation.db from Google Drive run: | DB_FILE_ID="${{ secrets.DOCUMENTATION_DB_FILE_ID }}" ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" - if [ -z "DB_FILE_ID" ]; then + if [ -z "$DB_FILE_ID" ]; then echo "ERROR: DOCUMENTATION_DB_FILE_ID secret not set" echo "Please set the DOCUMENTATION_DB_FILE_ID secret in repository settings" exit 1 @@ -151,84 +222,241 @@ jobs: echo "Successfully downloaded documentation.db ($FILE_SIZE_HUMAN)" - - name: Assemble Assets + - name: Verify staged ingredients run: | - flox activate -d flox/base -- ./gradlew :app:assembleAssets --no-daemon \ - -Dorg.gradle.jvmargs="-Xmx10g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ + missing=0 + for f in localMvnRepository.zip documentation.db core.cgt \ + android-sdk-arm64-v8a.zip android-sdk-armeabi-v7a.zip \ + bootstrap-arm64-v8a.zip bootstrap-armeabi-v7a.zip; do + if [ ! -s "assets/$f" ]; then + echo "ERROR: missing or empty assets/$f" + missing=1 + fi + done + if ! ls assets/gradle-*-bin.zip >/dev/null 2>&1; then + echo "ERROR: no assets/gradle-*-bin.zip staged" + missing=1 + fi + if ! ls assets/gradle-api-*.jar.zip >/dev/null 2>&1; then + echo "ERROR: no assets/gradle-api-*.jar.zip staged" + missing=1 + fi + [ "$missing" -eq 0 ] || exit 1 + command -v sqlite3 >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq sqlite3; } + sqlite3 assets/documentation.db \ + "SELECT 1 FROM pragma_table_info('Content') WHERE name='templateId';" | grep -q 1 \ + || { echo "ERROR: documentation.db predates the templateId column; docs requests will 500"; exit 1; } + echo "All ingredients staged" + + - name: Upload staged assets (ABI-independent) + uses: actions/upload-artifact@v4 + with: + name: debug-assets-common + path: | + assets/ + !assets/android-sdk-*.zip + !assets/bootstrap-*.zip + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + - name: Upload staged assets (v8) + uses: actions/upload-artifact@v4 + with: + name: debug-assets-v8 + path: | + assets/android-sdk-arm64-v8a.zip + assets/bootstrap-arm64-v8a.zip + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + - name: Upload staged assets (v7) + uses: actions/upload-artifact@v4 + with: + name: debug-assets-v7 + path: | + assets/android-sdk-armeabi-v7a.zip + assets/bootstrap-armeabi-v7a.zip + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + - name: Cleanup google-services.json + if: always() + run: | + rm -f app/google-services.json + echo "google-services.json cleaned up successfully" + + - name: Cleanup ssh + if: always() && (inputs.asset_source == 'site' || inputs.asset_source == 'candidate') + run: | + # Remove SSH key + rm -f ~/.ssh/id_rsa + # Clean up SSH known_hosts (remove the entry for this host) + if [ -n "$SCP_HOST" ]; then + ssh-keygen -R "$SCP_HOST" 2>/dev/null || true + fi + # Remove entire .ssh directory if empty + rmdir ~/.ssh 2>/dev/null || true + + zip: + name: Zip assets + runs-on: self-hosted + timeout-minutes: 60 + needs: prepare + strategy: + fail-fast: false + matrix: + include: + - abi: v8 + arch: arm64-v8a + label: 64-bit ARM + drive_file_id_secret: ASSETS_V8_FILE_ID + - abi: v7 + arch: armeabi-v7a + label: 32-bit ARM + drive_file_id_secret: ASSETS_V7_FILE_ID + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Remove stale staged assets + run: | + rm -f assets/*.zip assets/documentation.db assets/core.cgt + + - name: Check if Nix is installed + id: check_nix + run: | + if command -v nix >/dev/null 2>&1; then + echo "nix is installed" + echo "nix_installed=true" >> $GITHUB_ENV + else + echo "nix is not installed" + echo "nix_installed=false" >> $GITHUB_ENV + fi + + - name: Install Flox + if: env.nix_installed == 'false' + uses: flox/install-flox-action@v2 + + - name: Create google-services.json + env: + GOOGLE_SERVICES_JSON: ${{ secrets.GOOGLE_SERVICES_JSON }} + run: | + echo "$GOOGLE_SERVICES_JSON" > app/google-services.json + + - name: Authenticate to Google Cloud for Drive access + id: auth_drive + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.WIF_PROVIDER }} + service_account: ${{ secrets.IDENTITY_EMAIL }} + token_format: 'access_token' + access_token_scopes: 'https://www.googleapis.com/auth/drive' + + - name: Download staged assets (ABI-independent) + uses: actions/download-artifact@v4 + with: + name: debug-assets-common + path: assets + + - name: Download staged assets (this ABI) + uses: actions/download-artifact@v4 + with: + name: debug-assets-${{ matrix.abi }} + path: assets + + - name: Assemble assets zip + run: | + variant_upper=$(echo "${{ matrix.abi }}" | tr '[:lower:]' '[:upper:]') + flox activate -d flox/base -- ./gradlew :app:assemble${variant_upper}Assets --no-daemon \ + -Dorg.gradle.jvmargs="-Xmx6g -XX:MaxMetaspaceSize=2g -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" \ -Dandroid.aapt2.daemonHeapSize=4096M \ -Dorg.gradle.workers.max=1 \ -Dorg.gradle.parallel=false - - name: V8 Assets Path - id: assets_v8 + - name: Read the toolchain version + id: toolchain run: | - assets_path="app/build/outputs/assets/assets-arm64-v8a.zip" - echo "ASSETS_PATH=$assets_path" >> $GITHUB_OUTPUT + set -euo pipefail + CONSTANTS=composite-builds/build-deps-common/constants/src/main/java/org/adfa/constants/constants.kt + ver=$(grep 'GRADLE_DISTRIBUTION_VERSION' "$CONSTANTS" | grep -o '"[0-9.]*"' | tr -d '"') + [ -n "$ver" ] || { echo "ERROR: could not read GRADLE_DISTRIBUTION_VERSION"; exit 1; } + echo "GRADLE_VERSION=$ver" >> $GITHUB_OUTPUT + echo "toolchain: gradle $ver" - - name: V7 Assets Path - id: assets_v7 + - name: Verify assets zip + id: assets_zip + env: + GRADLE_VERSION: ${{ steps.toolchain.outputs.GRADLE_VERSION }} run: | - assets_path="app/build/outputs/assets/assets-armeabi-v7a.zip" - echo "ASSETS_PATH=$assets_path" >> $GITHUB_OUTPUT + zip_path="app/build/outputs/assets/assets-${{ matrix.arch }}.zip" + if [ ! -s "$zip_path" ]; then + echo "ERROR: $zip_path was not produced" + exit 1 + fi + unzip -tq "$zip_path" + for entry in android-sdk.zip bootstrap.zip localMvnRepository.zip documentation.db \ + core.cgt plugin-artifacts.zip plugin-maven-repo.zip \ + "gradle-$GRADLE_VERSION-bin.zip" "gradle-api-$GRADLE_VERSION.jar.zip"; do + unzip -l "$zip_path" | grep -qE "[[:space:]]$entry$" \ + || { echo "ERROR: $zip_path is missing entry $entry"; exit 1; } + done + echo "ASSETS_PATH=$zip_path" >> $GITHUB_OUTPUT + echo "$(du -h "$zip_path" | cut -f1) $zip_path" - - name: Upload asset zips to Google Drive + - name: Upload assets zip to Google Drive + id: drive + env: + ACCESS_TOKEN: ${{ steps.auth_drive.outputs.access_token }} + DRIVE_FILE_ID: ${{ secrets[matrix.drive_file_id_secret] }} + ASSETS_PATH: ${{ steps.assets_zip.outputs.ASSETS_PATH }} run: | - echo "Uploading assets v8 and v7 to Google Drive..." - - ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" - V8_FILE_ID="${{ secrets.ASSETS_V8_FILE_ID }}" - V7_FILE_ID="${{ secrets.ASSETS_V7_FILE_ID }}" - - V8_PATH="${{ steps.assets_v8.outputs.ASSETS_PATH }}" - V7_PATH="${{ steps.assets_v7.outputs.ASSETS_PATH }}" - - # Upload v8 - response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ - -H "Authorization: Bearer $ACCESS_TOKEN" \ - -H "Content-Type: application/zip" \ - --upload-file "${V8_PATH}" \ - "https://www.googleapis.com/upload/drive/v3/files/${V8_FILE_ID}?uploadType=media") - - if [[ "$response" -ne 200 ]]; then - echo "Upload of ${V8_PATH} failed with HTTP status $response" + if [ -z "$DRIVE_FILE_ID" ]; then + echo "ERROR: ${{ matrix.drive_file_id_secret }} is not set" exit 1 fi - # Upload v7 + echo "Uploading $ASSETS_PATH to Drive file $DRIVE_FILE_ID..." response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ -H "Authorization: Bearer $ACCESS_TOKEN" \ -H "Content-Type: application/zip" \ - --upload-file "${V7_PATH}" \ - "https://www.googleapis.com/upload/drive/v3/files/${V7_FILE_ID}?uploadType=media") + --upload-file "$ASSETS_PATH" \ + "https://www.googleapis.com/upload/drive/v3/files/${DRIVE_FILE_ID}?uploadType=media") if [[ "$response" -ne 200 ]]; then - echo "Upload of ${V7_PATH} failed with HTTP status $response" + echo "Upload of $ASSETS_PATH failed with HTTP status $response" exit 1 fi echo "Upload complete." + echo "DOWNLOAD_URL=https://drive.google.com/file/d/${DRIVE_FILE_ID}/view" >> $GITHUB_OUTPUT - - name: Send Rich Slack Notification + - name: Send Slack notification + if: inputs.notify_slack env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + ASSET_LABEL: ${{ matrix.label }} + DOWNLOAD_URL: ${{ steps.drive.outputs.DOWNLOAD_URL }} + BUILD_REF: ${{ inputs.ref }} + GRADLE_VERSION: ${{ steps.toolchain.outputs.GRADLE_VERSION }} run: | - - V8_FILE_ID="${{ secrets.ASSETS_V8_FILE_ID }}" - V7_FILE_ID="${{ secrets.ASSETS_V7_FILE_ID }}" - - GDRIVE_V8_LINK="" - GDRIVE_V7_LINK="" - jq -n \ - --arg v8_link "$GDRIVE_V8_LINK" \ - --arg v7_link "$GDRIVE_V7_LINK" \ + --arg label "$ASSET_LABEL" \ + --arg url "$DOWNLOAD_URL" \ + --arg ref "$BUILD_REF" \ + --arg gradle "$GRADLE_VERSION" \ '{ blocks: [ { type: "header", text: { type: "plain_text", - text: ":rocket: [Updated] New Assets Zips Available", + text: ":rocket: [Updated] New Assets Zip Available", emoji: true } }, @@ -236,36 +464,27 @@ jobs: type: "section", text: { type: "mrkdwn", - text: "*Assets 64-bit ARM Link:* \($v8_link)" + text: "*Assets \($label) Link:* <\($url)|Download Assets Zip \($label)>" } }, { - type: "section", - text: { - type: "mrkdwn", - text: "*Assets 32-bit ARM Link:* \($v7_link)" - } - } + type: "context", + elements: [ + { + type: "mrkdwn", + text: "Gradle \($gradle), built from `\($ref)`. Push to `/sdcard/Download/` on the device, then `adb shell touch` it." + } + ] + } ] }' > payload.json - + curl -X POST -H "Content-type: application/json" --data @payload.json "$SLACK_WEBHOOK" rm -f payload.json - - name: Cleanup google-services.json - if: always() - run: | - rm -f app/google-services.json - echo "google-services.json cleaned up successfully" - - name: Cleanup ssh + - name: Clean up outputs if: always() run: | - # Remove SSH key - rm -f ~/.ssh/id_rsa - # Clean up SSH known_hosts (remove the entry for this host) - if [ -n "$SCP_HOST" ]; then - ssh-keygen -R "$SCP_HOST" 2>/dev/null || true - fi - # Remove entire .ssh directory if empty - rmdir ~/.ssh 2>/dev/null || true + rm -f app/google-services.json + rm -rf app/build/outputs/assets/