From 3cd8fe5f4323c03a20c062b2fc0a5d9cdd1ffd92 Mon Sep 17 00:00:00 2001 From: Jeremy Beard Date: Mon, 19 Oct 2026 10:33:36 -0400 Subject: [PATCH] Add non-DAG bundle deploys Co-Authored-By: Claude --- .../workflows/e2e/get_bundle_info/action.yaml | 23 +- .../e2e/get_deployment_bundle/action.yaml | 82 ++++ .github/workflows/tests.yaml | 456 +++++++++++++++++- README.md | 53 +- action.yaml | 200 ++++++-- .../deployment-bundle.yaml | 27 ++ e2e-setup/mocks/configurable-git.sh | 19 + e2e-setup/mocks/dbt-deploy-git.sh | 2 + e2e-setup/mocks/non-dags-deploy-git.sh | 15 + e2e-setup/non-dags/queries/example.sql | 4 + 10 files changed, 830 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/e2e/get_deployment_bundle/action.yaml create mode 100644 e2e-setup/deployment-templates/deployment-bundle.yaml create mode 100755 e2e-setup/mocks/configurable-git.sh create mode 100755 e2e-setup/mocks/non-dags-deploy-git.sh create mode 100644 e2e-setup/non-dags/queries/example.sql diff --git a/.github/workflows/e2e/get_bundle_info/action.yaml b/.github/workflows/e2e/get_bundle_info/action.yaml index 5c2e3ff..92351cc 100644 --- a/.github/workflows/e2e/get_bundle_info/action.yaml +++ b/.github/workflows/e2e/get_bundle_info/action.yaml @@ -16,6 +16,9 @@ inputs: expected_status_code: description: The expected status code default: 200 + bundle_type: + description: The bundle type to look for among the deployment's bundle deploys + default: dbt outputs: desired_bundle_version: @@ -45,12 +48,22 @@ runs: exit 1 fi - # sort by updatedAt to fetch the latest deploy object - cat response.json | jq '.deploys | sort_by(.updatedAt)' > response_sorted.json + # the latest bundle deploy carrying a bundle of this type; a deploy can list several bundles, + # so match any of them rather than assuming it comes first + latest=$(jq -c --arg bundle_type "${{ inputs.bundle_type }}" ' + [.deploys[] | select(.type == "BUNDLE") + | {updatedAt, bundle: ([.bundles[]? | select(.bundleType == $bundle_type)] | first)} + | select(.bundle != null)] + | sort_by(.updatedAt) | last // empty' response.json) + echo "Latest ${{ inputs.bundle_type }} bundle deploy: ${latest:-none}" + if [[ -z "$latest" ]]; then + echo "Bundle deploys on the deployment:" + jq -c '[.deploys[] | select(.type == "BUNDLE") | {updatedAt, bundles: [.bundles[]? | {bundleType, desiredVersion}]}]' response.json + fi - desired_bundle_version=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].desiredVersion' | head -n 1) - bundle_type=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].bundleType' | head -n 1) - updated_at=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .updatedAt' | head -n 1) + desired_bundle_version=$(jq -r '.bundle.desiredVersion // ""' <<< "${latest:-null}") + bundle_type=$(jq -r '.bundle.bundleType // ""' <<< "${latest:-null}") + updated_at=$(jq -r '.updatedAt // ""' <<< "${latest:-null}") echo "desired_bundle_version=$desired_bundle_version" >> $GITHUB_OUTPUT echo "bundle_type=$bundle_type" >> $GITHUB_OUTPUT diff --git a/.github/workflows/e2e/get_deployment_bundle/action.yaml b/.github/workflows/e2e/get_deployment_bundle/action.yaml new file mode 100644 index 0000000..560e5c5 --- /dev/null +++ b/.github/workflows/e2e/get_deployment_bundle/action.yaml @@ -0,0 +1,82 @@ +name: Get Deployment Bundle +description: > + Look up a single bundle registered on a Deployment, by DAG bundle name or by + non-DAG mount path, and report its versions. Unlike get_bundle_info (which + reads the deploy history) this reads the bundle itself, so it can also report + that a bundle does not exist — which is what the skip-path tests assert. +inputs: + deployment_id: + description: The deployment ID + required: true + organization_id: + description: The organization ID + required: true + astro_api_token: + description: The Astronomer API token + required: true + astronomer_host: + description: The Astronomer host + required: true + bundle_name: + description: Name of the DAG bundle to look up. Mutually exclusive with mount_path. + default: "" + mount_path: + description: Mount path of the non-DAG bundle to look up. Mutually exclusive with bundle_name. + default: "" + +outputs: + found: + description: Whether a matching bundle exists on the Deployment + value: ${{ steps.get-deployment-bundle.outputs.found }} + bundle_id: + description: The ID of the matching bundle, empty when not found + value: ${{ steps.get-deployment-bundle.outputs.bundle_id }} + current_version: + description: The version the Deployment is currently serving, empty when not found + value: ${{ steps.get-deployment-bundle.outputs.current_version }} + desired_version: + description: The version the Deployment has been asked to serve, empty when not found + value: ${{ steps.get-deployment-bundle.outputs.desired_version }} + +runs: + using: "composite" + steps: + - name: Get Deployment Bundle + id: get-deployment-bundle + shell: bash + env: + BUNDLE_NAME: ${{ inputs.bundle_name }} + MOUNT_PATH: ${{ inputs.mount_path }} + run: | + if [[ -n "$BUNDLE_NAME" && -n "$MOUNT_PATH" ]] || [[ -z "$BUNDLE_NAME" && -z "$MOUNT_PATH" ]]; then + echo "Specify exactly one of bundle_name or mount_path" + exit 1 + fi + + STATUS_CODE=$(curl -s -w "%{http_code}" -o response.json -H "Authorization: Bearer ${{ inputs.astro_api_token }}" "https://api.${{ inputs.astronomer_host }}/v1alpha1/organizations/${{ inputs.organization_id }}/deployments/${{ inputs.deployment_id }}/bundles") + if [[ $STATUS_CODE -ne 200 ]]; then + echo "Failed to get expected status code from List Bundles API. Status code: $STATUS_CODE" + cat response.json + exit 1 + fi + + if [[ -n "$BUNDLE_NAME" ]]; then + bundle=$(jq -c --arg name "$BUNDLE_NAME" '.bundles[] | select(.isDagBundle == true and .name == $name)' response.json | head -n 1) + else + bundle=$(jq -c --arg path "$MOUNT_PATH" '.bundles[] | select(.isDagBundle != true and .nonDagMountPath == $path)' response.json | head -n 1) + fi + + if [[ -z "$bundle" ]]; then + echo "No matching bundle found on deployment ${{ inputs.deployment_id }}" + echo "found=false" >> $GITHUB_OUTPUT + echo "bundle_id=" >> $GITHUB_OUTPUT + echo "current_version=" >> $GITHUB_OUTPUT + echo "desired_version=" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Matched bundle: $bundle" + echo "found=true" >> $GITHUB_OUTPUT + echo "bundle_id=$(echo "$bundle" | jq -r '.id')" >> $GITHUB_OUTPUT + echo "current_version=$(echo "$bundle" | jq -r '.currentVersion // ""')" >> $GITHUB_OUTPUT + echo "desired_version=$(echo "$bundle" | jq -r '.desiredVersion // ""')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e937ac2..be10d52 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -51,7 +51,7 @@ jobs: # a: default (dags) -> infer (dags) -> custom image # b: default (image-and-dags) -> infer (image-and-dags) -> image only # c: pytest -> dag deploy -> no deploy -> dbt (no deploy asserts no dbt bundle yet, so it precedes dbt) - # once per kind (standard, hibernate), plus one deployment each for the preview and remote tests. + # once per kind (standard, hibernate), plus one deployment each for the preview, bundle and remote tests. # A job later in a chain waits on the earlier one only to take turns, not on its result. # DEPLOYMENTS maps each slot name to its deployment ID; test jobs look theirs up by slot. create-test-deployments: @@ -115,7 +115,7 @@ jobs: slots+=("$slot:$template") done done - slots+=(remote:deployment-remote.yaml) + slots+=(bundle:deployment-bundle.yaml remote:deployment-remote.yaml) # short random id keeps names unique across runs while staying within the name length limit test_uid=$(uuidgen | cut -c1-8) @@ -997,6 +997,379 @@ jobs: echo "DBT Deploy Action validation failed: DBT bundle was not updated" exit 1 + non-dags-deploy-test: + name: Non-DAG Bundle Deploy Test + # wake-on-deploy against a bundle deploy is covered by dbt-deploy-test. + runs-on: ubuntu-latest + needs: [create-test-deployments] + env: + E2E_DEPLOYMENT_ID: ${{ fromJSON(needs.create-test-deployments.outputs.DEPLOYMENTS)['bundle'] }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Patch deploy-action for manual dispatch + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Mock git commands + run: | + mv e2e-setup/mocks/non-dags-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + - name: Install Astro CLI + uses: astronomer/setup-astro-cli@v0.0.1 + with: + version: ${{ github.event.inputs.cli_version }} + + - name: Install dependencies + id: setup + run: | + + sudo apt-get install jq + + test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") + echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Non-DAG Bundle Deploy to Astro + uses: ./ + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-non-dags + non-dags-bundle-type: e2e-non-dags + description: "test-non-dags-deploy-action" + wait-time: 15m + + - name: Get non-DAG bundle info + id: get-non-dags-bundle-info + uses: ./.github/workflows/e2e/get_bundle_info + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + bundle_type: e2e-non-dags + + - name: Get non-DAG bundle by mount path + id: get-non-dags-bundle + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-non-dags + + - name: Validate Non-DAG Bundle Deploy Action + id: validate-non-dags-deploy-action + shell: bash + run: | + if [[ "${{ steps.get-non-dags-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then + echo "Non-DAG Bundle Deploy Action validation failed: bundle not found" + exit 1 + fi + + # the bundle must exist at the requested mount path, which is what DAGs reference at runtime + if [[ "${{ steps.get-non-dags-bundle.outputs.found }}" != "true" ]]; then + echo "Non-DAG Bundle Deploy Action validation failed: no bundle mounted at /usr/local/airflow/e2e-non-dags" + exit 1 + fi + + updated_at=$(date -u -d "${{ steps.get-non-dags-bundle-info.outputs.updated_at }}" +"%s.%N") + test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") + if (( $(echo "$updated_at > $test_start_time" | bc -l) )); then + echo "Non-DAG Bundle Deploy Action validation succeeded: bundle found" + exit 0 + fi + + echo "Non-DAG Bundle Deploy Action validation failed: bundle was not updated" + exit 1 + + # Asserts the skip side of the bundle deploy decision: no change under root-folder means no deploy, + # unless skip-unchanged is off, a bundle-trigger-path matched, or the diff cannot be computed. + # The git mock reports README.md. + bundle-skip-test: + name: Bundle Deploy Skip Test + runs-on: ubuntu-latest + needs: [non-dags-deploy-test, create-test-deployments] + if: ${{ !cancelled() && needs.create-test-deployments.result == 'success' }} + env: + E2E_CHANGED_FILES: README.md + E2E_DEPLOYMENT_ID: ${{ fromJSON(needs.create-test-deployments.outputs.DEPLOYMENTS)['bundle'] }} + MOUNT_PATH: /usr/local/airflow/e2e-skip + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Patch deploy-action for manual dispatch + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Mock git commands + run: | + mv e2e-setup/mocks/configurable-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + - name: Install Astro CLI + uses: astronomer/setup-astro-cli@v0.0.1 + with: + version: ${{ github.event.inputs.cli_version }} + + - name: Install dependencies + run: sudo apt-get install jq + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + # nothing under root-folder changed, so this must not deploy + - name: Deploy with no relevant changes + uses: ./ + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + description: "test-bundle-skip-action" + + - name: Get bundle after skipped deploy + id: get-bundle-skipped + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Validate the deploy was skipped + shell: bash + run: | + if [[ "${{ steps.get-bundle-skipped.outputs.found }}" == "true" ]]; then + echo "Bundle skip validation failed: a bundle was deployed at $MOUNT_PATH despite no relevant changes" + exit 1 + fi + echo "Bundle skip validation succeeded: no bundle deployed" + + # the deploy-action checks the repo out again, restoring the unpatched action.yaml + - name: Re-patch deploy-action before the next invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + # same unchanged tree, but a matching bundle-trigger-path must force the deploy + - name: Deploy via bundle-trigger-paths + uses: ./ + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + # comma- and newline-separated with stray whitespace, so the match also proves the list parsing + bundle-trigger-paths: | + docs/ , README.md + CHANGELOG.md + description: "test-bundle-trigger-paths-action" + wait-time: 15m + + - name: Get bundle after trigger-path deploy + id: get-bundle-triggered + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Validate bundle-trigger-paths forced the deploy + shell: bash + run: | + if [[ "${{ steps.get-bundle-triggered.outputs.found }}" != "true" ]]; then + echo "bundle-trigger-paths validation failed: no bundle deployed at $MOUNT_PATH" + exit 1 + fi + echo "bundle-trigger-paths validation succeeded: version ${{ steps.get-bundle-triggered.outputs.desired_version }}" + + - name: Re-patch deploy-action before the final invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + # same unchanged tree again, this time skip-unchanged=false must force a fresh deploy + - name: Deploy with skip-unchanged disabled + uses: ./ + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + skip-unchanged: false + description: "test-bundle-skip-unchanged-action" + wait-time: 15m + + - name: Get bundle after skip-unchanged deploy + id: get-bundle-forced + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Validate skip-unchanged=false forced the deploy + shell: bash + run: | + before="${{ steps.get-bundle-triggered.outputs.desired_version }}" + after="${{ steps.get-bundle-forced.outputs.desired_version }}" + echo "desired version before: '$before', after: '$after'" + + if [[ "$after" == "" || "$after" == "$before" ]]; then + echo "skip-unchanged validation failed: bundle was not redeployed" + exit 1 + fi + echo "skip-unchanged validation succeeded" + + # The remaining cases cannot tell what changed, so each must deploy despite the unchanged tree: + # an all-zero before (a push that creates a branch), no before/after at all (workflow_dispatch), + # and a diff base missing from the checkout (e.g. a force push). + - name: Re-patch deploy-action before the new-branch invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Deploy on a new branch push + uses: ./ + env: + DEPLOY_ACTION_DIFF_BEFORE: "0000000000000000000000000000000000000000" + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + wait-time: 15m + + - name: Get bundle after new-branch deploy + id: get-bundle-new-branch + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Re-patch deploy-action before the dispatch invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Deploy on a workflow dispatch + uses: ./ + env: + DEPLOY_ACTION_DIFF_BEFORE: "" + DEPLOY_ACTION_DIFF_AFTER: "" + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + wait-time: 15m + + - name: Get bundle after dispatch deploy + id: get-bundle-dispatch + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Re-patch deploy-action before the missing-diff-base invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Deploy with the diff base missing + uses: ./ + env: + E2E_MISSING_DIFF_BASE: "true" + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + root-folder: e2e-setup/non-dags + deploy-type: non-dags + mount-path: /usr/local/airflow/e2e-skip + non-dags-bundle-type: e2e-skip + wait-time: 15m + + - name: Get bundle after missing-diff-base deploy + id: get-bundle-missing-base + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-skip + + - name: Validate each undiffable run deployed + shell: bash + run: | + versions=( + "skip-unchanged=false:${{ steps.get-bundle-forced.outputs.desired_version }}" + "new branch:${{ steps.get-bundle-new-branch.outputs.desired_version }}" + "workflow dispatch:${{ steps.get-bundle-dispatch.outputs.desired_version }}" + "missing diff base:${{ steps.get-bundle-missing-base.outputs.desired_version }}" + ) + previous="" + for entry in "${versions[@]}"; do + name="${entry%%:*}" + version="${entry#*:}" + echo "$name: '$version'" + if [[ -n "$previous" && ( -z "$version" || "$version" == "$previous" ) ]]; then + echo "Fallback validation failed: the $name run did not redeploy the bundle" + exit 1 + fi + previous="$version" + done + echo "Fallback validation succeeded" + # Asserts the action rejects invalid input combinations before it touches Astro. A stub astro on PATH # records every call, so a rejection only counts if it came before anything beyond `astro version`; # an action that got further and then failed for an unrelated reason would otherwise pass. The @@ -1045,6 +1418,33 @@ jobs: deploy_type: dags-only image_name: custom-image:latest accepted: true + - case: "unknown deploy-type" + deploy_type: bundles + - case: "non-dags without mount-path" + deploy_type: non-dags + root_folder: e2e-setup/non-dags + - case: "non-dags without root-folder" + deploy_type: non-dags + mount_path: /usr/local/airflow/e2e-invalid + - case: "non-dags-bundle-type without non-dags" + deploy_type: dbt + non_dags_bundle_type: dbt + root_folder: e2e-setup/dbt + - case: "image-name with non-dags" + deploy_type: non-dags + mount_path: /usr/local/airflow/e2e-invalid + root_folder: e2e-setup/non-dags + image_name: custom-image:latest + - case: "parse with non-dags" + deploy_type: non-dags + mount_path: /usr/local/airflow/e2e-invalid + root_folder: e2e-setup/non-dags + parse: true + - case: "control: non-dags with root-folder and mount-path" + deploy_type: non-dags + mount_path: /usr/local/airflow/e2e-valid + root_folder: e2e-setup/non-dags + accepted: true env: E2E_STUB_CLI_VERSION: ${{ matrix.cli_version || '1.45.0' }} steps: @@ -1083,6 +1483,8 @@ jobs: parse: ${{ matrix.parse || 'false' }} pytest: ${{ matrix.pytest || 'false' }} build-secrets: ${{ matrix.build_secrets }} + mount-path: ${{ matrix.mount_path }} + non-dags-bundle-type: ${{ matrix.non_dags_bundle_type }} - name: Validate the action's verdict on the configuration shell: bash @@ -1523,6 +1925,49 @@ jobs: echo "DBT Deploy Action validation failed: DBT bundle was not updated" exit 1 + - name: Mock git commands for non-DAG bundle deploy + run: | + mv e2e-setup/mocks/non-dags-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + - name: Re-patch deploy-action for non-DAG bundle deploy + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Non-DAG Bundle Deploy to Deployment Preview + id: deployment-preview-non-dags-deploy + uses: ./ + with: + deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + action: deploy-deployment-preview + deploy-type: non-dags + root-folder: e2e-setup/non-dags + mount-path: /usr/local/airflow/e2e-non-dags + non-dags-bundle-type: e2e-non-dags + description: "test-non-dags-deploy-action" + preview-name: test-preview-${{ env.E2E_DEPLOYMENT_ID }} + wake-on-deploy: ${{ matrix.deployment == 'hibernate' }} + wait-time: 15m + + - name: Get non-DAG bundle on the preview + id: get-preview-non-dags-bundle + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + mount_path: /usr/local/airflow/e2e-non-dags + + - name: Validate Non-DAG Bundle Deploy to Preview + shell: bash + run: | + if [[ "${{ steps.get-preview-non-dags-bundle.outputs.found }}" != "true" ]]; then + echo "Non-DAG Bundle preview validation failed: no bundle mounted on the preview deployment" + exit 1 + fi + echo "Non-DAG Bundle preview validation succeeded: version ${{ steps.get-preview-non-dags-bundle.outputs.desired_version }}" + - name: Re-patch deploy-action for manual dispatch if: always() uses: ./.github/workflows/e2e/patch_action_for_dispatch @@ -1537,6 +1982,11 @@ jobs: workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} action: delete-deployment-preview preview-name: test-preview-${{ env.E2E_DEPLOYMENT_ID }} + # a bundle deploy type with a changed bundle in the mocked diff: only the delete guard keeps + # the action from deploying to the preview it just deleted + deploy-type: non-dags + root-folder: e2e-setup/non-dags + mount-path: /usr/local/airflow/e2e-non-dags - name: Validate Deployment Preview Deletion id: validate-deployment-preview-deletion @@ -1564,6 +2014,8 @@ jobs: no-deploy-test, custom-docker-image-test, dbt-deploy-test, + non-dags-deploy-test, + bundle-skip-test, deployment-preview-test, remote-deploy-test, remote-custom-image-test, diff --git a/README.md b/README.md index 6ee6c56..259ac59 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,14 @@ The following table lists the configuration options for the Deploy to Astro acti | Name | Default | Description | | ---|---|--- | | `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. | -| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags`, `image-only` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) | -| `image-trigger-paths` | `` | Comma- or newline-separated list of paths outside `root-folder` that should also trigger a deploy when changed. Useful in monorepos where files built or copied into the image at deploy time (e.g. generated protos) live outside the Astro project. A change under any of these paths forces an image deploy instead of being skipped. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `proto/`). Only applies to the `infer`, `image-and-dags`, and `image-only` deploy types. | -| `skip-unchanged` | `true` | When `true` (default), the action skips the deploy if no changed files fall under `root-folder` (or `image-trigger-paths`). Set to `false` to disable this diff-based skip and always run the deploy dictated by `deploy-type`. This is the closest replacement for the deprecated `deploy-image: true` behavior. Only applies to the `infer`, `dags-only`, `image-and-dags`, and `image-only` deploy types. | +| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs, a DBT project, or a non-DAG bundle. Possible options are `infer`, `dags-only`, `image-and-dags`, `image-only`, `dbt` or `non-dags`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) | +| `image-trigger-paths` | `` | Comma- or newline-separated list of paths outside `root-folder` that should also trigger a deploy when changed. Useful in monorepos where files built or copied into the image at deploy time (e.g. generated protos) live outside the Astro project. A change under any of these paths forces an image deploy instead of being skipped. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `proto/`). Only applies to the `infer`, `image-and-dags`, and `image-only` deploy types. For `dbt` and `non-dags` deploys, use `bundle-trigger-paths`. | +| `bundle-trigger-paths` | `` | Comma- or newline-separated list of paths outside `root-folder` that should also trigger a bundle deploy when changed. Useful in monorepos where a bundle depends on shared files that live outside its own directory. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `shared/macros/`). Only applies to the `dbt` and `non-dags` deploy types. | +| `skip-unchanged` | `true` | When `true` (default), the action skips the deploy if no changed files fall under `root-folder` (or `image-trigger-paths` / `bundle-trigger-paths`). Set to `false` to disable this diff-based skip and always run the deploy dictated by `deploy-type`. This is the closest replacement for the deprecated `deploy-image: true` behavior. | | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. | | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` | | `description` | | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. | -| `root-folder` | `.` | Path to the Astro project, or dbt project for dbt deploys. | +| `root-folder` | `.` | Path to the Astro project, or to the bundle directory for `dbt` and `non-dags` deploys. Required with `deploy-type: non-dags`. | | `parse` | `false` | When set to `true`, DAGs are parsed for errors before deploying to Astro. Note that when an image deploy is performed (i.e. `astro deploy`), parsing is also executed by default. Parsing is _not_ performed automatically for DAG-only deploys (i.e. `astro deploy --dags`). | | `pytest` | `false` | When set to `true`, all pytests in the `tests` directory of your Astro project are run before deploying to Astro. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest) | | `pytest-file` | (all tests run) | Specifies a custom pytest file to run with the pytest command. For example, you could specify `/tests/test-tags.py`.| @@ -67,7 +68,8 @@ The following table lists the configuration options for the Deploy to Astro acti | `checkout` | `true` | Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action. Your custom checkout step needs to have `fetch-depth` of `0` and `ref` equal to `${{ github.event.after }}` so all the commits in the PR are checked out. Look at the checkout step that runs within this action for reference. | | `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead. | | `build-secrets` | `` | Newline-delimited list of Docker build secrets, one `docker build --secret` specification per line (e.g. `id=mysecret,src=secrets.txt`). Each line is passed to `astro deploy` as its own `--build-secret` flag. See https://docs.docker.com/build/building/secrets/ for more information. | -| `mount-path` | `` | Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name} | +| `mount-path` | `` | Path to mount the bundle in Airflow, for reference by DAGs. Required with `deploy-type: non-dags`. With `deploy-type: dbt` it is optional and defaults to /usr/local/airflow/dbt/{dbt project name} | +| `non-dags-bundle-type` | `` | Free-form label identifying the kind of non-DAG bundle being deployed (e.g. `dbt`), shown in the Deploy History tab. Only applies to the `non-dags` deploy type. Requires Astro CLI 1.44.0 or higher. | | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. | | `sparse-checkout` | `` | Comma- or newline-separated list of cone-mode sparse-checkout patterns passed to `actions/checkout`. When set, only the listed paths (plus repo-root files) are checked out, which can dramatically reduce checkout size for large monorepos. Typically set to the same value as `root-folder`. Works only when `checkout` is set to `true`. Leave empty for a full checkout (default). | | `wake-on-deploy` | `false` | If true, the deployment will be woken up from hibernation before deploying. NOTE: This option overrides the deployment's hibernation override spec. | @@ -109,11 +111,15 @@ The following section describe each of the deploy type input value in detail to - if there are no file changes in the configured root-folder then it skips deploy - otherwise it would do a dbt deploy +6. `non-dags`: In this mode, deploy-action would run through all the file changes: + - if there are no file changes in the configured root-folder then it skips deploy + - otherwise it would deploy the root-folder directory as a non-DAG bundle, mounted at `mount-path` + ### Deploying on changes outside `root-folder` By default the deploy is skipped when no changed files fall under `root-folder`. In monorepos, some files that end up in the image are built or copied in at deploy time and live outside the Astro project (e.g. generated protos), so a change to them -would otherwise be skipped. Two inputs cover this: +would otherwise be skipped. Three inputs cover this: - `image-trigger-paths`: list the extra paths that feed the image. A change under any of them forces an image deploy while still skipping genuinely unrelated pushes. Applies to `infer`, `image-and-dags`, and `image-only`. @@ -127,6 +133,17 @@ would otherwise be skipped. Two inputs cover this: shared/schemas/ ``` +- `bundle-trigger-paths`: the same idea for bundle deploys. A change under any of them triggers a bundle deploy even when + nothing under `root-folder` changed. Applies to `dbt` and `non-dags`. + + ```yaml + with: + root-folder: dbt + deploy-type: dbt + bundle-trigger-paths: | + shared/macros/ + ``` + - `skip-unchanged: false`: disables the diff-based skip entirely, so the deploy dictated by `deploy-type` runs on every invocation. This is the closest replacement for the deprecated `deploy-image: true`. @@ -137,6 +154,30 @@ would otherwise be skipped. Two inputs cover this: skip-unchanged: false ``` +## Non-DAG bundle deploys + +A non-DAG bundle is any directory deployed to a Deployment and mounted at a path, for DAGs to reference at runtime — a dbt +project, a set of SQL files, a config tree. `deploy-type: dbt` is the older, dbt-specific form of this; `deploy-type: non-dags` +deploys any directory and requires Astro CLI 1.44.0 or higher. + +```yaml +- name: Deploy SQL assets to Astro + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: non-dags + root-folder: sql + mount-path: /usr/local/airflow/sql + non-dags-bundle-type: sql +``` + +`root-folder` is the directory to deploy, and is required. `mount-path` is also required, and is the path DAGs use to reach +the bundle's contents inside Airflow. `non-dags-bundle-type` is a free-form label shown in the Deploy History tab; it does +not change how the bundle is handled. + +> [!IMPORTANT] +> The bundle directory must not sit inside an Astro project. `astro-project/` and `sql/` as siblings works; `astro-project/sql/` +> is rejected by the Astro CLI. Keep the bundle a sibling of the Astro project in the repository. ## Installing the Astro CLI diff --git a/action.yaml b/action.yaml index 632f791..4bf500f 100644 --- a/action.yaml +++ b/action.yaml @@ -8,7 +8,7 @@ inputs: root-folder: required: false default: "" - description: "Path to the Astro project, or dbt project for dbt deploys." + description: "Path to the Astro project, or to the bundle directory for `dbt` and `non-dags` deploys. Required with `deploy-type: non-dags`." parse: required: false default: false @@ -82,7 +82,7 @@ inputs: deploy-type: required: false default: "infer" - description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', 'image-only', or 'dbt'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files. 'image-only' deploys the image without updating DAGs." + description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image, a DBT project, or a non-DAG bundle. Specify either 'infer', 'dags-only', 'image-and-dags', 'image-only', 'dbt', or 'non-dags'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files. 'image-only' deploys the image without updating DAGs. 'non-dags' deploys the 'root-folder' directory as a non-DAG bundle mounted at 'mount-path', and requires Astro CLI 1.44.0 or higher." image-trigger-paths: required: false default: "" @@ -91,15 +91,15 @@ inputs: Useful in monorepos where files built or copied into the image at deploy time (e.g. generated protos) live outside the Astro project. A change under any of these paths forces an image deploy instead of being skipped. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory - (e.g. `proto/`). Only applies to the `infer`, `image-and-dags`, and `image-only` deploy types. + (e.g. `proto/`). Only applies to the `infer`, `image-and-dags`, and `image-only` deploy types. For `dbt` and + `non-dags` deploys, use `bundle-trigger-paths` instead. skip-unchanged: required: false default: true description: > - When true (default), the action skips the deploy if no changed files fall under `root-folder` (or `image-trigger-paths`). - Set to false to disable this diff-based skip and always run the deploy dictated by `deploy-type` on every run. This is - the closest replacement for the deprecated `deploy-image: true` behavior. Only applies to the `infer`, `dags-only`, - `image-and-dags`, and `image-only` deploy types. + When true (default), the action skips the deploy if no changed files fall under `root-folder` (or + `image-trigger-paths` / `bundle-trigger-paths`). Set to false to disable this diff-based skip and always run the deploy dictated by + `deploy-type` on every run. This is the closest replacement for the deprecated `deploy-image: true` behavior. build-secrets: required: false description: > @@ -109,7 +109,22 @@ inputs: mount-path: required: false default: "" - description: "Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name}" + description: > + Path to mount the bundle in Airflow, for reference by DAGs. Required with `deploy-type: non-dags`. With + `deploy-type: dbt` it is optional and defaults to /usr/local/airflow/dbt/{dbt project name}. + non-dags-bundle-type: + required: false + default: "" + description: > + Free-form label identifying the kind of non-DAG bundle being deployed (e.g. `dbt`), shown in the Deploy History + tab. Only applies to the `non-dags` deploy type. Requires Astro CLI 1.44.0 or higher. + bundle-trigger-paths: + required: false + default: "" + description: > + Comma- or newline-separated list of paths outside `root-folder` that should also trigger a bundle deploy when changed. Useful in monorepos where a bundle depends on shared files that live outside its own directory. Paths are + matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory + (e.g. `shared/macros/`). Only applies to the `dbt` and `non-dags` deploy types. checkout-submodules: required: false default: false @@ -197,6 +212,15 @@ runs: fi fi + # Check if the Astro CLI version is less than 1.44.0 for non-DAG bundle deploys + if [[ "${{ inputs.deploy-type }}" == "non-dags" && $CLI_VERSION != "" ]]; then + REQUIRED_VERSION="1.44.0" + if ! semver -r ">=${REQUIRED_VERSION}" "${CLI_VERSION}"; then + echo "Non-DAG bundle deploys require Astro CLI version $REQUIRED_VERSION or higher" + exit 1 + fi + fi + # skip Astro CLI installation if already installed if command -v astro &> /dev/null; then echo "Astro CLI is already installed" @@ -238,6 +262,15 @@ runs: # inputs that only shape a deploy aren't validated for a delete, which deploys nothing, so a delete # step that shares a deploy step's inputs still removes the preview if [[ ${{ inputs.action }} != delete-deployment-preview ]]; then + # validate deploy-type input + case "${{ inputs.deploy-type }}" in + infer|dags-only|image-and-dags|image-only|dbt|non-dags) ;; + *) + echo "ERROR: you specified an improper deploy-type input. Deploy type must be infer, dags-only, image-and-dags, image-only, dbt, or non-dags." + exit 1 # terminate and indicate error + ;; + esac + # validate deploy-type-specific inputs # a custom image is deployed as it was built, so there is nothing to test or build, and dags-only never # deploys an image; the CLI rejects the equivalent flags alongside --image-name @@ -251,23 +284,50 @@ runs: echo "ERROR: parse and pytest cannot be used with image-name" exit 1 # terminate and indicate error fi - if [[ -n "$BUILD_SECRETS" && "${{ inputs.deploy-type }}" != "dbt" ]]; then + if [[ -n "$BUILD_SECRETS" && "${{ inputs.deploy-type }}" != "dbt" && "${{ inputs.deploy-type }}" != "non-dags" ]]; then echo "ERROR: build-secrets cannot be used with image-name" exit 1 # terminate and indicate error fi fi - # image-trigger-paths only affects image deploys, so it's a no-op for dags-only and dbt - if [[ -n "${{ inputs.image-trigger-paths }}" && ( "${{ inputs.deploy-type }}" == "dags-only" || "${{ inputs.deploy-type }}" == "dbt" ) ]]; then + # image-trigger-paths only affects image deploys, so it's a no-op for the DAG and bundle deploy types + if [[ -n "${{ inputs.image-trigger-paths }}" && ( "${{ inputs.deploy-type }}" == "dags-only" || "${{ inputs.deploy-type }}" == "dbt" || "${{ inputs.deploy-type }}" == "non-dags" ) ]]; then echo "ERROR: image-trigger-paths is only supported with deploy-type infer, image-and-dags, or image-only" exit 1 # terminate and indicate error fi - # skip-unchanged gates the image/dags diff skip, which the dbt deploy path doesn't use - if [[ "${{ inputs.skip-unchanged }}" == "false" && "${{ inputs.deploy-type }}" == "dbt" ]]; then - echo "ERROR: skip-unchanged is only supported with deploy-type infer, dags-only, image-and-dags, or image-only" + # bundle-trigger-paths is the bundle-deploy equivalent of image-trigger-paths + if [[ -n "$BUNDLE_TRIGGER_PATHS" && "${{ inputs.deploy-type }}" != "dbt" && "${{ inputs.deploy-type }}" != "non-dags" ]]; then + echo "ERROR: bundle-trigger-paths is only supported with deploy-type dbt or non-dags" + exit 1 # terminate and indicate error + fi + + # non-DAG bundles are mounted at a path, which the CLI cannot infer + if [[ "${{ inputs.deploy-type }}" == "non-dags" && -z "$MOUNT_PATH" ]]; then + echo "ERROR: mount-path is required with deploy-type non-dags" + exit 1 # terminate and indicate error + fi + # mount-path and non-dags-bundle-type only mean anything for a bundle deploy; a stray mount-path is + # tolerated so workflows passing it to every invocation keep working + if [[ -n "$MOUNT_PATH" && "${{ inputs.deploy-type }}" != "dbt" && "${{ inputs.deploy-type }}" != "non-dags" ]]; then + echo "::warning::mount-path is ignored unless deploy-type is dbt or non-dags" + fi + if [[ -n "$NON_DAGS_BUNDLE_TYPE" && "${{ inputs.deploy-type }}" != "non-dags" ]]; then + echo "ERROR: non-dags-bundle-type is only supported with deploy-type non-dags" + exit 1 # terminate and indicate error + fi + # a non-DAG bundle deploy builds and tests nothing; the CLI rejects the equivalent flags alongside --non-dags + if [[ "${{ inputs.deploy-type }}" == "non-dags" && ( "${{ inputs.image-name }}" != "no-custom-image" || "${{ inputs.parse }}" == "true" || "${{ inputs.pytest }}" == "true" || -n "$BUILD_SECRETS" ) ]]; then + echo "ERROR: image-name, parse, pytest, and build-secrets are not supported with deploy-type non-dags" + exit 1 # terminate and indicate error + fi + + # a non-DAG bundle is deployed from root-folder; left empty, the whole repository would be deployed as the bundle + if [[ "${{ inputs.deploy-type }}" == "non-dags" && -z "$ROOT_FOLDER" ]]; then + echo "ERROR: root-folder is required with deploy-type non-dags" exit 1 # terminate and indicate error fi fi + # Select workspace if specified if [[ "${{ inputs.workspace }}" != "" ]]; then astro workspace switch ${{ inputs.workspace }} @@ -404,6 +464,10 @@ runs: id: deployment-preview env: BUILD_SECRETS: ${{ inputs.build-secrets }} + ROOT_FOLDER: ${{ inputs.root-folder }} + BUNDLE_TRIGGER_PATHS: ${{ inputs.bundle-trigger-paths }} + MOUNT_PATH: ${{ inputs.mount-path }} + NON_DAGS_BUNDLE_TYPE: ${{ inputs.non-dags-bundle-type }} - name: Determine if DAG Deploy is enabled run: | @@ -417,43 +481,80 @@ runs: echo ::endgroup:: shell: bash id: dag-deploy-enabled - - name: Get DBT Deploy Options - if: ${{ inputs.deploy-type == 'dbt' }} + - name: Get Bundle Deploy Options + if: ${{ inputs.deploy-type == 'dbt' || inputs.deploy-type == 'non-dags' }} run: | - echo ::group::Get DBT Deploy Options - - if [[ "${{ inputs.root-folder }}" != "" ]]; then - cd ${{ inputs.root-folder }} - fi + echo ::group::Get Bundle Deploy Options # the diff below relies on history already present from the fetch-depth: 0 checkout; # don't add a `git fetch` here, it fails on private repos due to persist-credentials: false - DBT_DEPLOY=false + BUNDLE_DEPLOY=false + files=() - # case when the triggered event is a manual workflow dispatch or a new branch or tag creation, we would need to deploy the dbt project because we cannot determine that it does not need to be deployed + # collect bundle-trigger-paths (comma- or newline-separated), trimmed and non-empty + BUNDLE_TRIGGERS=() + while IFS= read -r trigger; do + trigger="${trigger#"${trigger%%[![:space:]]*}"}" + trigger="${trigger%"${trigger##*[![:space:]]}"}" + [[ -n "$trigger" ]] && BUNDLE_TRIGGERS+=("$trigger") + done <<< "${BUNDLE_TRIGGER_PATHS//,/$'\n'}" + + # case when the triggered event is a manual workflow dispatch or a new branch or tag creation, we would need to deploy the bundle because we cannot determine that it does not need to be deployed GITHUB_EVENT_BEFORE=${{ github.event.before }} GITHUB_EVENT_AFTER=${{ github.event.after }} if [[ "$GITHUB_EVENT_BEFORE" == "0000000000000000000000000000000000000000" || -z $GITHUB_EVENT_BEFORE && -z $GITHUB_EVENT_AFTER ]]; then - DBT_DEPLOY=true - files=() + echo "Manual workflow dispatch or a new branch or tag creation, hence missing github event before and/or after commit hash" + BUNDLE_DEPLOY=true + elif ! git cat-file -e "$GITHUB_EVENT_BEFORE" 2>/dev/null; then + echo "Commit $GITHUB_EVENT_BEFORE does not exist, falling back to a bundle deploy." + BUNDLE_DEPLOY=true else files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER) echo "files changed: $files" fi for file in $files; do - if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then - echo $file is part of configured root folder, so would be triggering a dbt deploy - DBT_DEPLOY=true + if [[ $file =~ ^"$ROOT_FOLDER".* ]]; then + echo $file is part of configured root folder, so would be triggering a bundle deploy + BUNDLE_DEPLOY=true fi + + # changes under any configured bundle-trigger-path also trigger a deploy, even when the + # file lives outside the bundle directory (e.g. shared macros the bundle depends on) + for trigger in "${BUNDLE_TRIGGERS[@]}"; do + if [[ $file == "$trigger"* ]]; then + echo "$file matches bundle-trigger-path '$trigger', triggering a bundle deploy" + BUNDLE_DEPLOY=true + fi + done done + # skip-unchanged=false disables the diff-based skip entirely and always deploys the bundle + if [[ "${{ inputs.skip-unchanged }}" == "false" ]]; then + BUNDLE_DEPLOY=true + fi + + # a freshly created deployment preview carries no bundle yet, so it always needs one deployed + if [[ ${{ inputs.action }} == create-deployment-preview ]]; then + BUNDLE_DEPLOY=true + fi + + # there is nothing left to deploy to once the preview has been deleted + if [[ ${{ inputs.action }} == delete-deployment-preview ]]; then + BUNDLE_DEPLOY=false + fi + options="" - if [[ $DBT_DEPLOY == true ]]; then - # Add mount path option - if [[ "${{ inputs.mount-path }}" != "" ]]; then - options="$options --mount-path=${{ inputs.mount-path }}" + if [[ $BUNDLE_DEPLOY == true ]]; then + if [[ "${{ inputs.deploy-type }}" == "non-dags" ]]; then + options="$options --non-dags-mount-path $(printf '%q' "$MOUNT_PATH")" + if [[ -n "$NON_DAGS_BUNDLE_TYPE" ]]; then + options="$options --non-dags-bundle-type $(printf '%q' "$NON_DAGS_BUNDLE_TYPE")" + fi + elif [[ -n "$MOUNT_PATH" ]]; then + # Add mount path option + options="$options --mount-path=$(printf '%q' "$MOUNT_PATH")" fi # Add description option @@ -462,11 +563,16 @@ runs: fi fi - echo "DBT_DEPLOY=$DBT_DEPLOY" >> $GITHUB_OUTPUT - echo "DBT_OPTIONS=$options" >> $GITHUB_OUTPUT + echo "BUNDLE_DEPLOY=$BUNDLE_DEPLOY" >> $GITHUB_OUTPUT + echo "BUNDLE_OPTIONS=$options" >> $GITHUB_OUTPUT echo ::endgroup:: shell: bash - id: dbt-deploy-options + id: bundle-deploy-options + env: + ROOT_FOLDER: ${{ inputs.root-folder }} + BUNDLE_TRIGGER_PATHS: ${{ inputs.bundle-trigger-paths }} + MOUNT_PATH: ${{ inputs.mount-path }} + NON_DAGS_BUNDLE_TYPE: ${{ inputs.non-dags-bundle-type }} - name: Get Deploy Type if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' || inputs.deploy-type == 'image-only' }} run: | @@ -762,7 +868,7 @@ runs: echo ::endgroup:: shell: bash - name: DBT Deploy to Astro - if: ${{ inputs.deploy-type == 'dbt' && steps.dbt-deploy-options.outputs.DBT_DEPLOY == 'true' }} + if: ${{ inputs.deploy-type == 'dbt' && steps.bundle-deploy-options.outputs.BUNDLE_DEPLOY == 'true' }} run: | echo ::group::DBT Deploy to Astro @@ -772,9 +878,27 @@ runs: fi if [[ "${{ inputs.wait-time }}" != "" ]]; then - astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --wait ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} --wait-time "${{ inputs.wait-time }}" + astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --wait ${{steps.bundle-deploy-options.outputs.BUNDLE_OPTIONS}} --wait-time "${{ inputs.wait-time }}" + else + astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --wait ${{steps.bundle-deploy-options.outputs.BUNDLE_OPTIONS}} + fi + echo ::endgroup:: + shell: bash + - name: Non-DAG Bundle Deploy to Astro + if: ${{ inputs.deploy-type == 'non-dags' && steps.bundle-deploy-options.outputs.BUNDLE_DEPLOY == 'true' }} + run: | + + echo ::group::Non-DAG Bundle Deploy to Astro + # Deploy root-folder as a non-DAG bundle, mounted for reference by DAGs. + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + + if [[ "${{ inputs.wait-time }}" != "" ]]; then + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --non-dags --wait ${{steps.bundle-deploy-options.outputs.BUNDLE_OPTIONS}} --wait-time "${{ inputs.wait-time }}" else - astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --wait ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --non-dags --wait ${{steps.bundle-deploy-options.outputs.BUNDLE_OPTIONS}} fi echo ::endgroup:: shell: bash diff --git a/e2e-setup/deployment-templates/deployment-bundle.yaml b/e2e-setup/deployment-templates/deployment-bundle.yaml new file mode 100644 index 0000000..0dbffd4 --- /dev/null +++ b/e2e-setup/deployment-templates/deployment-bundle.yaml @@ -0,0 +1,27 @@ +deployment: + configuration: + name: deploy-action-non-dev-e2e + description: "" + runtime_version: 3.3-2 + dag_deploy_enabled: true + ci_cd_enforcement: false + scheduler_size: SMALL + is_high_availability: false + is_development_mode: false + executor: CELERY + scheduler_count: 1 + workspace_name: Deploy Action E2E + deployment_type: STANDARD + cloud_provider: AWS + region: us-west-2 + default_task_pod_cpu: "0.25" + default_task_pod_memory: 0.5Gi + resource_quota_cpu: "10" + resource_quota_memory: 20Gi + workload_identity: "" + worker_queues: + - name: default + max_worker_count: 10 + min_worker_count: 0 + worker_concurrency: 5 + worker_type: A5 diff --git a/e2e-setup/mocks/configurable-git.sh b/e2e-setup/mocks/configurable-git.sh new file mode 100755 index 0000000..742af26 --- /dev/null +++ b/e2e-setup/mocks/configurable-git.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml, driven by env vars so one job can run several scenarios: +# E2E_CHANGED_FILES newline-separated files `git diff` reports as changed +# E2E_MISSING_DIFF_BASE when "true", `git cat-file` reports the diff base commit as missing + +if [[ "$1" == "diff" ]]; then + printf '%s\n' "$E2E_CHANGED_FILES" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +elif [[ "$1" == "cat-file" ]]; then + if [[ "$E2E_MISSING_DIFF_BASE" == "true" ]]; then + exit 1 + fi + echo "Handling git cat-file, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi diff --git a/e2e-setup/mocks/dbt-deploy-git.sh b/e2e-setup/mocks/dbt-deploy-git.sh index a415600..a9677bb 100644 --- a/e2e-setup/mocks/dbt-deploy-git.sh +++ b/e2e-setup/mocks/dbt-deploy-git.sh @@ -7,6 +7,8 @@ if [[ "$1" == "diff" ]]; then echo "e2e-setup/dbt/dbt_project.yml" elif [[ "$1" == "fetch" ]]; then echo "Handling git fetch, doing nothing" +elif [[ "$1" == "cat-file" ]]; then + echo "Handling git cat-file, doing nothing" else echo "Error: git mock script isn't configured to handle $1" >&2 exit 1 diff --git a/e2e-setup/mocks/non-dags-deploy-git.sh b/e2e-setup/mocks/non-dags-deploy-git.sh new file mode 100755 index 0000000..8732f5c --- /dev/null +++ b/e2e-setup/mocks/non-dags-deploy-git.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate a non-DAG bundle deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "e2e-setup/non-dags/queries/example.sql" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +elif [[ "$1" == "cat-file" ]]; then + echo "Handling git cat-file, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi diff --git a/e2e-setup/non-dags/queries/example.sql b/e2e-setup/non-dags/queries/example.sql new file mode 100644 index 0000000..8b0a145 --- /dev/null +++ b/e2e-setup/non-dags/queries/example.sql @@ -0,0 +1,4 @@ +-- Stand-in content for the non-DAG bundle deployed by the e2e tests. +-- A non-DAG bundle is any directory mounted into Airflow for DAGs to read at runtime, +-- so the contents only need to exist, not to be valid for any particular engine. +SELECT 1 AS example;