diff --git a/.github/actions/discord-notify/action.yml b/.github/actions/discord-notify/action.yml new file mode 100644 index 0000000..e9a214f --- /dev/null +++ b/.github/actions/discord-notify/action.yml @@ -0,0 +1,60 @@ +name: Discord deploy notification +description: Post a green (healthy) or red (failed) deploy-status embed to Discord. + +inputs: + webhook-url: + description: Discord webhook URL. When empty, the step is a no-op. + required: true + name: + description: Deploy target name shown in the title (e.g. kaapi-staging). + required: true + release: + description: Release identifier (tag or run number). + required: true + sha: + description: Full commit SHA; truncated to 7 chars for display. + required: true + run-url: + description: Link to the workflow run. + required: true + ok: + description: "'true' for a healthy deploy; anything else renders as failed." + required: true + failure-reason: + description: Optional extra line explaining a failure (e.g. CI blocked the release). + required: false + default: "" + +runs: + using: composite + steps: + - shell: bash + env: + DISCORD_WEBHOOK_URL: ${{ inputs.webhook-url }} + NAME: ${{ inputs.name }} + RELEASE: ${{ inputs.release }} + SHA: ${{ inputs.sha }} + RUN_URL: ${{ inputs.run-url }} + OK: ${{ inputs.ok }} + REASON: ${{ inputs.failure-reason }} + run: | + [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; } + if [ "$OK" = "true" ]; then + TITLE="🟢 $NAME deployment healthy"; COLOR=3066993 # green + else + TITLE="🔴 $NAME deployment failed"; COLOR=15158332 # red + fi + SHA_SHORT=$(echo "$SHA" | cut -c1-7) + + fields=$(jq -n --arg release "$RELEASE" --arg sha "$SHA_SHORT" \ + '[{name:"Release",value:$release,inline:true},{name:"SHA",value:$sha,inline:true}]') + # Only a failed deploy with a stated cause gets the extra Reason line. + if [ "$OK" != "true" ] && [ -n "$REASON" ]; then + fields=$(echo "$fields" | jq --arg r "$REASON" '. + [{name:"Reason",value:$r,inline:false}]') + fi + + payload=$(jq -n --arg title "$TITLE" --argjson color "$COLOR" \ + --arg url "$RUN_URL" --argjson fields "$fields" \ + '{embeds:[{title:$title,url:$url,color:$color,fields:$fields,timestamp:(now|todate)}]}') + curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \ + || echo "Discord notification failed to send" diff --git a/.github/workflows/cd-production.yml b/.github/workflows/cd-production.yml index 417d940..1723b75 100644 --- a/.github/workflows/cd-production.yml +++ b/.github/workflows/cd-production.yml @@ -10,7 +10,32 @@ concurrency: cancel-in-progress: false jobs: + verify-ci: + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Confirm CI passed on the tagged commit + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SHA="${{ github.sha }}" + echo "Checking Kaapi Guardrails CI conclusion for ${SHA}" + CONCLUSION=$(gh api \ + "repos/${{ github.repository }}/actions/runs?head_sha=${SHA}" \ + --jq 'first(.workflow_runs[] + | select(.name == "Kaapi Guardrails CI" and .status == "completed") + | .conclusion) // "missing"') + echo "Kaapi Guardrails CI conclusion: ${CONCLUSION}" + if [ "${CONCLUSION}" != "success" ]; then + echo "::error::Kaapi Guardrails CI has not passed for ${SHA} (conclusion: ${CONCLUSION}). Aborting release." + exit 1 + fi + deploy: + needs: verify-ci runs-on: ubuntu-latest environment: AWS_ENV_SECRETS @@ -21,8 +46,6 @@ jobs: steps: - name: Checkout the repo uses: actions/checkout@v7 - with: - ref: ${{ github.event.release.tag_name }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 @@ -52,12 +75,13 @@ jobs: ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo IMAGE_TAG: ${{ github.ref_name }} run: | - DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f" + DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull --quiet backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f" CMD_ID=$(aws ssm send-command \ --instance-ids "$INSTANCE_ID" \ --document-name "AWS-RunShellScript" \ --comment "Deploy kaapi-guardrails production" \ + --cloud-watch-output-config CloudWatchOutputEnabled=true \ --parameters "commands=[\"set -eux\", \"sudo chown -R ec2-user:ec2-user ${BUILD_DIRECTORY}\", \"sudo -iu ec2-user bash -lc \\\"${DEPLOY_CMD}\\\"\"]" \ --query "Command.CommandId" \ --output text) @@ -66,42 +90,57 @@ jobs: echo "Sent SSM command: $CMD_ID" - name: Wait for SSM command to finish + timeout-minutes: 20 env: INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }} CMD_ID: ${{ steps.ssm.outputs.cmd_id }} run: | - get_output() { - aws ssm get-command-invocation \ + while true; do + if aws ssm wait command-executed \ --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - } + --instance-id "$INSTANCE_ID"; then + break # waiter succeeds only on Status == Success + fi - for i in {1..20}; do STATUS=$(aws ssm get-command-invocation \ --command-id "$CMD_ID" \ --instance-id "$INSTANCE_ID" \ - --query "Status" \ - --output text) - - echo "Current Status: $STATUS" - - if [ "$STATUS" = "Success" ]; then - echo "Deployment completed successfully." - get_output - exit 0 - fi - - if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then - echo "Deployment failed." - get_output - exit 1 - fi - - sleep 25 + --query "Status" --output text 2>/dev/null || echo "Pending") + + case "$STATUS" in + Success) break ;; + Failed|Cancelled|Cancelling|TimedOut) + echo "Deployment ended with status: $STATUS" + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + exit 1 ;; + *) echo "Still running (status: $STATUS) — waiter capped out, re-waiting" ;; + esac done - echo "Deployment timed out after waiting too long." - get_output - exit 1 + echo "Deployment completed successfully." + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + + notify: + needs: [verify-ci, deploy] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/discord-notify + with: + webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }} + name: kaapi-guardrails-production + release: ${{ github.ref_name }} + sha: ${{ github.sha }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ok: ${{ needs.deploy.result == 'success' }} + # Call out the specific case where the release was blocked by CI. + failure-reason: ${{ needs.verify-ci.result != 'success' && 'CI never passed on the tagged commit — release blocked.' || '' }} diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml deleted file mode 100644 index 19690d4..0000000 --- a/.github/workflows/cd-staging.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Deploy Kaapi Guardrails Staging To EC2 - -on: - push: - branches: [main] - -concurrency: - group: guardrail-staging-ec2-deploy - cancel-in-progress: false - -jobs: - deploy: - runs-on: ubuntu-latest - environment: AWS_ENV_SECRETS - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout the repo - uses: actions/checkout@v7 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ secrets.EC2_DEPLOY_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build and Push Docker Image - env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo - run: | - docker build -t $REGISTRY/$REPOSITORY:latest ./backend - docker push $REGISTRY/$REPOSITORY:latest - - - name: Trigger deploy on EC2 via SSM - id: ssm - env: - INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }} - BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }} - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo - IMAGE_TAG: latest - run: | - DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard origin/main && aws ecr get-login-password --region '"${{ secrets.AWS_REGION }}"' | docker login --username AWS --password-stdin '"${ECR_REGISTRY}"' && export ECR_REGISTRY='"${ECR_REGISTRY}"' ECR_REPOSITORY='"${ECR_REPOSITORY}"' IMAGE_TAG='"${IMAGE_TAG}"' && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f\"' - - CMD_ID=$(aws ssm send-command \ - --instance-ids "$INSTANCE_ID" \ - --document-name "AWS-RunShellScript" \ - --comment "Deploy kaapi-guardrails staging" \ - --parameters commands="[\"set -eux\",\"sudo chown -R ec2-user:ec2-user ${BUILD_DIRECTORY}\",\"${DEPLOY_CMD}\"]" \ - --query "Command.CommandId" \ - --output text) - - echo "cmd_id=$CMD_ID" >> "$GITHUB_OUTPUT" - echo "Sent SSM command: $CMD_ID" - - - name: Wait for SSM command to finish - env: - INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }} - CMD_ID: ${{ steps.ssm.outputs.cmd_id }} - run: | - get_output() { - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - } - - for i in {1..20}; do - STATUS=$(aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query "Status" \ - --output text) - - echo "Current Status: $STATUS" - - if [ "$STATUS" = "Success" ]; then - echo "Deployment completed successfully." - get_output - exit 0 - fi - - if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then - echo "Deployment failed." - get_output - exit 1 - fi - - sleep 25 - done - - echo "Deployment timed out after waiting too long." - get_output - exit 1 diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..949031b --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,132 @@ +name: Deploy Kaapi Guardrails Staging To EC2 + +on: + workflow_run: + workflows: ["Kaapi Guardrails CI"] + branches: [main] + types: [completed] + workflow_dispatch: + +concurrency: + group: guardrail-staging-ec2-deploy + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + environment: Staging + # workflow_dispatch runs unconditionally; workflow_run only on a green CI push to main. + if: >- + ${{ github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' }} + + permissions: + id-token: write + contents: read + + env: + DEPLOY_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + + steps: + - name: Checkout the repo + uses: actions/checkout@v7 + with: + ref: ${{ env.DEPLOY_SHA }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.DEPLOY_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and Push Docker Image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo + run: | + echo "Building commit: $DEPLOY_SHA" + docker build -t $REGISTRY/$REPOSITORY:latest ./backend + docker push $REGISTRY/$REPOSITORY:latest + + - name: Trigger deploy on EC2 via SSM + id: ssm + env: + INSTANCE_ID: ${{ secrets.INSTANCE_ID }} + BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo + IMAGE_TAG: latest + run: | + echo "Deploying commit: $DEPLOY_SHA" + + DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard '"${DEPLOY_SHA}"' && aws ecr get-login-password --region '"${{ secrets.AWS_REGION }}"' | docker login --username AWS --password-stdin '"${ECR_REGISTRY}"' && export ECR_REGISTRY='"${ECR_REGISTRY}"' ECR_REPOSITORY='"${ECR_REPOSITORY}"' IMAGE_TAG='"${IMAGE_TAG}"' && docker compose pull --quiet backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f\"' + + CMD_ID=$(aws ssm send-command \ + --instance-ids "$INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --comment "Deploy kaapi-guardrails staging" \ + --cloud-watch-output-config CloudWatchOutputEnabled=true \ + --parameters commands="[\"set -eux\",\"sudo chown -R ec2-user:ec2-user ${BUILD_DIRECTORY}\",\"${DEPLOY_CMD}\"]" \ + --query "Command.CommandId" \ + --output text) + + echo "cmd_id=$CMD_ID" >> "$GITHUB_OUTPUT" + echo "Sent SSM command: $CMD_ID" + + - name: Wait for SSM command to finish + timeout-minutes: 20 + env: + INSTANCE_ID: ${{ secrets.INSTANCE_ID }} + CMD_ID: ${{ steps.ssm.outputs.cmd_id }} + run: | + while true; do + if aws ssm wait command-executed \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID"; then + break # waiter succeeds only on Status == Success + fi + + STATUS=$(aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query "Status" --output text 2>/dev/null || echo "Pending") + + case "$STATUS" in + Success) break ;; + Failed|Cancelled|Cancelling|TimedOut) + echo "Deployment ended with status: $STATUS" + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + exit 1 ;; + *) echo "Still running (status: $STATUS) — waiter capped out, re-waiting" ;; + esac + done + + echo "Deployment completed successfully." + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + + notify: + needs: [deploy] + if: ${{ always() && needs.deploy.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/discord-notify + with: + webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }} + name: kaapi-guardrails-staging + release: "#${{ github.run_number }}" + sha: ${{ github.event.workflow_run.head_sha || github.sha }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ok: ${{ needs.deploy.result == 'success' }}