feat(infra): update the deployment script

.github/actions/discord-notify/action.yml

@@ -0,0 +1,60 @@
+name: Discord deploy notification
+description: Post a green (healthy) or red (failed) deploy-status embed to Discord.
+
+inputs:
+  webhook-url:
+    description: Discord webhook URL. When empty, the step is a no-op.
+    required: true
+  name:
+    description: Deploy target name shown in the title (e.g. kaapi-staging).
+    required: true
+  release:
+    description: Release identifier (tag or run number).
+    required: true
+  sha:
+    description: Full commit SHA; truncated to 7 chars for display.
+    required: true
+  run-url:
+    description: Link to the workflow run.
+    required: true
+  ok:
+    description: "'true' for a healthy deploy; anything else renders as failed."
+    required: true
+  failure-reason:
+    description: Optional extra line explaining a failure (e.g. CI blocked the release).
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        DISCORD_WEBHOOK_URL: ${{ inputs.webhook-url }}
+        NAME: ${{ inputs.name }}
+        RELEASE: ${{ inputs.release }}
+        SHA: ${{ inputs.sha }}
+        RUN_URL: ${{ inputs.run-url }}
+        OK: ${{ inputs.ok }}
+        REASON: ${{ inputs.failure-reason }}
+      run: |
+        [ -z "$DISCORD_WEBHOOK_URL" ] && { echo "No webhook configured, skipping"; exit 0; }
+        if [ "$OK" = "true" ]; then
+          TITLE="🟢 $NAME deployment healthy"; COLOR=3066993   # green
+        else
+          TITLE="🔴 $NAME deployment failed";  COLOR=15158332  # red
+        fi
+        SHA_SHORT=$(echo "$SHA" | cut -c1-7)
+
+        fields=$(jq -n --arg release "$RELEASE" --arg sha "$SHA_SHORT" \
+          '[{name:"Release",value:$release,inline:true},{name:"SHA",value:$sha,inline:true}]')
+        # Only a failed deploy with a stated cause gets the extra Reason line.
+        if [ "$OK" != "true" ] && [ -n "$REASON" ]; then
+          fields=$(echo "$fields" | jq --arg r "$REASON" '. + [{name:"Reason",value:$r,inline:false}]')
+        fi
+
+        payload=$(jq -n --arg title "$TITLE" --argjson color "$COLOR" \
+          --arg url "$RUN_URL" --argjson fields "$fields" \
+          '{embeds:[{title:$title,url:$url,color:$color,fields:$fields,timestamp:(now|todate)}]}')
+        curl -sf -H "Content-Type: application/json" -X POST -d "$payload" "$DISCORD_WEBHOOK_URL" \
+          || echo "Discord notification failed to send"

.github/workflows/cd-production.yml

@@ -10,7 +10,32 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  verify-ci:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Confirm CI passed on the tagged commit
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          SHA="${{ github.sha }}"
+          echo "Checking Kaapi Guardrails CI conclusion for ${SHA}"
+          CONCLUSION=$(gh api \
+            "repos/${{ github.repository }}/actions/runs?head_sha=${SHA}" \
+            --jq 'first(.workflow_runs[]
+                    | select(.name == "Kaapi Guardrails CI" and .status == "completed")
+                    | .conclusion) // "missing"')
+          echo "Kaapi Guardrails CI conclusion: ${CONCLUSION}"
+          if [ "${CONCLUSION}" != "success" ]; then
+            echo "::error::Kaapi Guardrails CI has not passed for ${SHA} (conclusion: ${CONCLUSION}). Aborting release."
+            exit 1
+          fi
+
   deploy:
+    needs: verify-ci
     runs-on: ubuntu-latest
     environment: AWS_ENV_SECRETS
 
@@ -21,8 +46,6 @@ jobs:
     steps:
       - name: Checkout the repo
         uses: actions/checkout@v7
-        with:
-          ref: ${{ github.event.release.tag_name }}
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
@@ -52,12 +75,13 @@ jobs:
           ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo
           IMAGE_TAG: ${{ github.ref_name }}
         run: |
-          DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f"
+          DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull --quiet backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f"
 
           CMD_ID=$(aws ssm send-command \
             --instance-ids "$INSTANCE_ID" \
             --document-name "AWS-RunShellScript" \
             --comment "Deploy kaapi-guardrails production" \
+            --cloud-watch-output-config CloudWatchOutputEnabled=true \
             --parameters "commands=[\"set -eux\", \"sudo chown -R ec2-user:ec2-user ${BUILD_DIRECTORY}\", \"sudo -iu ec2-user bash -lc \\\"${DEPLOY_CMD}\\\"\"]" \
             --query "Command.CommandId" \
             --output text)
@@ -66,42 +90,57 @@ jobs:
           echo "Sent SSM command: $CMD_ID"
 
       - name: Wait for SSM command to finish
+        timeout-minutes: 20
         env:
           INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }}
           CMD_ID: ${{ steps.ssm.outputs.cmd_id }}
         run: |
-          get_output() {
-            aws ssm get-command-invocation \
+          while true; do
+            if aws ssm wait command-executed \
               --command-id "$CMD_ID" \
-              --instance-id "$INSTANCE_ID" \
-              --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-              --output json
-          }
+              --instance-id "$INSTANCE_ID"; then
+              break   # waiter succeeds only on Status == Success
+            fi
 
-          for i in {1..20}; do
             STATUS=$(aws ssm get-command-invocation \
               --command-id "$CMD_ID" \
               --instance-id "$INSTANCE_ID" \
-              --query "Status" \
-              --output text)
-
-            echo "Current Status: $STATUS"
-
-            if [ "$STATUS" = "Success" ]; then
-              echo "Deployment completed successfully."
-              get_output
-              exit 0
-            fi
-
-            if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then
-              echo "Deployment failed."
-              get_output
-              exit 1
-            fi
-
-            sleep 25
+              --query "Status" --output text 2>/dev/null || echo "Pending")
+
+            case "$STATUS" in
+              Success) break ;;
+              Failed|Cancelled|Cancelling|TimedOut)
+                echo "Deployment ended with status: $STATUS"
+                aws ssm get-command-invocation \
+                  --command-id "$CMD_ID" \
+                  --instance-id "$INSTANCE_ID" \
+                  --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
+                  --output json
+                exit 1 ;;
+              *) echo "Still running (status: $STATUS) — waiter capped out, re-waiting" ;;
+            esac
           done
 
-          echo "Deployment timed out after waiting too long."
-          get_output
-          exit 1
+          echo "Deployment completed successfully."
+          aws ssm get-command-invocation \
+            --command-id "$CMD_ID" \
+            --instance-id "$INSTANCE_ID" \
+            --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
+            --output json
+
+  notify:
+    needs: [verify-ci, deploy]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ./.github/actions/discord-notify
+        with:
+          webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          name: kaapi-guardrails-production
+          release: ${{ github.ref_name }}
+          sha: ${{ github.sha }}
+          run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          ok: ${{ needs.deploy.result == 'success' }}
+          # Call out the specific case where the release was blocked by CI.
+          failure-reason: ${{ needs.verify-ci.result != 'success' && 'CI never passed on the tagged commit — release blocked.' || '' }}