feat(sentry): Build release id from git

.github/workflows/create-release.yml

@@ -44,6 +44,12 @@ jobs:
     steps:
       - name: Checkout the repo
         uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Resolve release id
+        id: release
+        run: echo "id=kaapi-backend@${GITHUB_REF_NAME}+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
@@ -63,6 +69,7 @@ jobs:
         run: |
           docker build \
             --build-arg GIT_SHA=${{ github.sha }} \
+            --build-arg RELEASE_TAG=${{ github.ref_name }} \
             -t $REGISTRY/$REPOSITORY:latest \
             -t $REGISTRY/$REPOSITORY:$TAG \
             ./backend
@@ -120,6 +127,18 @@ jobs:
           service: ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task
           task-definition: ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task
 
+      - name: Register release and deploy in Sentry
+        uses: getsentry/action-release@v3
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
+        with:
+          release: ${{ steps.release.outputs.id }}
+          environment: production
+          set_commits: auto
+          ignore_missing: true
+
   notify:
     needs: [verify-ci, build]
     if: always()

.github/workflows/deploy-staging-ecs.yml

@@ -18,6 +18,12 @@ jobs:
     steps:
       - name: checkout the repo
         uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Resolve release id
+        id: release
+        run: echo "id=kaapi-backend@main+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
@@ -35,7 +41,7 @@ jobs:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo
         run: |
-          docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:latest ./backend
+          docker build --build-arg GIT_SHA=${{ github.sha }} --build-arg RELEASE_TAG=main -t $REGISTRY/$REPOSITORY:latest ./backend
           docker push $REGISTRY/$REPOSITORY:latest
 
       - name: Run database migrations
@@ -77,3 +83,15 @@ jobs:
         with:
           cluster: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster
           service: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service
+
+      - name: Register release and deploy in Sentry
+        uses: getsentry/action-release@v3
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ vars.SENTRY_STAGING_PROJECT }}
+        with:
+          release: ${{ steps.release.outputs.id }}
+          environment: staging
+          set_commits: auto
+          ignore_missing: true

backend/Dockerfile

@@ -46,6 +46,8 @@ COPY alembic.ini /app/alembic.ini
 
 ARG GIT_SHA=unknown
 ENV GIT_SHA=$GIT_SHA
+ARG RELEASE_TAG=""
+ENV RELEASE_TAG=$RELEASE_TAG
 
 # Expose port 80
 EXPOSE 80

backend/app/core/config.py

@@ -160,6 +160,9 @@ def AWS_S3_BUCKET(self) -> str:
     CRON_SERVICE_NAME: str = "kaapi-cron"
 
     SENTRY_TRACES_SAMPLE_RATE: float = 1.0
+    SENTRY_RELEASE: str | None = None
+    GIT_SHA: str = "unknown"
+    RELEASE_TAG: str | None = None
     SENTRY_SEND_DEFAULT_PII: bool = False
     SENTRY_ERROR_SAMPLE_RATE: float = 1.0
     SENTRY_PROFILE_SESSION_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0)

backend/app/core/telemetry/sentry/init.py

@@ -1,4 +1,4 @@
-"""sentry_sdk.init() with shared sampling, profiling, PII and before_send filters."""
+"""sentry_sdk.init() with shared sampling, profiling, PII and before_send filters; release id resolution."""
 
 import logging
 
@@ -14,6 +14,21 @@
     genai_privacy_integrations,
 )
 
+GIT_SHA_SHORT_LEN = 12
+GIT_SHA_UNKNOWN = "unknown"
+
+
+def resolve_sentry_release() -> str:
+    """Release id shared by API and worker: SENTRY_RELEASE, else <service>@<tag>+<sha> from the build."""
+    if settings.SENTRY_RELEASE:
+        return settings.SENTRY_RELEASE
+    version = settings.RELEASE_TAG or settings.API_VERSION
+    release = f"{settings.BACKEND_SERVICE_NAME}@{version}"
+    sha = settings.GIT_SHA[:GIT_SHA_SHORT_LEN]
+    if sha and sha != GIT_SHA_UNKNOWN:
+        return f"{release}+{sha}"
+    return release
+
 
 def init_sentry(
     *,
@@ -26,7 +41,7 @@ def init_sentry(
     sentry_sdk.init(
         dsn=str(settings.SENTRY_DSN),
         environment=settings.ENVIRONMENT,
-        release=settings.API_VERSION,
+        release=resolve_sentry_release(),
         instrumenter="otel",
         traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
         sample_rate=settings.SENTRY_ERROR_SAMPLE_RATE,

backend/app/tests/core/telemetry/sentry/test_sentry_init.py

@@ -13,6 +13,53 @@ def setup_once() -> None:
         return None
 
 
+class TestResolveSentryRelease:
+    def test_uses_sentry_release_when_set(self) -> None:
+        with patch.object(
+            sentry_init.settings, "SENTRY_RELEASE", "[email protected]"
+        ):
+            assert sentry_init.resolve_sentry_release() == "[email protected]"
+
+    def test_falls_back_to_service_and_version(self) -> None:
+        with (
+            patch.object(sentry_init.settings, "SENTRY_RELEASE", None),
+            patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"),
+            patch.object(sentry_init.settings, "API_VERSION", "1.2.3"),
+            patch.object(sentry_init.settings, "RELEASE_TAG", None),
+            patch.object(sentry_init.settings, "GIT_SHA", "unknown"),
+        ):
+            assert sentry_init.resolve_sentry_release() == "[email protected]"
+
+    def test_uses_release_tag_and_short_sha_from_build(self) -> None:
+        with (
+            patch.object(sentry_init.settings, "SENTRY_RELEASE", None),
+            patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"),
+            patch.object(sentry_init.settings, "RELEASE_TAG", "v1.7.1"),
+            patch.object(
+                sentry_init.settings,
+                "GIT_SHA",
+                "3c5333f8ca8288407464a8f9065e77d1aab71454",
+            ),
+        ):
+            assert (
+                sentry_init.resolve_sentry_release()
+                == "[email protected]+3c5333f8ca82"
+            )
+
+    def test_sha_without_tag_falls_back_to_api_version(self) -> None:
+        with (
+            patch.object(sentry_init.settings, "SENTRY_RELEASE", None),
+            patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"),
+            patch.object(sentry_init.settings, "API_VERSION", "1.2.3"),
+            patch.object(sentry_init.settings, "RELEASE_TAG", None),
+            patch.object(sentry_init.settings, "GIT_SHA", "abcdef123456789"),
+        ):
+            assert (
+                sentry_init.resolve_sentry_release()
+                == "[email protected]+abcdef123456"
+            )
+
+
 class TestInitSentry:
     def test_noop_without_dsn(self) -> None:
         fake = MagicMock()

backend/app/tests/core/test_config.py

@@ -9,6 +9,7 @@ class TestSentryConfigDefaults:
         ("field", "expected"),
         [
             ("SENTRY_TRACES_SAMPLE_RATE", 1.0),
+            ("SENTRY_RELEASE", None),
             ("SENTRY_SEND_DEFAULT_PII", False),
             ("SENTRY_ERROR_SAMPLE_RATE", 1.0),
             ("SENTRY_PROFILE_SESSION_SAMPLE_RATE", 1.0),

docs/runbooks/sentry-release-tracking.md

@@ -0,0 +1,157 @@
+# Sentry Release Tracking: git sha se release, deploy se link
+
+**Audience:** Prashant / jo bhi kaapi-backend ka deploy pipeline sambhalta hai.
+**Time:** ~30 min. Code aur workflow changes PR `feat/sentry-release-tracking` mein hain; baaki Sentry/GitHub settings haath se.
+
+## Abhi kya galat hai
+
+- Sentry mein release `0.5.0` April 2026 se static hai. Har deploy same release pe jaata hai, isliye:
+  - "Regressed in release", "Resolved in next release" kaam nahi karte.
+  - Suspect commits nahi dikhte (`commitCount: 0`, `deployCount: 0`).
+  - Kaunsa deploy kab gaya, Sentry se pata nahi chalta.
+- `resolve_sentry_release()` sirf `SENTRY_RELEASE` env ya `kaapi-backend@API_VERSION` deta hai. `API_VERSION` haath se badalta hai, deploy pe nahi.
+- Achhi khabar: Dockerfile mein `ARG GIT_SHA` / `ENV GIT_SHA` already hai, aur `create-release.yml` + `deploy-staging-ecs.yml` dono `--build-arg GIT_SHA=${{ github.sha }}` pass karte hain. Container ke andar sha available hai, bas koi padhta nahi.
+- Environment theek hai: prod traffic `production` naam se aa raha hai. Sentry mein ek purana `prod` environment bhi hai, usko hide karna hai.
+
+## Target
+
+Release string: `kaapi-backend@<tag>+<sha12>`
+Example: `[email protected]+3c5333f8ca82`
+
+- `<tag>` = git tag jo prod deploy trigger karta hai (`v0.5.1`). Staging pe `main`.
+- `+<sha12>` = commit sha ke pehle 12 chars. Sentry `+` ke baad wala build metadata maanta hai.
+- Same string API aur Celery dono se jaaye, warna Sentry do release dikhayega.
+
+## Step 1: Code, release string sha se banao
+
+**File:** `backend/app/core/config.py`
+
+`SENTRY_RELEASE` ke paas do settings add karo:
+
+```python
+GIT_SHA: str = "unknown"
+RELEASE_TAG: str | None = None
+```
+
+Dockerfile ka `ENV GIT_SHA` pydantic-settings automatically uthayega. `RELEASE_TAG` workflow se aayega (Step 2).
+
+**File:** `backend/app/core/telemetry/sentry/init.py`
+
+`resolve_sentry_release()` badlo:
+
+```python
+GIT_SHA_SHORT_LEN = 12
+
+
+def resolve_sentry_release() -> str:
+    """Release id shared by API and worker; SENTRY_RELEASE overrides, else tag+sha from the build."""
+    if settings.SENTRY_RELEASE:
+        return settings.SENTRY_RELEASE
+    version = settings.RELEASE_TAG or settings.API_VERSION
+    sha = settings.GIT_SHA[:GIT_SHA_SHORT_LEN]
+    if sha and sha != "unknown":
+        return f"{settings.BACKEND_SERVICE_NAME}@{version}+{sha}"
+    return f"{settings.BACKEND_SERVICE_NAME}@{version}"
+```
+
+Local dev pe `GIT_SHA` unknown hoga, to purana format hi aayega. Koi `.env` change nahi.
+
+**Tests:** `backend/app/tests/core/telemetry/sentry/test_sentry_init.py` mein `TestResolveSentryRelease` mein do case add karo: sha set hai to `+sha` aata hai, `unknown` hai to nahi aata.
+
+Commit: `feat(telemetry): build Sentry release from git sha and release tag`
+
+## Step 2: Dockerfile, tag bhi container tak pahunchao
+
+**File:** `backend/Dockerfile`, `ARG GIT_SHA` ke neeche:
+
+```dockerfile
+ARG RELEASE_TAG=""
+ENV RELEASE_TAG=$RELEASE_TAG
+```
+
+**File:** `.github/workflows/create-release.yml`, "Build and Push Docker Image" step mein:
+
+```yaml
+docker build \
+  --build-arg GIT_SHA=${{ github.sha }} \
+  --build-arg RELEASE_TAG=${{ github.ref_name }} \
+  ...
+```
+
+**File:** `.github/workflows/deploy-staging-ecs.yml`, same step:
+
+```yaml
+docker build --build-arg GIT_SHA=${{ github.sha }} --build-arg RELEASE_TAG=main ...
+```
+
+Staging release `kaapi-backend@main+<sha>` banega. Har main push pe naya release, sahi hai.
+
+## Step 3: Sentry ko batao ki release bana aur deploy hua
+
+Ye step suspect commits aur deploy markers deta hai. Bina iske Sentry ko release string dikhegi par commits nahi.
+
+### 3a. Sentry token banao
+
+Sentry → Settings → Developer Settings → **Organization Tokens** (user token nahi, org token) → Create. Scopes: `project:releases`, `org:read`. Naam: `github-actions-releases`.
+
+GitHub repo → Settings → Secrets and variables → Actions:
+- Secret `SENTRY_AUTH_TOKEN` = upar wala token
+- Variable `SENTRY_ORG` = `project-tech4dev`
+- Variable `SENTRY_PROJECT` = `kaapi-production`
+- Variable `SENTRY_STAGING_PROJECT` = `kaapi-staging`
+
+Chat mein jo user token diya tha wo yahan mat use karo, aur usko rotate kar do.
+
+### 3b. Repo link verify
+
+Sentry → Settings → Integrations → GitHub → `ProjectTech4DevAI/kaapi-backend` already linked hai (active). Kuch nahi karna.
+
+### 3c. Workflow step (PR mein already hai, yahan sirf samajhne ke liye)
+
+Dono workflows mein checkout ke baad ek step release id banata hai, taaki code aur Sentry mein string exactly same ho (sha ke 12 chars):
+
+```yaml
+      - name: Resolve release id
+        id: release
+        run: echo "id=kaapi-backend@${GITHUB_REF_NAME}+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT"
+```
+
+Staging mein `${GITHUB_REF_NAME}` ki jagah `main` hard-coded hai, kyunki wo workflow `workflow_dispatch` se chalta hai.
+
+Deploy steps ke **baad** (deploy fail hua to Sentry mein deploy marker nahi banna chahiye):
+
+```yaml
+      - name: Register release and deploy in Sentry
+        uses: getsentry/action-release@v3
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}        # staging: vars.SENTRY_STAGING_PROJECT
+        with:
+          release: ${{ steps.release.outputs.id }}
+          environment: production                           # staging: staging
+          set_commits: auto
+          ignore_missing: true
+```
+
+Checkout step pe `fetch-depth: 0` hai, warna `set_commits: auto` ko pichhle release tak ke commits nahi milte.
+
+## Step 4: Sentry UI cleanup
+
+1. Sentry → kaapi-production → Settings → Environments → `prod` → **Hide**. Sirf `production` rahe.
+2. Settings → Inbound Filters → **Filter out known web crawlers** → ON. (Alag kaam, par yahin ho jaata hai.)
+
+## Step 5: Verify, pehle deploy ke baad
+
+1. Sentry → Releases → naya entry `[email protected]+...` dikhe, `production` environment ke saath, commit count > 0.
+2. Kisi bhi naye issue pe "First seen in release" wahi string ho.
+3. API aur Celery dono se same release aa rahi hai:
+   Discover → spans → `field: release, count()` → sirf ek release row active deploy ke liye.
+4. Purani `0.5.0` release pe naye events aana band.
+
+Agar step 3 mein API aur worker alag release dikhayein, to Celery task-definition mein `GIT_SHA`/`RELEASE_TAG` env override to nahi? Dono same image use karte hain, to nahi hona chahiye, par ECS task def mein explicit env check kar lo.
+
+## Baad mein (optional)
+
+- `SENTRY_RELEASE` env ab kisi ko set nahi karna chahiye. Agar ECS task def ya Secrets Manager mein set hai, hata do, warna wo override kar dega.
+- Release health (crash-free sessions) backend ke liye nahi chahiye, `auto_session_tracking` off hi rehne do.

docs/wiki/cross-cutting/observability.md

@@ -14,6 +14,7 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai
 - `core/telemetry/sentry/filters.py` — `before_send_transaction_filter` (drops probe transactions, scrubs genai content; DB spans are kept so Sentry Queries has data) and `before_send_error_filter` (drops probe events, always drops the request body, scrubs headers/cookies/query when `SENTRY_SEND_DEFAULT_PII` off, redacts LLM task kwargs).
 - GenAI content never reaches Sentry, in either PII mode. Four locks in `core/telemetry/sentry/filters.py`, wired by `init_sentry()` in `core/telemetry/sentry/init.py` for both API and worker: `scrub_genai_content` strips `_GENAI_CONTENT_KEYS` (prompt/completion span+log attributes) from every event, `before_send_log_filter` does the same for logs, `genai_privacy_integrations()` pins Sentry's auto-enabled AI integrations to `include_prompts=False`, and `include_local_variables=False` / `max_request_body_size="never"` keep prompts out of stack frames and request bodies. Langfuse (isolated tracer provider, `core/langfuse/langfuse.py`) stays the only sink for message bodies.
 - Corollary for log lines on LLM paths: log lengths/ids/types, never prompt or response text — `enable_logs=True` ships every INFO record to Sentry.
+- Release: `resolve_sentry_release()` in `core/telemetry/sentry/init.py`: `SENTRY_RELEASE` override, else `<service>@<RELEASE_TAG or API_VERSION>+<git sha 12>`; `GIT_SHA`/`RELEASE_TAG` come from Docker build args, and the deploy workflows register the same id in Sentry with `getsentry/action-release` (see `docs/runbooks/sentry-release-tracking.md`).
 - Sampling/profiling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_PROFILE_SESSION_SAMPLE_RATE`, `SENTRY_PROFILE_LIFECYCLE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior).
 - Trace propagation: `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace.
 - Error capture: generic handler in `core/exception_handlers.py` calls `capture_exception`.