diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index b8f2ab46d..0d4fcb6eb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -44,6 +44,12 @@ jobs: steps: - name: Checkout the repo uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Resolve release id + id: release + run: echo "id=kaapi-backend@${GITHUB_REF_NAME}+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 @@ -63,6 +69,7 @@ jobs: run: | docker build \ --build-arg GIT_SHA=${{ github.sha }} \ + --build-arg RELEASE_TAG=${{ github.ref_name }} \ -t $REGISTRY/$REPOSITORY:latest \ -t $REGISTRY/$REPOSITORY:$TAG \ ./backend @@ -120,6 +127,18 @@ jobs: service: ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task task-definition: ${{ vars.AWS_RESOURCE_PREFIX }}-celery-task + - name: Register release and deploy in Sentry + uses: getsentry/action-release@v3 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ vars.SENTRY_ORG }} + SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }} + with: + release: ${{ steps.release.outputs.id }} + environment: production + set_commits: auto + ignore_missing: true + notify: needs: [verify-ci, build] if: always() diff --git a/.github/workflows/deploy-staging-ecs.yml b/.github/workflows/deploy-staging-ecs.yml index dc780e83a..4343bbe99 100644 --- a/.github/workflows/deploy-staging-ecs.yml +++ b/.github/workflows/deploy-staging-ecs.yml @@ -18,6 +18,12 @@ jobs: steps: - name: checkout the repo uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Resolve release id + id: release + run: echo "id=kaapi-backend@main+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 @@ -35,7 +41,7 @@ jobs: REGISTRY: ${{ steps.login-ecr.outputs.registry }} REPOSITORY: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-repo run: | - docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:latest ./backend + docker build --build-arg GIT_SHA=${{ github.sha }} --build-arg RELEASE_TAG=main -t $REGISTRY/$REPOSITORY:latest ./backend docker push $REGISTRY/$REPOSITORY:latest - name: Run database migrations @@ -77,3 +83,15 @@ jobs: with: cluster: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-cluster service: ${{ vars.AWS_RESOURCE_PREFIX }}-staging-service + + - name: Register release and deploy in Sentry + uses: getsentry/action-release@v3 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ vars.SENTRY_ORG }} + SENTRY_PROJECT: ${{ vars.SENTRY_STAGING_PROJECT }} + with: + release: ${{ steps.release.outputs.id }} + environment: staging + set_commits: auto + ignore_missing: true diff --git a/backend/Dockerfile b/backend/Dockerfile index 7bc1453d4..84e5ef477 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -46,6 +46,8 @@ COPY alembic.ini /app/alembic.ini ARG GIT_SHA=unknown ENV GIT_SHA=$GIT_SHA +ARG RELEASE_TAG="" +ENV RELEASE_TAG=$RELEASE_TAG # Expose port 80 EXPOSE 80 diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ce851ec25..b255bb58b 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -160,6 +160,9 @@ def AWS_S3_BUCKET(self) -> str: CRON_SERVICE_NAME: str = "kaapi-cron" SENTRY_TRACES_SAMPLE_RATE: float = 1.0 + SENTRY_RELEASE: str | None = None + GIT_SHA: str = "unknown" + RELEASE_TAG: str | None = None SENTRY_SEND_DEFAULT_PII: bool = False SENTRY_ERROR_SAMPLE_RATE: float = 1.0 SENTRY_PROFILE_SESSION_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0) diff --git a/backend/app/core/telemetry/sentry/init.py b/backend/app/core/telemetry/sentry/init.py index b2ba18dc2..5c6ed0f12 100644 --- a/backend/app/core/telemetry/sentry/init.py +++ b/backend/app/core/telemetry/sentry/init.py @@ -1,4 +1,4 @@ -"""sentry_sdk.init() with shared sampling, profiling, PII and before_send filters.""" +"""sentry_sdk.init() with shared sampling, profiling, PII and before_send filters; release id resolution.""" import logging @@ -14,6 +14,21 @@ genai_privacy_integrations, ) +GIT_SHA_SHORT_LEN = 12 +GIT_SHA_UNKNOWN = "unknown" + + +def resolve_sentry_release() -> str: + """Release id shared by API and worker: SENTRY_RELEASE, else @+ from the build.""" + if settings.SENTRY_RELEASE: + return settings.SENTRY_RELEASE + version = settings.RELEASE_TAG or settings.API_VERSION + release = f"{settings.BACKEND_SERVICE_NAME}@{version}" + sha = settings.GIT_SHA[:GIT_SHA_SHORT_LEN] + if sha and sha != GIT_SHA_UNKNOWN: + return f"{release}+{sha}" + return release + def init_sentry( *, @@ -26,7 +41,7 @@ def init_sentry( sentry_sdk.init( dsn=str(settings.SENTRY_DSN), environment=settings.ENVIRONMENT, - release=settings.API_VERSION, + release=resolve_sentry_release(), instrumenter="otel", traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE, sample_rate=settings.SENTRY_ERROR_SAMPLE_RATE, diff --git a/backend/app/tests/core/telemetry/sentry/test_sentry_init.py b/backend/app/tests/core/telemetry/sentry/test_sentry_init.py index 5b5bea0b3..69677b0df 100644 --- a/backend/app/tests/core/telemetry/sentry/test_sentry_init.py +++ b/backend/app/tests/core/telemetry/sentry/test_sentry_init.py @@ -13,6 +13,53 @@ def setup_once() -> None: return None +class TestResolveSentryRelease: + def test_uses_sentry_release_when_set(self) -> None: + with patch.object( + sentry_init.settings, "SENTRY_RELEASE", "kaapi-backend@9.9.9" + ): + assert sentry_init.resolve_sentry_release() == "kaapi-backend@9.9.9" + + def test_falls_back_to_service_and_version(self) -> None: + with ( + patch.object(sentry_init.settings, "SENTRY_RELEASE", None), + patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"), + patch.object(sentry_init.settings, "API_VERSION", "1.2.3"), + patch.object(sentry_init.settings, "RELEASE_TAG", None), + patch.object(sentry_init.settings, "GIT_SHA", "unknown"), + ): + assert sentry_init.resolve_sentry_release() == "kaapi-backend@1.2.3" + + def test_uses_release_tag_and_short_sha_from_build(self) -> None: + with ( + patch.object(sentry_init.settings, "SENTRY_RELEASE", None), + patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"), + patch.object(sentry_init.settings, "RELEASE_TAG", "v1.7.1"), + patch.object( + sentry_init.settings, + "GIT_SHA", + "3c5333f8ca8288407464a8f9065e77d1aab71454", + ), + ): + assert ( + sentry_init.resolve_sentry_release() + == "kaapi-backend@v1.7.1+3c5333f8ca82" + ) + + def test_sha_without_tag_falls_back_to_api_version(self) -> None: + with ( + patch.object(sentry_init.settings, "SENTRY_RELEASE", None), + patch.object(sentry_init.settings, "BACKEND_SERVICE_NAME", "kaapi-backend"), + patch.object(sentry_init.settings, "API_VERSION", "1.2.3"), + patch.object(sentry_init.settings, "RELEASE_TAG", None), + patch.object(sentry_init.settings, "GIT_SHA", "abcdef123456789"), + ): + assert ( + sentry_init.resolve_sentry_release() + == "kaapi-backend@1.2.3+abcdef123456" + ) + + class TestInitSentry: def test_noop_without_dsn(self) -> None: fake = MagicMock() diff --git a/backend/app/tests/core/test_config.py b/backend/app/tests/core/test_config.py index 4be4e932d..fc7d466bd 100644 --- a/backend/app/tests/core/test_config.py +++ b/backend/app/tests/core/test_config.py @@ -9,6 +9,7 @@ class TestSentryConfigDefaults: ("field", "expected"), [ ("SENTRY_TRACES_SAMPLE_RATE", 1.0), + ("SENTRY_RELEASE", None), ("SENTRY_SEND_DEFAULT_PII", False), ("SENTRY_ERROR_SAMPLE_RATE", 1.0), ("SENTRY_PROFILE_SESSION_SAMPLE_RATE", 1.0), diff --git a/docs/runbooks/sentry-release-tracking.md b/docs/runbooks/sentry-release-tracking.md new file mode 100644 index 000000000..1e3e02bca --- /dev/null +++ b/docs/runbooks/sentry-release-tracking.md @@ -0,0 +1,157 @@ +# Sentry Release Tracking: git sha se release, deploy se link + +**Audience:** Prashant / jo bhi kaapi-backend ka deploy pipeline sambhalta hai. +**Time:** ~30 min. Code aur workflow changes PR `feat/sentry-release-tracking` mein hain; baaki Sentry/GitHub settings haath se. + +## Abhi kya galat hai + +- Sentry mein release `0.5.0` April 2026 se static hai. Har deploy same release pe jaata hai, isliye: + - "Regressed in release", "Resolved in next release" kaam nahi karte. + - Suspect commits nahi dikhte (`commitCount: 0`, `deployCount: 0`). + - Kaunsa deploy kab gaya, Sentry se pata nahi chalta. +- `resolve_sentry_release()` sirf `SENTRY_RELEASE` env ya `kaapi-backend@API_VERSION` deta hai. `API_VERSION` haath se badalta hai, deploy pe nahi. +- Achhi khabar: Dockerfile mein `ARG GIT_SHA` / `ENV GIT_SHA` already hai, aur `create-release.yml` + `deploy-staging-ecs.yml` dono `--build-arg GIT_SHA=${{ github.sha }}` pass karte hain. Container ke andar sha available hai, bas koi padhta nahi. +- Environment theek hai: prod traffic `production` naam se aa raha hai. Sentry mein ek purana `prod` environment bhi hai, usko hide karna hai. + +## Target + +Release string: `kaapi-backend@+` +Example: `kaapi-backend@v0.5.1+3c5333f8ca82` + +- `` = git tag jo prod deploy trigger karta hai (`v0.5.1`). Staging pe `main`. +- `+` = commit sha ke pehle 12 chars. Sentry `+` ke baad wala build metadata maanta hai. +- Same string API aur Celery dono se jaaye, warna Sentry do release dikhayega. + +## Step 1: Code, release string sha se banao + +**File:** `backend/app/core/config.py` + +`SENTRY_RELEASE` ke paas do settings add karo: + +```python +GIT_SHA: str = "unknown" +RELEASE_TAG: str | None = None +``` + +Dockerfile ka `ENV GIT_SHA` pydantic-settings automatically uthayega. `RELEASE_TAG` workflow se aayega (Step 2). + +**File:** `backend/app/core/telemetry/sentry/init.py` + +`resolve_sentry_release()` badlo: + +```python +GIT_SHA_SHORT_LEN = 12 + + +def resolve_sentry_release() -> str: + """Release id shared by API and worker; SENTRY_RELEASE overrides, else tag+sha from the build.""" + if settings.SENTRY_RELEASE: + return settings.SENTRY_RELEASE + version = settings.RELEASE_TAG or settings.API_VERSION + sha = settings.GIT_SHA[:GIT_SHA_SHORT_LEN] + if sha and sha != "unknown": + return f"{settings.BACKEND_SERVICE_NAME}@{version}+{sha}" + return f"{settings.BACKEND_SERVICE_NAME}@{version}" +``` + +Local dev pe `GIT_SHA` unknown hoga, to purana format hi aayega. Koi `.env` change nahi. + +**Tests:** `backend/app/tests/core/telemetry/sentry/test_sentry_init.py` mein `TestResolveSentryRelease` mein do case add karo: sha set hai to `+sha` aata hai, `unknown` hai to nahi aata. + +Commit: `feat(telemetry): build Sentry release from git sha and release tag` + +## Step 2: Dockerfile, tag bhi container tak pahunchao + +**File:** `backend/Dockerfile`, `ARG GIT_SHA` ke neeche: + +```dockerfile +ARG RELEASE_TAG="" +ENV RELEASE_TAG=$RELEASE_TAG +``` + +**File:** `.github/workflows/create-release.yml`, "Build and Push Docker Image" step mein: + +```yaml +docker build \ + --build-arg GIT_SHA=${{ github.sha }} \ + --build-arg RELEASE_TAG=${{ github.ref_name }} \ + ... +``` + +**File:** `.github/workflows/deploy-staging-ecs.yml`, same step: + +```yaml +docker build --build-arg GIT_SHA=${{ github.sha }} --build-arg RELEASE_TAG=main ... +``` + +Staging release `kaapi-backend@main+` banega. Har main push pe naya release, sahi hai. + +## Step 3: Sentry ko batao ki release bana aur deploy hua + +Ye step suspect commits aur deploy markers deta hai. Bina iske Sentry ko release string dikhegi par commits nahi. + +### 3a. Sentry token banao + +Sentry → Settings → Developer Settings → **Organization Tokens** (user token nahi, org token) → Create. Scopes: `project:releases`, `org:read`. Naam: `github-actions-releases`. + +GitHub repo → Settings → Secrets and variables → Actions: +- Secret `SENTRY_AUTH_TOKEN` = upar wala token +- Variable `SENTRY_ORG` = `project-tech4dev` +- Variable `SENTRY_PROJECT` = `kaapi-production` +- Variable `SENTRY_STAGING_PROJECT` = `kaapi-staging` + +Chat mein jo user token diya tha wo yahan mat use karo, aur usko rotate kar do. + +### 3b. Repo link verify + +Sentry → Settings → Integrations → GitHub → `ProjectTech4DevAI/kaapi-backend` already linked hai (active). Kuch nahi karna. + +### 3c. Workflow step (PR mein already hai, yahan sirf samajhne ke liye) + +Dono workflows mein checkout ke baad ek step release id banata hai, taaki code aur Sentry mein string exactly same ho (sha ke 12 chars): + +```yaml + - name: Resolve release id + id: release + run: echo "id=kaapi-backend@${GITHUB_REF_NAME}+${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" +``` + +Staging mein `${GITHUB_REF_NAME}` ki jagah `main` hard-coded hai, kyunki wo workflow `workflow_dispatch` se chalta hai. + +Deploy steps ke **baad** (deploy fail hua to Sentry mein deploy marker nahi banna chahiye): + +```yaml + - name: Register release and deploy in Sentry + uses: getsentry/action-release@v3 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ vars.SENTRY_ORG }} + SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }} # staging: vars.SENTRY_STAGING_PROJECT + with: + release: ${{ steps.release.outputs.id }} + environment: production # staging: staging + set_commits: auto + ignore_missing: true +``` + +Checkout step pe `fetch-depth: 0` hai, warna `set_commits: auto` ko pichhle release tak ke commits nahi milte. + +## Step 4: Sentry UI cleanup + +1. Sentry → kaapi-production → Settings → Environments → `prod` → **Hide**. Sirf `production` rahe. +2. Settings → Inbound Filters → **Filter out known web crawlers** → ON. (Alag kaam, par yahin ho jaata hai.) + +## Step 5: Verify, pehle deploy ke baad + +1. Sentry → Releases → naya entry `kaapi-backend@v0.5.1+...` dikhe, `production` environment ke saath, commit count > 0. +2. Kisi bhi naye issue pe "First seen in release" wahi string ho. +3. API aur Celery dono se same release aa rahi hai: + Discover → spans → `field: release, count()` → sirf ek release row active deploy ke liye. +4. Purani `0.5.0` release pe naye events aana band. + +Agar step 3 mein API aur worker alag release dikhayein, to Celery task-definition mein `GIT_SHA`/`RELEASE_TAG` env override to nahi? Dono same image use karte hain, to nahi hona chahiye, par ECS task def mein explicit env check kar lo. + +## Baad mein (optional) + +- `SENTRY_RELEASE` env ab kisi ko set nahi karna chahiye. Agar ECS task def ya Secrets Manager mein set hai, hata do, warna wo override kar dega. +- Release health (crash-free sessions) backend ke liye nahi chahiye, `auto_session_tracking` off hi rehne do. diff --git a/docs/wiki/cross-cutting/observability.md b/docs/wiki/cross-cutting/observability.md index 3bf6a80d1..7ab0bad08 100644 --- a/docs/wiki/cross-cutting/observability.md +++ b/docs/wiki/cross-cutting/observability.md @@ -14,6 +14,7 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai - `core/telemetry/sentry/filters.py` — `before_send_transaction_filter` (drops probe transactions, scrubs genai content; DB spans are kept so Sentry Queries has data) and `before_send_error_filter` (drops probe events, always drops the request body, scrubs headers/cookies/query when `SENTRY_SEND_DEFAULT_PII` off, redacts LLM task kwargs). - GenAI content never reaches Sentry, in either PII mode. Four locks in `core/telemetry/sentry/filters.py`, wired by `init_sentry()` in `core/telemetry/sentry/init.py` for both API and worker: `scrub_genai_content` strips `_GENAI_CONTENT_KEYS` (prompt/completion span+log attributes) from every event, `before_send_log_filter` does the same for logs, `genai_privacy_integrations()` pins Sentry's auto-enabled AI integrations to `include_prompts=False`, and `include_local_variables=False` / `max_request_body_size="never"` keep prompts out of stack frames and request bodies. Langfuse (isolated tracer provider, `core/langfuse/langfuse.py`) stays the only sink for message bodies. - Corollary for log lines on LLM paths: log lengths/ids/types, never prompt or response text — `enable_logs=True` ships every INFO record to Sentry. +- Release: `resolve_sentry_release()` in `core/telemetry/sentry/init.py`: `SENTRY_RELEASE` override, else `@+`; `GIT_SHA`/`RELEASE_TAG` come from Docker build args, and the deploy workflows register the same id in Sentry with `getsentry/action-release` (see `docs/runbooks/sentry-release-tracking.md`). - Sampling/profiling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_PROFILE_SESSION_SAMPLE_RATE`, `SENTRY_PROFILE_LIFECYCLE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior). - Trace propagation: `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace. - Error capture: generic handler in `core/exception_handlers.py` calls `capture_exception`.