feat(http): Add observability metrics tracking

backend/app/api/deps.py

@@ -15,7 +15,7 @@
 from app.core.config import settings
 from app.core.db import engine
 from app.core.security import api_key_manager
-from app.core.telemetry import set_request_log_context
+from app.core.telemetry import bind_sentry_user, set_request_log_context
 from app.crud.organization import validate_organization
 from app.crud.project import validate_project
 from app.models import (
@@ -45,13 +45,7 @@ def get_db() -> Generator[Session, None, None]:
 
 
 def _set_tenant_span_attributes(auth_context: AuthContext) -> None:
-    """Tag the active OTel span and log context with tenant info after auth.
-
-    Sets org/project on:
-    - OTel span   → Sentry traces filterable by tenant
-    - log context → every log record in this request carries org_id/project_id
-    - Sentry scope → tags on all events for this request
-    """
+    """Stamp org/project on the span, log context and Sentry scope after auth."""
     span = trace.get_current_span()
     if span.is_recording():
         span.set_attribute("user.id", str(auth_context.user.id))
@@ -60,10 +54,10 @@ def _set_tenant_span_attributes(auth_context: AuthContext) -> None:
         if auth_context.project:
             span.set_attribute("tenant.project_id", auth_context.project.id)
 
-    set_request_log_context(
-        org_id=auth_context.organization.id if auth_context.organization else None,
-        project_id=auth_context.project.id if auth_context.project else None,
-    )
+    org_id = auth_context.organization.id if auth_context.organization else None
+    project_id = auth_context.project.id if auth_context.project else None
+    set_request_log_context(org_id=org_id, project_id=project_id)
+    bind_sentry_user(user_id=auth_context.user.id, org_id=org_id, project_id=project_id)
 
 
 def _authenticate_with_jwt(session: Session, token: str) -> AuthContext:

backend/app/celery/celery_app.py

@@ -2,18 +2,17 @@
 
 from celery import Celery
 from celery.signals import (
-    worker_init,
+    setup_logging,
     task_failure,
     task_postrun,
     task_prerun,
-    setup_logging,
+    worker_init,
     worker_process_init,
 )
 from kombu import Exchange, Queue
 
 from app.core.config import settings
 from app.core.logger import configure_logging
-from app.core.sentry_filters import before_send_filter, before_send_transaction_filter
 
 logger = logging.getLogger(__name__)
 _telemetry_initialized = False
@@ -35,37 +34,25 @@ def _initialize_worker_observability() -> None:
         import sentry_sdk
         from sentry_sdk.integrations.celery import CeleryIntegration
         from sentry_sdk.integrations.httpx import HttpxIntegration
-        from sentry_sdk.integrations.logging import LoggingIntegration
         from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 
-        sentry_sdk.init(
-            dsn=str(settings.SENTRY_DSN),
-            environment=settings.ENVIRONMENT,
-            release=settings.API_VERSION,
-            instrumenter="otel",
-            traces_sample_rate=1.0,
-            max_request_body_size="never",
-            enable_logs=True,
-            before_send=before_send_filter,
-            before_send_transaction=before_send_transaction_filter,
+        from app.core.telemetry import init_sentry
+
+        init_sentry(
             integrations=[
-                LoggingIntegration(
-                    level=logging.INFO,
-                    sentry_logs_level=logging.INFO,
-                ),
-                CeleryIntegration(
-                    propagate_traces=False,
-                    monitor_beat_tasks=False,
-                ),
-            ],
-            disabled_integrations=[
-                SqlalchemyIntegration(),
-                HttpxIntegration(),
+                CeleryIntegration(propagate_traces=True, monitor_beat_tasks=False)
             ],
+            disabled_integrations=[SqlalchemyIntegration(), HttpxIntegration()],
         )
+
+        @task_prerun.connect(weak=False)
+        def _tag_sentry_task(task: "celery.Task", **_: object) -> None:  # type: ignore[name-defined]
+            """Tag the task's Sentry scope with its name so Issues can be filtered by task."""
+            sentry_sdk.set_tag("celery.task_name", task.name)
+
         _sentry_initialized = True
 
-    if not _telemetry_initialized:
+    if settings.OTEL_ENABLED and not _telemetry_initialized:
         from app.core.telemetry import flush_telemetry, setup_telemetry
 
         setup_telemetry(service_name="kaapi-celery")
@@ -88,9 +75,10 @@ def log_pool_status(task: "celery.Task", **_: object) -> None:  # type: ignore[n
     If checked_out equals pool size right when a task starts, connections
     are being held across tasks (likely across LLM API calls) and leaking.
     """
-    from app.core.db import engine
     from sqlalchemy.pool import QueuePool
 
+    from app.core.db import engine
+
     pool = engine.pool
     if isinstance(pool, QueuePool):
         logger.info(
@@ -106,9 +94,10 @@ def log_pool_status_post(task: "celery.Task", **_: object) -> None:  # type: ign
     Compare with task_prerun log — if checked_out is the same or higher after
     the task, connections were not returned and are leaking.
     """
-    from app.core.db import engine
     from sqlalchemy.pool import QueuePool
 
+    from app.core.db import engine
+
     pool = engine.pool
     if isinstance(pool, QueuePool):
         logger.info(
@@ -126,9 +115,10 @@ def log_pool_status_failure(
     Failures are the most likely path for sessions to leak — exceptions can
     bypass session cleanup if not guarded by a context manager.
     """
-    from app.core.db import engine
     from sqlalchemy.pool import QueuePool
 
+    from app.core.db import engine
+
     pool = engine.pool
     if isinstance(pool, QueuePool):
         logger.warning(

backend/app/celery/tasks/job_execution.py

@@ -26,6 +26,7 @@
 from app.celery.celery_app import celery_app
 from app.celery.utils import gevent_timeout
 from app.core.config import settings
+from app.core.telemetry import LogModuleEnum, log_context
 
 if TYPE_CHECKING:
     from app.services.notifications.eval_completion import (
@@ -41,6 +42,8 @@
 # app/core/logger.py and app/celery/utils.py).
 DEFAULT_TRACE_ID = "N/A"
 
+SENTRY_NO_PROPAGATE_HEADERS: dict[str, bool] = {"sentry-propagate-traces": False}
+
 
 def _set_trace(trace_id: str) -> None:
     correlation_id.set(trace_id)
@@ -403,6 +406,7 @@ def run_assessment_api_batch(
                 "trace_id": trace_id,
             },
             countdown=POLL_COUNTDOWN_SECONDS,
+            headers=SENTRY_NO_PROPAGATE_HEADERS.copy(),
         )
     return result
 
@@ -451,12 +455,13 @@ def run_evaluation_fast_chunk(
         f"task_id={current_task.request.id}"
     )
 
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_fast_evaluation_chunk(
-            eval_run_id=eval_run_id, chunk_index=chunk_index
-        ),
-    )
+    with log_context(module=LogModuleEnum.FASTEVALS):
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_fast_evaluation_chunk(
+                eval_run_id=eval_run_id, chunk_index=chunk_index
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=6)
@@ -478,10 +483,11 @@ def run_evaluation_fast_aggregate(
         f"eval_run_id={eval_run_id} | task_id={current_task.request.id}"
     )
 
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_fast_evaluation_aggregate(eval_run_id=eval_run_id),
-    )
+    with log_context(module=LogModuleEnum.FASTEVALS):
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_fast_evaluation_aggregate(eval_run_id=eval_run_id),
+        )
 
 
 # Priority 6 (fast-eval tier): same band as run_prompt_improvement — one graph

backend/app/core/config.py

@@ -6,6 +6,7 @@
 
 from pydantic import (
     EmailStr,
+    Field,
     HttpUrl,
     PostgresDsn,
     computed_field,
@@ -158,6 +159,12 @@ def AWS_S3_BUCKET(self) -> str:
     BACKEND_SERVICE_NAME: str = "kaapi-backend"
     CRON_SERVICE_NAME: str = "kaapi-cron"
 
+    SENTRY_TRACES_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0)
+    SENTRY_SEND_DEFAULT_PII: bool = False
+    SENTRY_ERROR_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0)
+    SENTRY_PROFILE_SESSION_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0)
+    SENTRY_PROFILE_LIFECYCLE: Literal["manual", "trace"] = "trace"
+
     # Threshold Request Rate per minute
     THRESHOLD_LLM_CALL_RATE: int = 15
     THRESHOLD_COLLECTIONS_RATE: int = 3

backend/app/core/db.py

@@ -1,6 +1,7 @@
 from sqlmodel import Session, create_engine, select
 
 from app import crud
+from app.core.config import settings
 from app.core.telemetry import instrument_db_engine
 from app.models import User, UserCreate
 
@@ -33,7 +34,8 @@ def get_engine():
 
 # Create a default engine for backward compatibility
 engine = get_engine()
-instrument_db_engine(engine)
+if settings.OTEL_ENABLED:
+    instrument_db_engine(engine)
 
 
 # make sure all SQLModel models are imported (app.models) before initializing DB

backend/app/core/exception_handlers.py

@@ -1,6 +1,7 @@
 import re
 from collections import defaultdict
 
+import sentry_sdk
 from fastapi import FastAPI, Request, HTTPException
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
@@ -109,6 +110,7 @@ async def http_exception_handler(
 
     @app.exception_handler(Exception)
     async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse:
+        sentry_sdk.capture_exception(exc)
         return JSONResponse(
             status_code=HTTP_500_INTERNAL_SERVER_ERROR,
             content=APIResponse.failure_response(

backend/app/core/langfuse/langfuse.py

@@ -321,9 +321,6 @@ def wrapper(
             completion_config: NativeCompletionConfig, query: QueryParams, **kwargs
         ):
             if not credentials:
-                logger.info(
-                    "[observe_llm_execution] No credentials - skipping observability"
-                )
                 return func(completion_config, query, **kwargs)
 
             try:

backend/app/core/logger.py

@@ -8,6 +8,7 @@
 from asgi_correlation_id import correlation_id
 
 from app.core.config import settings
+from app.core.telemetry.context import LogContextFilter
 
 LOG_DIR = settings.LOG_DIR
 os.makedirs(LOG_DIR, exist_ok=True)
@@ -17,7 +18,7 @@
 LOGGING_LEVEL = logging.INFO
 LOGGING_FORMAT = (
     "%(asctime)s - [%(service_name)s] - [%(correlation_id)s] - "
-    "%(levelname)s - %(name)s - %(message)s"
+    "%(levelname)s - %(_log_location)s - %(message)s"
 )
 _service_name_context: ContextVar[str | None] = ContextVar(
     "kaapi_service_name", default=None
@@ -102,14 +103,17 @@ def configure_logging(service_name: str | None = None) -> None:
     stream_handler.setFormatter(formatter)
     stream_handler.addFilter(CorrelationIdFilter())
     stream_handler.addFilter(ServiceNameFilter(resolved_service_name))
+    stream_handler.addFilter(LogContextFilter())
 
     file_handler = RotatingFileHandler(
         LOG_FILE_PATH, maxBytes=10 * 1024 * 1024, backupCount=5
     )
     file_handler.setFormatter(formatter)
     file_handler.addFilter(CorrelationIdFilter())
     file_handler.addFilter(ServiceNameFilter(resolved_service_name))
+    file_handler.addFilter(LogContextFilter())
 
+    root_logger.addFilter(LogContextFilter())
     root_logger.handlers.clear()
     root_logger.addHandler(stream_handler)
     root_logger.addHandler(file_handler)