From 10ed0a48b7d209137680bded6aee02804c764136 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:54:19 +0530 Subject: [PATCH 01/12] Refactor error handling across services to use domain-specific exceptions - Updated error logging to include exception information for better traceability. - Introduced `ConflictError`, `UpstreamError`, and `ServiceUnavailableError` for more granular HTTP status codes. - Changed HTTP status codes in various tests and service methods to align with new error handling conventions. - Added tests to ensure correct behavior when handling exceptions, including cases for missing resources, state conflicts, and upstream errors. - Updated documentation to reflect changes in exception handling and status code mappings. --- backend/app/api/routes/collections.py | 11 +-- .../api/routes/stt_evaluations/evaluation.py | 11 +-- .../api/routes/tts_evaluations/evaluation.py | 11 +-- backend/app/core/exception_handlers.py | 24 +++++- backend/app/core/exceptions.py | 43 +++++++++++ backend/app/core/security.py | 17 ++++- backend/app/crud/collection/collection.py | 9 ++- backend/app/crud/config/config.py | 16 ++++ backend/app/crud/config/version.py | 22 +++++- backend/app/crud/credentials.py | 46 +++++++----- .../crud/document/doc_transformation_job.py | 2 +- backend/app/crud/document/document.py | 2 +- backend/app/crud/evaluations/batch.py | 49 ++++++++++-- backend/app/crud/evaluations/core.py | 16 ++-- backend/app/crud/evaluations/cron_utils.py | 3 +- backend/app/crud/evaluations/embeddings.py | 55 +++++++++----- backend/app/crud/evaluations/langfuse.py | 6 +- backend/app/crud/evaluations/processing.py | 6 +- backend/app/crud/onboarding.py | 17 ++++- backend/app/crud/stt_evaluations/batch.py | 3 +- backend/app/crud/tts_evaluations/batch.py | 3 +- .../services/collections/providers/gemini.py | 1 + backend/app/services/documents/helpers.py | 5 +- backend/app/services/evaluations/dataset.py | 10 ++- backend/app/services/evaluations/fast.py | 32 +++++--- backend/app/services/guardrails/jobs.py | 7 +- backend/app/services/llm/jobs.py | 10 +-- backend/app/services/stt_evaluations/audio.py | 14 ++-- .../documents/test_route_document_upload.py | 2 +- backend/app/tests/api/routes/test_creds.py | 4 +- .../app/tests/api/routes/test_evaluation.py | 13 ++-- .../tests/api/routes/test_tts_evaluation.py | 4 +- .../app/tests/core/test_exception_handlers.py | 61 ++++++++++++++- backend/app/tests/core/test_security.py | 28 +++++++ ...test_crud_collection_read_one_if_delete.py | 58 +++++++++++++++ .../crud/evaluations/test_batch_errors.py | 54 ++++++++++++++ backend/app/tests/crud/test_credentials.py | 74 +++++++++++++++++++ backend/app/tests/crud/test_onboarding.py | 51 +++++++++++++ .../tests/services/guardrails/test_jobs.py | 9 ++- backend/app/tests/services/llm/test_jobs.py | 19 ++--- .../services/stt_evaluations/test_audio.py | 10 ++- docs/wiki/cross-cutting/exceptions.md | 20 ++++- 42 files changed, 705 insertions(+), 153 deletions(-) create mode 100644 backend/app/core/exceptions.py create mode 100644 backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py create mode 100644 backend/app/tests/crud/evaluations/test_batch_errors.py diff --git a/backend/app/api/routes/collections.py b/backend/app/api/routes/collections.py index 6134855ea..ae83e67dc 100644 --- a/backend/app/api/routes/collections.py +++ b/backend/app/api/routes/collections.py @@ -2,7 +2,7 @@ from uuid import UUID from typing import List -from fastapi import APIRouter, HTTPException, Query, Body, Depends +from fastapi import APIRouter, Query, Body, Depends from fastapi import Path as FastPath from app.api.deps import SessionDep, AuthContextDep @@ -14,7 +14,6 @@ CollectionJobCrud, DocumentCollectionCrud, ) -from app.crud.collection.collection import CollectionNameConflictError from app.core.cloud import get_cloud_storage from app.models import ( CollectionJobStatus, @@ -215,13 +214,7 @@ def update_collection( organization_id=current_user.organization_.id, ): collection_crud = CollectionCrud(session, current_user.project_.id) - try: - collection = collection_crud.update(collection_id, patch) - except CollectionNameConflictError as err: - raise HTTPException( - status_code=409, - detail=f"Collection '{err.name}' already exists. Choose a different name.", - ) + collection = collection_crud.update(collection_id, patch) logger.info( f"[update_collection] Collection updated | {{'collection_id': '{collection_id}'}}" diff --git a/backend/app/api/routes/stt_evaluations/evaluation.py b/backend/app/api/routes/stt_evaluations/evaluation.py index 6b96f4414..9495ef3f1 100644 --- a/backend/app/api/routes/stt_evaluations/evaluation.py +++ b/backend/app/api/routes/stt_evaluations/evaluation.py @@ -9,6 +9,7 @@ from app.api.permissions import Permission, require_permission from app.celery.utils import start_stt_batch_submission from app.core.cloud import get_cloud_storage +from app.core.exceptions import ServiceUnavailableError from app.crud.stt_evaluations import ( create_stt_run, get_results_by_run_id, @@ -62,7 +63,7 @@ def start_stt_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=400, detail="Dataset has no samples") + raise HTTPException(status_code=422, detail="Dataset has no samples") # Use language_id from the dataset language_id = dataset.language_id @@ -97,7 +98,8 @@ def start_stt_evaluation( except Exception as e: logger.error( f"[start_stt_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}" + f"run_id: {run.id}, error: {str(e)}", + exc_info=True, ) update_stt_run( session=session, @@ -105,9 +107,8 @@ def start_stt_evaluation( status="failed", error_message=f"Failed to queue batch submission: {str(e)}", ) - raise HTTPException( - status_code=500, - detail=f"Failed to queue batch submission: {e}", + raise ServiceUnavailableError( + "Could not queue the batch submission. Retry shortly." ) return APIResponse.success_response( diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index 6751d8219..145bc0f7c 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -9,6 +9,7 @@ from app.api.permissions import Permission, require_permission from app.celery.utils import start_tts_batch_submission from app.core.cloud import get_cloud_storage +from app.core.exceptions import ServiceUnavailableError from app.crud.tts_evaluations import ( create_tts_run, get_results_by_run_id, @@ -66,7 +67,7 @@ def start_tts_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=400, detail="Dataset has no samples") + raise HTTPException(status_code=422, detail="Dataset has no samples") language_id = dataset.language_id @@ -101,7 +102,8 @@ def start_tts_evaluation( except Exception as e: logger.error( f"[start_tts_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}" + f"run_id: {run.id}, error: {str(e)}", + exc_info=True, ) update_tts_run( session=session, @@ -109,9 +111,8 @@ def start_tts_evaluation( status="failed", error_message=f"Failed to queue batch submission: {str(e)}", ) - raise HTTPException( - status_code=500, - detail=f"Failed to queue batch submission: {e}", + raise ServiceUnavailableError( + "Could not queue the batch submission. Retry shortly." ) return APIResponse.success_response( diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 4d1e4d747..31b62291e 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -1,6 +1,8 @@ +import logging import re from collections import defaultdict +from asgi_correlation_id import correlation_id from fastapi import FastAPI, Request, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse @@ -9,8 +11,13 @@ HTTP_500_INTERNAL_SERVER_ERROR, ) +from app.core.exceptions import KaapiError from app.utils import APIResponse +logger = logging.getLogger(__name__) + +GENERIC_ERROR_DETAIL = "An unexpected error occurred." + _BRANCH_PATTERN = re.compile(r"^[A-Z]|[\[\]()]") @@ -107,11 +114,22 @@ async def http_exception_handler( content=APIResponse.failure_response(detail).model_dump(), ) + @app.exception_handler(KaapiError) + async def kaapi_error_handler(request: Request, exc: KaapiError) -> JSONResponse: + return JSONResponse( + status_code=exc.status_code, + content=APIResponse.failure_response(exc.detail).model_dump(), + ) + @app.exception_handler(Exception) async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: + # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. + logger.error( + f"[generic_error_handler] Unhandled exception | " + f"path: {request.url.path}, trace_id: {correlation_id.get() or 'N/A'}", + exc_info=True, + ) return JSONResponse( status_code=HTTP_500_INTERNAL_SERVER_ERROR, - content=APIResponse.failure_response( - str(exc) or "An unexpected error occurred." - ).model_dump(), + content=APIResponse.failure_response(GENERIC_ERROR_DETAIL).model_dump(), ) diff --git a/backend/app/core/exceptions.py b/backend/app/core/exceptions.py new file mode 100644 index 000000000..854cf68c3 --- /dev/null +++ b/backend/app/core/exceptions.py @@ -0,0 +1,43 @@ +"""Domain exceptions carrying the status code they surface as. + +Raise from crud/service instead of a bare `ValueError`, which falls through to the +generic 500 handler. Mapped to responses by `app.core.exception_handlers`. +""" + +from starlette import status + + +class KaapiError(Exception): + status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR + + def __init__(self, detail: str) -> None: + super().__init__(detail) + self.detail = detail + + +class NotFoundError(KaapiError): + status_code = status.HTTP_404_NOT_FOUND + + +class ConflictError(KaapiError): + status_code = status.HTTP_409_CONFLICT + + +class InvalidValueError(KaapiError): + status_code = status.HTTP_400_BAD_REQUEST + + +class InvalidPayloadError(KaapiError): + status_code = status.HTTP_422_UNPROCESSABLE_CONTENT + + +class UpstreamError(KaapiError): + status_code = status.HTTP_502_BAD_GATEWAY + + def __init__(self, detail: str, provider: str) -> None: + super().__init__(detail) + self.provider = provider + + +class ServiceUnavailableError(KaapiError): + status_code = status.HTTP_503_SERVICE_UNAVAILABLE diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 406f46ed8..977f91596 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -26,6 +26,7 @@ from sqlmodel import Session, and_, select from app.core.config import settings +from app.core.exceptions import UpstreamError from app.models import APIKey, AuthContext, Organization, Project, User logger = logging.getLogger(__name__) @@ -212,8 +213,12 @@ def encrypt_credentials(credentials: dict[str, Any]) -> str: return get_fernet().encrypt(credentials_str.encode()).decode() except Exception as e: # Log the real cause (may carry AWS ARNs); never surface it to callers. - logger.error(f"[encrypt_credentials] Encryption failed | error: {e}") - raise ValueError("Failed to encrypt credentials") + logger.error( + f"[encrypt_credentials] Encryption failed | error: {e}", exc_info=True + ) + raise UpstreamError( + "Failed to encrypt credentials. Retry shortly.", provider="kms" + ) def decrypt_credentials(encrypted_credentials: str) -> dict[str, Any]: @@ -232,8 +237,12 @@ def decrypt_credentials(encrypted_credentials: str) -> dict[str, Any]: return json.loads(decrypted_str) except Exception as e: # Log the real cause (may carry AWS ARNs); never surface it to callers. - logger.error(f"[decrypt_credentials] Decryption failed | error: {e}") - raise ValueError("Failed to decrypt credentials") + logger.error( + f"[decrypt_credentials] Decryption failed | error: {e}", exc_info=True + ) + raise UpstreamError( + "Failed to decrypt credentials. Retry shortly.", provider="kms" + ) class APIKeyManager: diff --git a/backend/app/crud/collection/collection.py b/backend/app/crud/collection/collection.py index 96c4e82be..a7ae2b124 100644 --- a/backend/app/crud/collection/collection.py +++ b/backend/app/crud/collection/collection.py @@ -9,18 +9,21 @@ from sqlalchemy.exc import IntegrityError from app.models import Document, Collection, CollectionUpdate, DocumentCollection +from app.core.exceptions import ConflictError from app.core.util import now from app.crud.document_collection import DocumentCollectionCrud logger = logging.getLogger(__name__) -class CollectionNameConflictError(Exception): +class CollectionNameConflictError(ConflictError): """Raised when a collection name conflicts with an existing active collection.""" def __init__(self, name: str | None) -> None: self.name = name - super().__init__(f"Collection name '{name}' already exists") + super().__init__( + f"Collection '{name}' already exists. Choose a different name." + ) class CollectionCrud: @@ -111,7 +114,7 @@ def read_one_if_delete(self, collection_id: UUID) -> Collection: "[CollectionCrud.read_one_if_delete] Collection already deleted | " f"{{'project_id': '{self.project_id}', 'collection_id': '{collection_id}'}}" ) - raise HTTPException(status_code=400, detail="Collection already deleted") + raise ConflictError("Collection already deleted") return collection diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index fdfd055f0..2a8402c46 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -2,8 +2,10 @@ from uuid import UUID from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, and_, select +from app.core.exceptions import ConflictError, KaapiError from app.core.util import now from app.crud.model_config import validate_blob_model_or_raise from app.models import ( @@ -67,6 +69,20 @@ def create_or_raise( return config, version + except (HTTPException, KaapiError): + self.session.rollback() + raise + + except IntegrityError as e: + self.session.rollback() + logger.warning( + f"[ConfigCrud.create] Duplicate configuration name | " + f"{{'name': '{config_create.name}', 'project_id': {self.project_id}, 'error': '{str(e)}'}}", + ) + raise ConflictError( + f"Config with name '{config_create.name}' already exists in this project." + ) + except Exception as e: self.session.rollback() logger.error( diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index a0a312c93..d3a0ac62d 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -4,9 +4,11 @@ from fastapi import HTTPException from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import defer from sqlmodel import Session, and_, select +from app.core.exceptions import ConflictError, KaapiError from app.core.util import now from app.crud.model_config import is_reasoning_model, validate_blob_model_or_raise from app.models import ( @@ -82,7 +84,7 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: f"{{'config_id': '{self.config_id}', 'error_count': {len(validation_errors)}, " f"'fields': {['.'.join(str(part) for part in err['loc']) for err in validation_errors]}}}" ) - raise HTTPException(status_code=400, detail=validation_errors) + raise HTTPException(status_code=422, detail=validation_errors) validate_blob_model_or_raise(self.session, validated_blob) @@ -101,16 +103,30 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: self.session.refresh(version) logger.info( - f"[ConfigVersionCrud.create_from_partial] Version created successfully | " + f"[ConfigVersionCrud.create_or_raise] Version created successfully | " f"{{'config_id': '{self.config_id}', 'version_id': '{version.id}'}}" ) return version + except (HTTPException, KaapiError): + self.session.rollback() + raise + + except IntegrityError as e: + self.session.rollback() + logger.warning( + f"[ConfigVersionCrud.create_or_raise] Version conflict | " + f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", + ) + raise ConflictError( + "A version with this number already exists for this config." + ) + except Exception as e: self.session.rollback() logger.error( - f"[ConfigVersionCrud.create_from_partial] Failed to create version | " + f"[ConfigVersionCrud.create_or_raise] Failed to create version | " f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", exc_info=True, ) diff --git a/backend/app/crud/credentials.py b/backend/app/crud/credentials.py index cc070852b..9a345d8b1 100644 --- a/backend/app/crud/credentials.py +++ b/backend/app/crud/credentials.py @@ -5,7 +5,9 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from app.core.exception_handlers import HTTPException +from fastapi import HTTPException + +from app.core.exceptions import ConflictError, KaapiError, NotFoundError from app.core.providers import validate_provider, validate_provider_credentials from app.core.security import decrypt_credentials, encrypt_credentials from app.core.util import now @@ -54,21 +56,21 @@ def set_creds_for_org( created_credentials.append(credential) except IntegrityError as e: session.rollback() - logger.error( + is_duplicate = ( + "uq_credential_org_project_provider" in str(e) + or "unique constraint" in str(e).lower() + ) + log = logger.warning if is_duplicate else logger.error + log( f"[set_creds_for_org] Integrity error while adding credentials | organization_id {organization_id}, project_id {project_id}, provider {provider}: {str(e)}", exc_info=True, ) - # Check if it's a duplicate constraint violation - if ( - "uq_credential_org_project_provider" in str(e) - or "unique constraint" in str(e).lower() - ): - raise HTTPException( - status_code=400, - detail=f"Credentials for provider '{provider}' already exist for this organization and project combination", + if is_duplicate: + raise ConflictError( + f"Credentials for provider '{provider}' already exist for this organization and project combination" ) - raise ValueError( - f"Error while adding credentials for provider {provider}: {str(e)}" + raise KaapiError( + f"Could not store credentials for provider '{provider}'. Contact Kaapi if it persists." ) logger.info( f"[set_creds_for_org] Successfully created credentials | organization_id {organization_id}, project_id {project_id}" @@ -187,7 +189,9 @@ def update_creds_for_org( ) -> list[Credential]: """Updates credentials for a specific provider of an organization.""" if not creds_in.provider or not creds_in.credential: - raise ValueError("Provider and credential must be provided") + raise HTTPException( + status_code=400, detail="Provider and credential must be provided" + ) # Auto-unwrap nested format: {"google": {"api_key": "..."}} -> {"api_key": "..."} # so the same payload shape works for both create and update. @@ -280,13 +284,10 @@ def remove_provider_credential( rows_deleted = result.rowcount if rows_deleted == 0: session.rollback() - logger.error( - f"[remove_provider_credential] Failed to delete credential | organization_id {org_id}, provider {provider}, project_id {project_id}" - ) - raise HTTPException( - status_code=500, - detail="Failed to delete provider credential", + logger.warning( + f"[remove_provider_credential] Credential not found | organization_id {org_id}, provider {provider}, project_id {project_id}" ) + raise NotFoundError(f"No credentials found for provider '{provider}'.") session.commit() logger.info( f"[remove_provider_credential] Successfully deleted credential | provider {provider}, organization_id {org_id}, project_id {project_id}" @@ -318,6 +319,13 @@ def remove_creds_for_org(*, session: Session, org_id: int, project_id: int) -> N rows_deleted = result.rowcount + if rows_deleted == 0: + session.rollback() + logger.warning( + f"[remove_creds_for_org] No credentials to delete | organization_id {org_id}, project_id {project_id}" + ) + raise NotFoundError("No credentials found for this organization and project.") + if rows_deleted < expected_count: logger.error( f"[remove_creds_for_org] Failed to delete all credentials | organization_id {org_id}, project_id {project_id}, expected {expected_count}, deleted {rows_deleted}" diff --git a/backend/app/crud/document/doc_transformation_job.py b/backend/app/crud/document/doc_transformation_job.py index 3e329df70..35b29cc1e 100644 --- a/backend/app/crud/document/doc_transformation_job.py +++ b/backend/app/crud/document/doc_transformation_job.py @@ -2,6 +2,7 @@ from uuid import UUID from typing import List, Optional +from fastapi import HTTPException from sqlmodel import Session, select, and_ from app.crud import DocumentCrud @@ -13,7 +14,6 @@ ) from app.models.document import Document from app.core.util import now -from app.core.exception_handlers import HTTPException logger = logging.getLogger(__name__) diff --git a/backend/app/crud/document/document.py b/backend/app/crud/document/document.py index ef2cddb91..7634d0a67 100644 --- a/backend/app/crud/document/document.py +++ b/backend/app/crud/document/document.py @@ -1,11 +1,11 @@ import logging from uuid import UUID +from fastapi import HTTPException from sqlmodel import Session, select, and_ from app.models import Document from app.core.util import now -from app.core.exception_handlers import HTTPException logger = logging.getLogger(__name__) diff --git a/backend/app/crud/evaluations/batch.py b/backend/app/crud/evaluations/batch.py index 7b8892644..3be1dc3e5 100644 --- a/backend/app/crud/evaluations/batch.py +++ b/backend/app/crud/evaluations/batch.py @@ -11,6 +11,8 @@ from typing import Any from langfuse import Langfuse +from langfuse.api import NotFoundError as LangfuseNotFoundError +from langfuse.api.core import ApiError as LangfuseApiError from sqlmodel import Session from app.core.batch import ( @@ -26,6 +28,12 @@ map_kaapi_to_google_params, map_kaapi_to_openai_params, ) +from app.core.exceptions import ( + InvalidPayloadError, + InvalidValueError, + NotFoundError, + UpstreamError, +) from app.services.llm.providers.registry import LLMProvider from app.utils import get_openai_client @@ -44,18 +52,41 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str, List of dataset items with input and expected_output Raises: - ValueError: If dataset not found or empty + NotFoundError, UpstreamError, InvalidPayloadError """ try: dataset = langfuse.get_dataset(dataset_name) - except Exception as e: + except LangfuseNotFoundError as e: logger.warning( - f"[fetch_dataset_items] Failed to fetch dataset | dataset={dataset_name} | {e}" + f"[fetch_dataset_items] Dataset not found | dataset={dataset_name} | {e}" + ) + raise NotFoundError(f"Dataset '{dataset_name}' not found in Langfuse.") + except LangfuseApiError as e: + logger.error( + f"[fetch_dataset_items] Langfuse rejected the request | " + f"dataset={dataset_name}, code={getattr(e, 'status_code', 'unknown')}", + exc_info=True, + ) + raise UpstreamError( + f"Langfuse could not return dataset '{dataset_name}' " + f"(code: {getattr(e, 'status_code', 'unknown')}). Retry shortly.", + provider="langfuse", + ) + except Exception: + logger.error( + f"[fetch_dataset_items] Failed to reach Langfuse | dataset={dataset_name}", + exc_info=True, + ) + raise UpstreamError( + f"Could not reach Langfuse to fetch dataset '{dataset_name}'. Retry shortly.", + provider="langfuse", ) - raise ValueError(f"Dataset '{dataset_name}' not found: {e}") if not dataset.items: - raise ValueError(f"Dataset '{dataset_name}' is empty") + logger.warning( + f"[fetch_dataset_items] Dataset is empty | dataset={dataset_name}" + ) + raise InvalidPayloadError(f"Dataset '{dataset_name}' is empty.") items = [] for item in dataset.items: @@ -200,7 +231,7 @@ def start_evaluation_batch( dataset_items=dataset_items, openai_params=mapped_params ) if not jsonl_data: - raise ValueError( + raise InvalidPayloadError( "Evaluation dataset did not produce any JSONL entries (missing questions?)." ) @@ -240,7 +271,7 @@ def start_evaluation_batch( dataset_items=dataset_items, google_params=mapped_params ) if not jsonl_data: - raise ValueError( + raise InvalidPayloadError( "Evaluation dataset did not produce any JSONL entries (missing questions?)." ) @@ -271,7 +302,9 @@ def start_evaluation_batch( ) else: - raise ValueError(f"Unsupported provider for evaluation batches: {provider}") + raise InvalidValueError( + f"Unsupported provider for evaluation batches: {provider}" + ) eval_run.batch_job_id = batch_job.id eval_run.status = "processing" diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index ed15f720e..864d80a7c 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -103,7 +103,10 @@ def create_evaluation_run( session.refresh(eval_run) except Exception as e: session.rollback() - logger.error(f"Failed to create EvaluationRun: {e}", exc_info=True) + logger.error( + f"[create_evaluation_run] Failed to create EvaluationRun: {e}", + exc_info=True, + ) raise logger.info( @@ -146,7 +149,7 @@ def list_evaluation_runs( runs = session.exec(statement).all() logger.info( - f"Found {len(runs)} evaluation runs for org_id={organization_id}, " + f"[list_evaluation_runs] Found {len(runs)} evaluation runs for org_id={organization_id}, " f"project_id={project_id}" ) @@ -183,12 +186,12 @@ def get_evaluation_run_by_id( if eval_run: logger.info( - f"Found evaluation run {evaluation_id}: status={eval_run.status}, " + f"[get_evaluation_run_by_id] Found evaluation run {evaluation_id}: status={eval_run.status}, " f"batch_job_id={eval_run.batch_job_id}" ) else: logger.warning( - f"Evaluation run {evaluation_id} not found or not accessible " + f"[get_evaluation_run_by_id] Evaluation run {evaluation_id} not found or not accessible " f"for org_id={organization_id}, project_id={project_id}" ) @@ -236,7 +239,10 @@ def update_evaluation_run( session.refresh(eval_run) except Exception as e: session.rollback() - logger.error(f"Failed to update EvaluationRun: {e}", exc_info=True) + logger.error( + f"[update_evaluation_run] Failed to update EvaluationRun: {e}", + exc_info=True, + ) raise if should_notify: diff --git a/backend/app/crud/evaluations/cron_utils.py b/backend/app/crud/evaluations/cron_utils.py index 6cfc06a0d..52758db77 100644 --- a/backend/app/crud/evaluations/cron_utils.py +++ b/backend/app/crud/evaluations/cron_utils.py @@ -324,7 +324,8 @@ async def poll_all_pending_evaluations_by_type( except Exception as client_err: logger.error( f"[{fn_name}] Failed to get Gemini client | " - f"org_id={org_id} | project_id={project_id} | error={client_err}" + f"org_id={org_id} | project_id={project_id} | error={client_err}", + exc_info=True, ) for run in project_runs: update_run_fn( diff --git a/backend/app/crud/evaluations/embeddings.py b/backend/app/crud/evaluations/embeddings.py index 5d1458271..552235ab1 100644 --- a/backend/app/crud/evaluations/embeddings.py +++ b/backend/app/crud/evaluations/embeddings.py @@ -102,13 +102,15 @@ def build_embedding_jsonl( ground_truth = result.get("ground_truth", "") if not item_id: - logger.warning("Skipping result with no item_id") + logger.warning("[build_embedding_jsonl] Skipping result with no item_id") continue # Get trace_id from mapping trace_id = trace_id_mapping.get(item_id) if not trace_id: - logger.warning(f"Skipping item {item_id} - no trace_id found") + logger.warning( + f"[build_embedding_jsonl] Skipping item {item_id} - no trace_id found" + ) skipped.append( {"item_id": item_id, "trace_id": None, "reason": "missing_trace_id"} ) @@ -117,7 +119,9 @@ def build_embedding_jsonl( # Empty output/ground_truth can't be embedded; record the reason. if not generated_output or not ground_truth: reason = "empty_output" if not generated_output else "empty_ground_truth" - logger.warning(f"Skipping item {item_id} - {reason}") + logger.warning( + f"[build_embedding_jsonl] Skipping item {item_id} - {reason}" + ) skipped.append({"item_id": item_id, "trace_id": trace_id, "reason": reason}) continue @@ -140,7 +144,7 @@ def build_embedding_jsonl( jsonl_data.append(batch_request) logger.info( - f"Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" + f"[build_embedding_jsonl] Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" ) return jsonl_data, skipped @@ -160,7 +164,9 @@ def parse_embedding_results( - failed_trace_ids: trace_ids whose embedding could not be parsed (flagged embedding_failed downstream). Lines with no trace_id are dropped silently. """ - logger.info(f"Parsing embedding results from {len(raw_results)} lines") + logger.info( + f"[parse_embedding_results] Parsing embedding results from {len(raw_results)} lines" + ) embedding_pairs = [] failed_trace_ids: list[str] = [] @@ -171,7 +177,9 @@ def parse_embedding_results( # Extract BATCH_KEY (which is now the Langfuse trace_id) trace_id = response.get(BATCH_KEY) if not trace_id: - logger.warning(f"Line {line_num}: No {BATCH_KEY} found, skipping") + logger.warning( + f"[parse_embedding_results] Line {line_num}: No {BATCH_KEY} found, skipping" + ) continue # Handle errors in batch processing @@ -189,7 +197,7 @@ def parse_embedding_results( if len(embedding_data) < 2: logger.warning( - f"Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" + f"[parse_embedding_results] Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" ) failed_trace_ids.append(trace_id) continue @@ -214,7 +222,7 @@ def parse_embedding_results( if output_embedding is None or ground_truth_embedding is None: logger.warning( - f"Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " + f"[parse_embedding_results] Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " f"ground_truth={ground_truth_embedding is not None})" ) failed_trace_ids.append(trace_id) @@ -229,13 +237,16 @@ def parse_embedding_results( ) except Exception as e: - logger.error(f"Line {line_num}: Unexpected error: {e}", exc_info=True) + logger.error( + f"[parse_embedding_results] Line {line_num}: Unexpected error: {e}", + exc_info=True, + ) if trace_id: failed_trace_ids.append(trace_id) continue logger.info( - f"Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " + f"[parse_embedding_results] Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " f"| failed={len(failed_trace_ids)}" ) return embedding_pairs, failed_trace_ids @@ -293,7 +304,9 @@ def calculate_average_similarity( "per_item_scores": [...] # Individual scores with trace_ids } """ - logger.info(f"Calculating similarity for {len(embedding_pairs)} pairs") + logger.info( + f"[calculate_average_similarity] Calculating similarity for {len(embedding_pairs)} pairs" + ) if not embedding_pairs: return { @@ -323,12 +336,15 @@ def calculate_average_similarity( except Exception as e: logger.error( - f"Error calculating similarity for trace {pair.get('trace_id')}: {e}" + f"[calculate_average_similarity] Error calculating similarity for trace {pair.get('trace_id')}: {e}", + exc_info=True, ) continue if not similarities: - logger.warning("No valid similarities calculated") + logger.warning( + "[calculate_average_similarity] No valid similarities calculated" + ) return { "cosine_similarity_avg": 0.0, "cosine_similarity_std": 0.0, @@ -347,7 +363,7 @@ def calculate_average_similarity( } logger.info( - f"Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " + f"[calculate_average_similarity] Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " f"std={stats['cosine_similarity_std']:.3f}" ) @@ -384,7 +400,9 @@ def start_embedding_batch( Exception: If any step fails """ try: - logger.info(f"Starting embedding batch for evaluation run {eval_run.id}") + logger.info( + f"[start_embedding_batch] Starting embedding batch for evaluation run {eval_run.id}" + ) # Use default embedding model embedding_model = EMBEDDING_MODEL @@ -440,7 +458,7 @@ def start_embedding_batch( session.refresh(eval_run) logger.info( - f"Successfully started embedding batch: batch_job_id={batch_job.id}, " + f"[start_embedding_batch] Successfully started embedding batch: batch_job_id={batch_job.id}, " f"provider_batch_id={batch_job.provider_batch_id} " f"for evaluation run {eval_run.id} with {batch_job.total_items} items" ) @@ -448,6 +466,9 @@ def start_embedding_batch( return eval_run except Exception as e: - logger.error(f"Failed to start embedding batch: {e}", exc_info=True) + logger.error( + f"[start_embedding_batch] Failed to start embedding batch: {e}", + exc_info=True, + ) # Don't update eval_run status here - let caller decide raise diff --git a/backend/app/crud/evaluations/langfuse.py b/backend/app/crud/evaluations/langfuse.py index 53f6f2473..e5f3e01dd 100644 --- a/backend/app/crud/evaluations/langfuse.py +++ b/backend/app/crud/evaluations/langfuse.py @@ -205,7 +205,8 @@ def _create_single_trace(result: dict[str, Any]) -> tuple[str, str] | None: if getattr(e, "status_code", None) == 429: logger.error( f"[create_langfuse_dataset_run] Langfuse rate limit (429) | " - f"item_id={item_id}" + f"item_id={item_id}", + exc_info=True, ) else: logger.error( @@ -609,7 +610,8 @@ def _fetch_single_trace(trace_id: str) -> TraceData | None: f"[fetch_trace_scores_from_langfuse] Circuit breaker triggered | " f"consecutive_failures={consecutive_failures} | " f"total_failures={total_failures} | " - f"total_traces={len(trace_ids)}" + f"total_traces={len(trace_ids)}", + exc_info=True, ) raise RuntimeError( f"Langfuse API unavailable: {consecutive_failures} consecutive " diff --git a/backend/app/crud/evaluations/processing.py b/backend/app/crud/evaluations/processing.py index 331a9b66f..407c46b36 100644 --- a/backend/app/crud/evaluations/processing.py +++ b/backend/app/crud/evaluations/processing.py @@ -326,7 +326,8 @@ def parse_evaluation_output( except Exception as e: logger.error( - f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}" + f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}", + exc_info=True, ) continue @@ -1199,7 +1200,8 @@ async def poll_all_pending_evaluations(session: Session) -> dict[str, Any]: ) except HTTPException as http_exc: logger.error( - f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}" + f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}", + exc_info=True, ) # Mark all runs in this project as failed due to client configuration error for eval_run in project_runs: diff --git a/backend/app/crud/onboarding.py b/backend/app/crud/onboarding.py index e754414d0..a324a2baa 100644 --- a/backend/app/crud/onboarding.py +++ b/backend/app/crud/onboarding.py @@ -1,7 +1,9 @@ import logging from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session +from app.core.exceptions import ConflictError from app.core.security import encrypt_credentials, get_password_hash from app.crud import ( api_key_manager, @@ -136,7 +138,20 @@ def onboard_project( created_credentials.append(cred_row) - session.commit() + # Pre-checks are read-then-write; a concurrent onboard only collides here. + try: + session.commit() + except IntegrityError as e: + session.rollback() + logger.warning( + f"[onboard_project] Conflicting concurrent onboarding | " + f"org={onboard_in.organization_name}, project={onboard_in.project_name} | {e}" + ) + raise ConflictError( + f"Project '{onboard_in.project_name}' already exists for " + f"organization '{onboard_in.organization_name}'" + ) + cred_ids = [c.id for c in created_credentials] logger.info( diff --git a/backend/app/crud/stt_evaluations/batch.py b/backend/app/crud/stt_evaluations/batch.py index c9ffe61fa..0a5aaf013 100644 --- a/backend/app/crud/stt_evaluations/batch.py +++ b/backend/app/crud/stt_evaluations/batch.py @@ -227,7 +227,8 @@ def _upload_to_gemini(sample: STTSample) -> _UploadResult: except Exception as e: logger.error( f"[start_stt_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}" + f"model: {model}, error: {str(e)}", + exc_info=True, ) if not batch_jobs: diff --git a/backend/app/crud/tts_evaluations/batch.py b/backend/app/crud/tts_evaluations/batch.py index 3a9abb543..fff0757a6 100644 --- a/backend/app/crud/tts_evaluations/batch.py +++ b/backend/app/crud/tts_evaluations/batch.py @@ -136,7 +136,8 @@ def start_tts_evaluation_batch( except Exception as e: logger.error( f"[start_tts_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}" + f"model: {model}, error: {str(e)}", + exc_info=True, ) pending = get_pending_results_for_run( session=session, run_id=run.id, provider=model diff --git a/backend/app/services/collections/providers/gemini.py b/backend/app/services/collections/providers/gemini.py index cdf2e50c2..ab880e513 100644 --- a/backend/app/services/collections/providers/gemini.py +++ b/backend/app/services/collections/providers/gemini.py @@ -151,6 +151,7 @@ def upload_files( doc.id, uploaded.name, str(delete_err), + exc_info=True, ) doc.file_id.pop(GOOGLE_AISTUDIO_PROVIDER, None) raise diff --git a/backend/app/services/documents/helpers.py b/backend/app/services/documents/helpers.py index 7d78b6160..db86e49d3 100644 --- a/backend/app/services/documents/helpers.py +++ b/backend/app/services/documents/helpers.py @@ -56,12 +56,13 @@ def pre_transform_validation( - resolve actual transformer (or None if no target_format) Returns: (source_format, actual_transformer_or_none) - Raises: HTTPException(400) on client errors. + Raises: HTTPException(422) on an unreadable source format, 400 on an + unsupported transformation or transformer. """ try: source_format = get_file_format(src_filename) except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=422, detail=str(e)) actual_transformer: Optional[str] = None if target_format: diff --git a/backend/app/services/evaluations/dataset.py b/backend/app/services/evaluations/dataset.py index e1603a605..bb9abf472 100644 --- a/backend/app/services/evaluations/dataset.py +++ b/backend/app/services/evaluations/dataset.py @@ -6,6 +6,7 @@ from sqlmodel import Session from app.core.cloud import get_cloud_storage +from app.core.exceptions import UpstreamError from app.crud.evaluations import ( create_evaluation_dataset, upload_csv_to_object_store, @@ -134,13 +135,14 @@ def upload_dataset( f"dataset={dataset_name} | id={langfuse_dataset_id}" ) - except Exception as e: + except Exception: logger.error( - f"[upload_dataset] Failed to upload dataset to Langfuse | {e}", + f"[upload_dataset] Failed to upload dataset to Langfuse | dataset={dataset_name}", exc_info=True, ) - raise HTTPException( - status_code=500, detail=f"Failed to upload dataset to Langfuse: {e}" + raise UpstreamError( + f"Langfuse could not accept dataset '{dataset_name}'. Retry shortly.", + provider="langfuse", ) # Step 5: Store metadata in database diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index 20b0e94c6..dcb56c8f1 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -21,6 +21,7 @@ from app.core.cloud import get_cloud_storage from app.core.config import settings from app.core.db import engine +from app.core.exceptions import InvalidPayloadError, KaapiError from app.crud.evaluations import ( get_dataset_by_id, resolve_evaluation_config, @@ -55,6 +56,17 @@ ERR_DATASET_TOO_LARGE_FOR_FAST = "dataset_too_large_for_fast" +def _fail_run(*, session: Session, eval_run: EvaluationRun, reason: str) -> None: + update_evaluation_run( + session=session, + eval_run=eval_run, + update=EvaluationRunUpdate( + status="failed", + error_message=f"Failed to start fast eval: {reason}", + ), + ) + + def is_dataset_fast_eligible(*, original_items_count: int) -> bool: """A dataset is eligible for fast mode when its unique-row count is within cap.""" return original_items_count <= settings.EVAL_FAST_MAX_UNIQUE_ROWS @@ -287,7 +299,7 @@ def validate_and_start_fast_evaluation( ) total_items = len(dataset_items) if total_items == 0: - raise ValueError(f"Dataset '{dataset.name}' returned no items") + raise InvalidPayloadError(f"Dataset '{dataset.name}' returned no items") n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) # total_items isn't on EvaluationRunUpdate; set it directly, then flip to @@ -310,20 +322,22 @@ def validate_and_start_fast_evaluation( f"eval_run_id={eval_run.id} | total_items={total_items} | " f"n_chunks={n_chunks}" ) + except KaapiError as exc: + log = logger.error if exc.status_code >= 500 else logger.warning + log( + f"[validate_and_start_fast_evaluation] Failed to start run | " + f"eval_run_id={eval_run.id} | code={exc.status_code}", + exc_info=True, + ) + _fail_run(session=session, eval_run=eval_run, reason=exc.detail) + raise except Exception as exc: logger.error( f"[validate_and_start_fast_evaluation] Failed to start run | " f"eval_run_id={eval_run.id} | error={exc}", exc_info=True, ) - update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate( - status="failed", - error_message=f"Failed to start fast eval: {exc}", - ), - ) + _fail_run(session=session, eval_run=eval_run, reason=str(exc)) raise HTTPException( status_code=500, detail="Failed to start fast evaluation", diff --git a/backend/app/services/guardrails/jobs.py b/backend/app/services/guardrails/jobs.py index 95f97c923..724caff96 100644 --- a/backend/app/services/guardrails/jobs.py +++ b/backend/app/services/guardrails/jobs.py @@ -3,12 +3,12 @@ from uuid import UUID, uuid4 from asgi_correlation_id import correlation_id -from fastapi import HTTPException from opentelemetry import trace from sqlmodel import Session from app.celery.utils import start_guardrails_job from app.core.db import engine +from app.core.exceptions import ServiceUnavailableError from app.core.telemetry import log_context from app.crud.jobs import JobCrud from app.models import Job, JobStatus, JobType, JobUpdate @@ -84,9 +84,8 @@ def start_job( job_id=job.id, job_update=JobUpdate(status=JobStatus.FAILED, error_message=str(e)), ) - raise HTTPException( - status_code=500, - detail="Internal server error while scheduling guardrails job", + raise ServiceUnavailableError( + "Could not queue the guardrails job. Retry shortly." ) logger.info( diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index e95ec9cb1..514d689d6 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -17,6 +17,7 @@ from app.celery.utils import start_llm_chain_job, start_llm_job from app.core.db import engine +from app.core.exceptions import ServiceUnavailableError from app.core.providers import Provider from app.core.langfuse.langfuse import observe_llm_execution from app.core.telemetry import ( @@ -178,8 +179,8 @@ def start_job( ) job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) - raise HTTPException( - status_code=500, detail="Internal server error while executing LLM call" + raise ServiceUnavailableError( + "Could not queue the LLM call. Retry shortly." ) _set_traceability_attributes(span, task_id=str(task_id)) @@ -238,9 +239,8 @@ def start_chain_job( ) job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) - raise HTTPException( - status_code=500, - detail="Internal server error while executing LLM chain job", + raise ServiceUnavailableError( + "Could not queue the LLM chain job. Retry shortly." ) _set_traceability_attributes(span, task_id=str(task_id)) diff --git a/backend/app/services/stt_evaluations/audio.py b/backend/app/services/stt_evaluations/audio.py index 8f890d7e9..13d9aed9f 100644 --- a/backend/app/services/stt_evaluations/audio.py +++ b/backend/app/services/stt_evaluations/audio.py @@ -8,7 +8,8 @@ from sqlmodel import Session from app.core.cloud.storage import get_cloud_storage -from app.core.exception_handlers import HTTPException +from app.core.exceptions import KaapiError, UpstreamError +from fastapi import HTTPException from app.crud.file import create_file from app.models.file import FileType, AudioUploadResponse from app.services.stt_evaluations.constants import ( @@ -116,14 +117,15 @@ def upload_audio_file( content_type=content_type, ) - except HTTPException: + except (HTTPException, KaapiError): raise except Exception as e: logger.error( f"[upload_audio_file] Failed to upload audio | " - f"project_id: {project_id}, error: {str(e)}" + f"project_id: {project_id}, error: {str(e)}", + exc_info=True, ) - raise HTTPException( - status_code=500, - detail="Failed to upload audio file. Please try again later.", + raise UpstreamError( + "Object storage could not accept the audio file. Retry shortly.", + provider="object-storage", ) diff --git a/backend/app/tests/api/routes/documents/test_route_document_upload.py b/backend/app/tests/api/routes/documents/test_route_document_upload.py index 909249a42..2eca7ac5c 100644 --- a/backend/app/tests/api/routes/documents/test_route_document_upload.py +++ b/backend/app/tests/api/routes/documents/test_route_document_upload.py @@ -320,7 +320,7 @@ def test_upload_with_unsupported_file_extension( try: response = uploader.put(route, unsupported_file, target_format="markdown") - assert response.status_code == 400 + assert response.status_code == 422 error_data = response.json() assert "Unsupported file extension: .xyz" in error_data["error"] finally: diff --git a/backend/app/tests/api/routes/test_creds.py b/backend/app/tests/api/routes/test_creds.py index c5fd9c1ea..f171fbf1a 100644 --- a/backend/app/tests/api/routes/test_creds.py +++ b/backend/app/tests/api/routes/test_creds.py @@ -237,7 +237,7 @@ def test_duplicate_credential_creation_fails( client: TestClient, user_api_key: TestAuthContext, ) -> None: - """Test that creating duplicate credentials fails with 400.""" + """Test that creating duplicate credentials fails with 409.""" api_key = "sk-" + generate_random_string(10) duplicate_credential = { "organization_id": user_api_key.organization_id, @@ -257,7 +257,7 @@ def test_duplicate_credential_creation_fails( headers={"X-API-KEY": user_api_key.key}, ) - assert response.status_code == 400 + assert response.status_code == 409 assert "already exist" in response.json()["error"] diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index 18f0dfa72..7d9a3cc2b 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient from sqlmodel import Session, select +from app.core.exceptions import InvalidPayloadError, InvalidValueError from app.core.util import now from app.crud.evaluations.batch import ( build_google_evaluation_jsonl, @@ -439,13 +440,13 @@ def test_upload_with_duplication_factor_boundary_minimum( class TestDatasetUploadErrors: """Test error handling.""" - def test_upload_returns_500_when_langfuse_upload_raises( + def test_upload_returns_502_when_langfuse_upload_raises( self, client: TestClient, user_api_key_header: dict[str, str], valid_csv_content: str, ) -> None: - """Tracing enabled but the Langfuse upload fails → 500.""" + """Tracing enabled but the Langfuse upload fails → 502.""" with ( patch("app.core.cloud.get_cloud_storage") as _mock_storage, patch( @@ -474,12 +475,12 @@ def test_upload_returns_500_when_langfuse_upload_raises( headers=user_api_key_header, ) - assert response.status_code == 500, response.text + assert response.status_code == 502, response.text response_data = response.json() error_str = response_data.get( "detail", response_data.get("error", str(response_data)) ) - assert "Failed to upload dataset to Langfuse" in str(error_str) + assert "Langfuse could not accept dataset" in str(error_str) def test_upload_invalid_csv_format( self, client: TestClient, user_api_key_header: dict[str, str] @@ -1133,7 +1134,7 @@ def test_unsupported_provider_marks_failed( ) -> None: mock_fetch.return_value = _BATCH_DATASET_ITEMS - with pytest.raises(ValueError, match="Unsupported provider"): + with pytest.raises(InvalidValueError, match="Unsupported provider"): start_evaluation_batch( langfuse=MagicMock(), session=db, @@ -1157,7 +1158,7 @@ def test_empty_jsonl_marks_failed( ] mock_map.return_value = ({"model": "gpt-4o"}, []) - with pytest.raises(ValueError, match="did not produce any JSONL"): + with pytest.raises(InvalidPayloadError, match="did not produce any JSONL"): start_evaluation_batch( langfuse=MagicMock(), session=db, diff --git a/backend/app/tests/api/routes/test_tts_evaluation.py b/backend/app/tests/api/routes/test_tts_evaluation.py index ae9ade1c8..0e3957c53 100644 --- a/backend/app/tests/api/routes/test_tts_evaluation.py +++ b/backend/app/tests/api/routes/test_tts_evaluation.py @@ -626,7 +626,7 @@ def test_start_evaluation_celery_failure( headers=user_api_key_header, ) - assert response.status_code == 500 + assert response.status_code == 503 assert "queue" in response.json()["error"].lower() def test_start_evaluation_invalid_dataset( @@ -674,7 +674,7 @@ def test_start_evaluation_empty_dataset( headers=user_api_key_header, ) - assert response.status_code == 400 + assert response.status_code == 422 assert "no samples" in response.json()["error"].lower() def test_start_evaluation_unsupported_model( diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index 4955136be..2449ba8e5 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,7 +1,22 @@ +import pytest +from fastapi import FastAPI from fastapi.testclient import TestClient from app.core.config import settings -from app.core.exception_handlers import _sanitize_validation_errors +from app.core.exception_handlers import ( + GENERIC_ERROR_DETAIL, + _sanitize_validation_errors, + register_exception_handlers, +) +from app.core.exceptions import ( + ConflictError, + InvalidPayloadError, + InvalidValueError, + KaapiError, + NotFoundError, + ServiceUnavailableError, + UpstreamError, +) from app.tests.utils.auth import TestAuthContext @@ -138,3 +153,47 @@ def test_union_noise_filtered( for error in response.json()["errors"]: assert "openai-native" not in error["message"] assert "NativeCompletionConfig" not in error["field"] + + +def _app_raising(exc: Exception) -> TestClient: + app = FastAPI() + register_exception_handlers(app) + + @app.get("/boom") + def boom() -> None: + raise exc + + return TestClient(app, raise_server_exceptions=False) + + +class TestKaapiErrorHandler: + """Each domain exception surfaces as its own status code.""" + + @pytest.mark.parametrize( + "exc, expected", + [ + (NotFoundError("gone"), 404), + (ConflictError("dupe"), 409), + (InvalidValueError("bad value"), 400), + (InvalidPayloadError("bad shape"), 422), + (UpstreamError("langfuse down", provider="langfuse"), 502), + (ServiceUnavailableError("broker down"), 503), + (KaapiError("our bug"), 500), + ], + ) + def test_status_code_and_body(self, exc: KaapiError, expected: int) -> None: + response = _app_raising(exc).get("/boom") + + assert response.status_code == expected + body = response.json() + assert body["success"] is False + assert body["error"] == exc.detail + + +class TestGenericErrorHandler: + def test_does_not_leak_exception_text(self) -> None: + response = _app_raising(ValueError("secret arn:aws:kms:key/abc")).get("/boom") + + assert response.status_code == 500 + assert response.json()["error"] == GENERIC_ERROR_DETAIL + assert "arn:aws:kms" not in response.text diff --git a/backend/app/tests/core/test_security.py b/backend/app/tests/core/test_security.py index 412c48c25..5d844479c 100644 --- a/backend/app/tests/core/test_security.py +++ b/backend/app/tests/core/test_security.py @@ -1,4 +1,5 @@ from datetime import timedelta +from unittest.mock import MagicMock import boto3 import jwt @@ -8,6 +9,7 @@ import app.core.security as security from app.core.config import settings +from app.core.exceptions import UpstreamError from app.core.security import ( ALGORITHM, KMS_CIPHERTEXT_PREFIX, @@ -78,6 +80,32 @@ def test_dual_read_fernet_row_with_kms_active(self, monkeypatch, kms_key): monkeypatch.setattr(settings, "ENVIRONMENT", "staging") assert decrypt_credentials(fernet_encrypted) == creds + def test_kms_encrypt_failure_raises_502_without_leaking_cause( + self, monkeypatch, kms_key + ): + broken = MagicMock() + broken.encrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") + monkeypatch.setattr(security, "_kms_client", broken) + + with pytest.raises(UpstreamError) as exc: + encrypt_credentials({"api_key": "sk-1"}) + + assert exc.value.status_code == 502 + assert exc.value.provider == "kms" + assert "arn:aws:kms" not in exc.value.detail + + def test_kms_decrypt_failure_raises_502(self, monkeypatch, kms_key): + encrypted = encrypt_credentials({"api_key": "sk-1"}) + broken = MagicMock() + broken.decrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") + monkeypatch.setattr(security, "_kms_client", broken) + + with pytest.raises(UpstreamError) as exc: + decrypt_credentials(encrypted) + + assert exc.value.status_code == 502 + assert "arn:aws:kms" not in exc.value.detail + class TestAPIKeyManager: """Test suite for APIKeyManager class.""" diff --git a/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py new file mode 100644 index 000000000..aeb56d6dd --- /dev/null +++ b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py @@ -0,0 +1,58 @@ +"""Deleting an already-deleted collection is a conflict, not bad input.""" + +import uuid + +import pytest +from sqlmodel import Session + +from fastapi import HTTPException + +from app.core.exceptions import ConflictError +from app.core.util import now +from app.crud import CollectionCrud +from app.models import Collection, ProviderType +from app.tests.utils.utils import get_project + + +def _make_collection(db: Session, project_id: int, deleted: bool) -> Collection: + collection = Collection( + project_id=project_id, + knowledge_base_id="vs_test", + knowledge_base_provider="openai vector store", + provider=ProviderType.openai, + deleted_at=now() if deleted else None, + ) + db.add(collection) + db.commit() + db.refresh(collection) + return collection + + +def test_already_deleted_raises_409(db: Session) -> None: + project = get_project(db) + collection = _make_collection(db, project.id, deleted=True) + + with pytest.raises(ConflictError) as exc: + CollectionCrud(db, project.id).read_one_if_delete(collection.id) + + assert exc.value.status_code == 409 + assert "already deleted" in exc.value.detail + + +def test_active_collection_is_returned(db: Session) -> None: + project = get_project(db) + collection = _make_collection(db, project.id, deleted=False) + + assert ( + CollectionCrud(db, project.id).read_one_if_delete(collection.id).id + == collection.id + ) + + +def test_missing_collection_still_raises_404(db: Session) -> None: + project = get_project(db) + + with pytest.raises(HTTPException) as exc: + CollectionCrud(db, project.id).read_one_if_delete(uuid.uuid4()) + + assert exc.value.status_code == 404 diff --git a/backend/app/tests/crud/evaluations/test_batch_errors.py b/backend/app/tests/crud/evaluations/test_batch_errors.py new file mode 100644 index 000000000..b91973269 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_batch_errors.py @@ -0,0 +1,54 @@ +"""Langfuse failures map to distinct status codes rather than a flat 500.""" + +from unittest.mock import MagicMock + +import pytest +from langfuse.api import NotFoundError as LangfuseNotFoundError +from langfuse.api.core import ApiError as LangfuseApiError + +from app.core.exceptions import InvalidPayloadError, NotFoundError, UpstreamError +from app.crud.evaluations.batch import fetch_dataset_items + + +def test_dataset_missing_raises_404() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = LangfuseNotFoundError( + body={"message": "Dataset not found"} + ) + + with pytest.raises(NotFoundError) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="missing") + + assert exc.value.status_code == 404 + assert "missing" in exc.value.detail + + +def test_langfuse_api_error_raises_502() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = LangfuseApiError(status_code=500, body="boom") + + with pytest.raises(UpstreamError) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 502 + assert exc.value.provider == "langfuse" + + +def test_langfuse_unreachable_raises_502() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = ConnectionError("dns failure") + + with pytest.raises(UpstreamError) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 502 + + +def test_empty_dataset_raises_422() -> None: + langfuse = MagicMock() + langfuse.get_dataset.return_value = MagicMock(items=[]) + + with pytest.raises(InvalidPayloadError) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 422 diff --git a/backend/app/tests/crud/test_credentials.py b/backend/app/tests/crud/test_credentials.py index 8b3269ca6..79d9df6fa 100644 --- a/backend/app/tests/crud/test_credentials.py +++ b/backend/app/tests/crud/test_credentials.py @@ -1,4 +1,7 @@ +from unittest.mock import MagicMock + import pytest +from fastapi import HTTPException from sqlmodel import Session from app.crud import ( @@ -10,6 +13,7 @@ remove_creds_for_org, ) from app.models import CredsCreate, CredsUpdate +from app.core.exceptions import ConflictError, NotFoundError from app.core.providers import Provider from app.tests.utils.test_data import ( create_test_project, @@ -353,3 +357,73 @@ def test_langfuse_credential_validation(db: Session) -> None: ) assert len(created_credentials) == 1 assert created_credentials[0].provider == "langfuse" + + +def test_remove_provider_credential_missing_raises_404(db: Session) -> None: + """Deleting a credential that was never stored is a 404, not a 500.""" + project = create_test_project(db) + + with pytest.raises(HTTPException) as exc: + remove_provider_credential( + session=db, + org_id=project.organization_id, + provider="openai", + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_remove_provider_credential_race_deletes_nothing_raises_404( + db: Session, monkeypatch +) -> None: + """Row vanishes between the existence check and the delete: 404, not 500.""" + _, project = create_test_credential(db) + monkeypatch.setattr( + "app.crud.credentials.get_provider_credential", lambda **k: MagicMock() + ) + monkeypatch.setattr(db, "exec", lambda *a, **k: MagicMock(rowcount=0)) + + with pytest.raises(NotFoundError) as exc: + remove_provider_credential( + session=db, + org_id=project.organization_id, + provider="openai", + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_remove_creds_for_org_when_none_exist_raises_404(db: Session) -> None: + project = create_test_project(db) + + with pytest.raises(HTTPException) as exc: + remove_creds_for_org( + session=db, + org_id=project.organization_id, + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_duplicate_provider_raises_409(db: Session) -> None: + """Unique-constraint violation is a conflict, not bad input.""" + _, project = create_test_credential(db) + + creds_create = CredsCreate( + is_active=True, + credential={"openai": {"api_key": "another-key"}}, + ) + + with pytest.raises(ConflictError) as exc: + set_creds_for_org( + session=db, + creds_add=creds_create, + organization_id=project.organization_id, + project_id=project.id, + ) + + assert exc.value.status_code == 409 + assert "already exist" in exc.value.detail diff --git a/backend/app/tests/crud/test_onboarding.py b/backend/app/tests/crud/test_onboarding.py index 13f652bfa..6f9ffd835 100644 --- a/backend/app/tests/crud/test_onboarding.py +++ b/backend/app/tests/crud/test_onboarding.py @@ -1,7 +1,10 @@ import pytest from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select +from app.core.exceptions import ConflictError, UpstreamError + from app.crud.onboarding import onboard_project from app.crud import ( get_organization_by_name, @@ -289,3 +292,51 @@ def test_onboard_project_response_data_integrity(db: Session) -> None: assert project.name == response.project_name assert project.organization_id == response.organization_id assert user.email == response.user_email + + +def test_onboard_kms_failure_raises_502(db: Session, monkeypatch) -> None: + """A KMS outage while encrypting credentials is upstream, not our bug.""" + monkeypatch.setattr( + "app.crud.onboarding.encrypt_credentials", + lambda _: (_ for _ in ()).throw(UpstreamError("kms down", provider="kms")), + ) + + with pytest.raises(UpstreamError) as exc: + onboard_project( + session=db, + onboard_in=OnboardingRequest( + organization_name=random_lower_string(), + project_name=random_lower_string(), + email=random_email(), + user_name=random_lower_string(), + password=random_lower_string(), + credentials=[{"openai": {"api_key": "sk-test"}}], + ), + ) + + assert exc.value.status_code == 502 + + +def test_onboard_concurrent_commit_conflict_raises_409( + db: Session, monkeypatch +) -> None: + """The pre-checks are racy; a collision at commit is a 409, not a 500.""" + + def _raise_integrity_error() -> None: + raise IntegrityError("INSERT", {}, Exception("duplicate key")) + + monkeypatch.setattr(db, "commit", _raise_integrity_error) + + with pytest.raises(ConflictError) as exc: + onboard_project( + session=db, + onboard_in=OnboardingRequest( + organization_name=random_lower_string(), + project_name=random_lower_string(), + email=random_email(), + user_name=random_lower_string(), + password=random_lower_string(), + ), + ) + + assert exc.value.status_code == 409 diff --git a/backend/app/tests/services/guardrails/test_jobs.py b/backend/app/tests/services/guardrails/test_jobs.py index 84bca8290..648e171e9 100644 --- a/backend/app/tests/services/guardrails/test_jobs.py +++ b/backend/app/tests/services/guardrails/test_jobs.py @@ -3,7 +3,8 @@ from uuid import uuid4 import pytest -from fastapi import HTTPException + +from app.core.exceptions import ServiceUnavailableError from sqlmodel import Session, select from app.models import Job, JobStatus, JobType @@ -143,12 +144,12 @@ def test_start_job_happy_path_creates_row_and_enqueues( assert kwargs["job_id"] == str(job.id) -def test_start_job_celery_failure_marks_failed_and_raises_500( +def test_start_job_celery_failure_marks_failed_and_raises_503( db: Session, user_api_key: TestAuthContext ) -> None: with patch("app.services.guardrails.jobs.start_guardrails_job") as mock_enqueue: mock_enqueue.side_effect = RuntimeError("broker down") - with pytest.raises(HTTPException) as exc: + with pytest.raises(ServiceUnavailableError) as exc: start_job( db=db, request=_make_request(), @@ -156,7 +157,7 @@ def test_start_job_celery_failure_marks_failed_and_raises_500( organization_id=user_api_key.organization_id, ) - assert exc.value.status_code == 500 + assert exc.value.status_code == 503 rows = db.exec(select(Job).where(Job.project_id == user_api_key.project_id)).all() failed = [r for r in rows if r.status == JobStatus.FAILED] diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index 2f56b14a0..fffb40bc4 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -5,7 +5,8 @@ from celery.exceptions import SoftTimeLimitExceeded from gevent import Timeout -from fastapi import HTTPException + +from app.core.exceptions import ServiceUnavailableError from sqlmodel import Session, select from app.crud import JobCrud @@ -95,13 +96,11 @@ def test_start_job_celery_scheduling_fails( with patch("app.services.llm.jobs.start_llm_job") as mock_schedule: mock_schedule.side_effect = Exception("Celery connection failed") - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(ServiceUnavailableError) as exc_info: start_job(db, llm_call_request, project.id, project.organization_id) - assert exc_info.value.status_code == 500 - assert "Internal server error while executing LLM call" in str( - exc_info.value.detail - ) + assert exc_info.value.status_code == 503 + assert "Could not queue the LLM call" in str(exc_info.value.detail) def test_start_job_exception_during_job_creation( self, db: Session, llm_call_request: LLMCallRequest @@ -2006,13 +2005,11 @@ def test_start_chain_job_celery_failure(self, db: Session, chain_request): mock_job.id = uuid4() mock_job_crud_class.return_value.create.return_value = mock_job - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(ServiceUnavailableError) as exc_info: start_chain_job(db, chain_request, project.id, project.organization_id) - assert exc_info.value.status_code == 500 - assert "Internal server error while executing LLM chain job" in str( - exc_info.value.detail - ) + assert exc_info.value.status_code == 503 + assert "Could not queue the LLM chain job" in str(exc_info.value.detail) class TestExecuteChainJob: diff --git a/backend/app/tests/services/stt_evaluations/test_audio.py b/backend/app/tests/services/stt_evaluations/test_audio.py index a1c1d2342..3c84b717f 100644 --- a/backend/app/tests/services/stt_evaluations/test_audio.py +++ b/backend/app/tests/services/stt_evaluations/test_audio.py @@ -5,7 +5,9 @@ import pytest from fastapi import UploadFile -from app.core.exception_handlers import HTTPException +from fastapi import HTTPException + +from app.core.exceptions import UpstreamError from app.services.stt_evaluations.audio import ( _resolve_extension, _validate_audio_file, @@ -178,15 +180,15 @@ def test_upload_storage_error(self, mock_get_storage) -> None: mock_storage.put.side_effect = Exception("S3 connection failed") mock_get_storage.return_value = mock_storage - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(UpstreamError) as exc_info: upload_audio_file( session=MagicMock(), file=_make_upload_file(), organization_id=1, project_id=1, ) - assert exc_info.value.status_code == 500 - assert "Failed to upload audio file" in str(exc_info.value.detail) + assert exc_info.value.status_code == 502 + assert "Object storage could not accept" in str(exc_info.value.detail) @patch("app.services.stt_evaluations.audio.create_file") @patch("app.services.stt_evaluations.audio.get_cloud_storage") diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index 99da15713..aebc88ce8 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -3,9 +3,27 @@ All paths relative to `backend/app/`. ## Global handlers -- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. +- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. - `core/middleware.py` — request-level middleware. +## Domain exceptions +- `core/exceptions.py` — `KaapiError` base plus `NotFoundError` (404), `ConflictError` (409), `InvalidValueError` (400), `InvalidPayloadError` (422), `UpstreamError` (502, takes `provider=`), `ServiceUnavailableError` (503). Handled by `kaapi_error_handler`, which reads `status_code` off the class. +- Never raise a bare `ValueError` from crud/service for a caller-facing failure: nothing catches it, so it becomes a 500 with a generic body. Use one of the above. +- `KaapiError` is the preferred raise in crud/service (it keeps the layer transport-agnostic), but the migration is partial: ~78 `raise HTTPException` sites remain in `crud/` and are still correct as long as the code is right. Prefer `KaapiError` in new code; convert neighbours opportunistically rather than mixing both idioms inside one function. +- Code selection, applied across evaluations, knowledge-base, llm-call, tenancy, and platform: + +| Condition | Code | +|---|---| +| Resource absent (ours, or one we reference upstream) | 404 | +| State conflict (already exists, already deleted) | 409 | +| Payload unparseable or wrong shape | 422 | +| Valid shape, unacceptable value | 400 | +| Upstream errored or unreachable (Langfuse, OpenAI, Gemini, KMS, object storage) | 502 | +| Our async infra could not accept the work (Celery broker) | 503 | +| Our bug (DB write failed, unexpected exception) | 500 | + +- A broad `except Exception` around a DB write must re-raise `HTTPException`/`KaapiError` first, otherwise typed errors get flattened to 500. + ## Provider/SDK error convention - `.claude/conventions/error-handling.md` — the standardized pattern for wrapping provider/SDK exceptions at service/crud call sites. Follow it for any new OpenAI/Gemini/Anthropic call site. From 64cafc57204105a6ea31907d835cb70eb290422e Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:48:58 +0530 Subject: [PATCH 02/12] Refactor exception handling to use HTTPException for consistent status codes - Replaced custom exceptions (InvalidPayloadError, ServiceUnavailableError, etc.) with HTTPException across various services and CRUD operations to standardize error responses. - Updated the exception handling logic in evaluations, guardrails, LLM jobs, STT evaluations, and onboarding to raise HTTPException with appropriate status codes. - Removed references to custom exceptions in tests and replaced them with HTTPException assertions. - Added a new exception taxonomy in app/core/exceptions.py to categorize domain-specific errors. - Updated documentation to reflect changes in exception handling and status code consistency. --- backend/app/api/routes/collections.py | 11 +- .../api/routes/stt_evaluations/evaluation.py | 6 +- .../api/routes/tts_evaluations/evaluation.py | 6 +- backend/app/core/exception_handlers.py | 8 - backend/app/core/exceptions.py | 43 ---- backend/app/core/security.py | 10 +- backend/app/crud/collection/collection.py | 9 +- backend/app/crud/config/config.py | 8 +- backend/app/crud/config/version.py | 8 +- backend/app/crud/credentials.py | 20 +- backend/app/crud/evaluations/batch.py | 45 ++-- backend/app/crud/onboarding.py | 8 +- backend/app/services/evaluations/dataset.py | 7 +- backend/app/services/evaluations/fast.py | 7 +- backend/app/services/guardrails/jobs.py | 7 +- backend/app/services/llm/jobs.py | 10 +- backend/app/services/stt_evaluations/audio.py | 9 +- .../app/tests/api/routes/test_evaluation.py | 5 +- .../app/tests/core/test_exception_handlers.py | 34 --- backend/app/tests/core/test_security.py | 7 +- ...test_crud_collection_read_one_if_delete.py | 3 +- .../crud/evaluations/test_batch_errors.py | 11 +- backend/app/tests/crud/test_credentials.py | 5 +- backend/app/tests/crud/test_onboarding.py | 9 +- .../tests/services/guardrails/test_jobs.py | 3 +- backend/app/tests/services/llm/test_jobs.py | 5 +- .../services/stt_evaluations/test_audio.py | 3 +- docs/wiki/cross-cutting/exceptions.md | 9 +- features/status-code-consistency/PLAN.md | 195 ++++++++++++++++++ 29 files changed, 312 insertions(+), 199 deletions(-) delete mode 100644 backend/app/core/exceptions.py create mode 100644 features/status-code-consistency/PLAN.md diff --git a/backend/app/api/routes/collections.py b/backend/app/api/routes/collections.py index ae83e67dc..4487fcd3a 100644 --- a/backend/app/api/routes/collections.py +++ b/backend/app/api/routes/collections.py @@ -2,10 +2,11 @@ from uuid import UUID from typing import List -from fastapi import APIRouter, Query, Body, Depends +from fastapi import APIRouter, HTTPException, Query, Body, Depends from fastapi import Path as FastPath from app.api.deps import SessionDep, AuthContextDep +from app.crud.collection.collection import CollectionNameConflictError from app.api.permissions import Permission, require_permission from app.core.telemetry import log_context from app.core.rate_monitor import monitor_rate @@ -214,7 +215,13 @@ def update_collection( organization_id=current_user.organization_.id, ): collection_crud = CollectionCrud(session, current_user.project_.id) - collection = collection_crud.update(collection_id, patch) + try: + collection = collection_crud.update(collection_id, patch) + except CollectionNameConflictError as err: + raise HTTPException( + status_code=409, + detail=f"Collection '{err.name}' already exists. Choose a different name.", + ) logger.info( f"[update_collection] Collection updated | {{'collection_id': '{collection_id}'}}" diff --git a/backend/app/api/routes/stt_evaluations/evaluation.py b/backend/app/api/routes/stt_evaluations/evaluation.py index 9495ef3f1..c5dc34283 100644 --- a/backend/app/api/routes/stt_evaluations/evaluation.py +++ b/backend/app/api/routes/stt_evaluations/evaluation.py @@ -9,7 +9,6 @@ from app.api.permissions import Permission, require_permission from app.celery.utils import start_stt_batch_submission from app.core.cloud import get_cloud_storage -from app.core.exceptions import ServiceUnavailableError from app.crud.stt_evaluations import ( create_stt_run, get_results_by_run_id, @@ -107,8 +106,9 @@ def start_stt_evaluation( status="failed", error_message=f"Failed to queue batch submission: {str(e)}", ) - raise ServiceUnavailableError( - "Could not queue the batch submission. Retry shortly." + raise HTTPException( + status_code=503, + detail="Could not queue the batch submission. Retry shortly.", ) return APIResponse.success_response( diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index 145bc0f7c..e27d39666 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -9,7 +9,6 @@ from app.api.permissions import Permission, require_permission from app.celery.utils import start_tts_batch_submission from app.core.cloud import get_cloud_storage -from app.core.exceptions import ServiceUnavailableError from app.crud.tts_evaluations import ( create_tts_run, get_results_by_run_id, @@ -111,8 +110,9 @@ def start_tts_evaluation( status="failed", error_message=f"Failed to queue batch submission: {str(e)}", ) - raise ServiceUnavailableError( - "Could not queue the batch submission. Retry shortly." + raise HTTPException( + status_code=503, + detail="Could not queue the batch submission. Retry shortly.", ) return APIResponse.success_response( diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 31b62291e..ed8665a48 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -11,7 +11,6 @@ HTTP_500_INTERNAL_SERVER_ERROR, ) -from app.core.exceptions import KaapiError from app.utils import APIResponse logger = logging.getLogger(__name__) @@ -114,13 +113,6 @@ async def http_exception_handler( content=APIResponse.failure_response(detail).model_dump(), ) - @app.exception_handler(KaapiError) - async def kaapi_error_handler(request: Request, exc: KaapiError) -> JSONResponse: - return JSONResponse( - status_code=exc.status_code, - content=APIResponse.failure_response(exc.detail).model_dump(), - ) - @app.exception_handler(Exception) async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. diff --git a/backend/app/core/exceptions.py b/backend/app/core/exceptions.py deleted file mode 100644 index 854cf68c3..000000000 --- a/backend/app/core/exceptions.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Domain exceptions carrying the status code they surface as. - -Raise from crud/service instead of a bare `ValueError`, which falls through to the -generic 500 handler. Mapped to responses by `app.core.exception_handlers`. -""" - -from starlette import status - - -class KaapiError(Exception): - status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR - - def __init__(self, detail: str) -> None: - super().__init__(detail) - self.detail = detail - - -class NotFoundError(KaapiError): - status_code = status.HTTP_404_NOT_FOUND - - -class ConflictError(KaapiError): - status_code = status.HTTP_409_CONFLICT - - -class InvalidValueError(KaapiError): - status_code = status.HTTP_400_BAD_REQUEST - - -class InvalidPayloadError(KaapiError): - status_code = status.HTTP_422_UNPROCESSABLE_CONTENT - - -class UpstreamError(KaapiError): - status_code = status.HTTP_502_BAD_GATEWAY - - def __init__(self, detail: str, provider: str) -> None: - super().__init__(detail) - self.provider = provider - - -class ServiceUnavailableError(KaapiError): - status_code = status.HTTP_503_SERVICE_UNAVAILABLE diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 977f91596..ad1393da5 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -10,6 +10,7 @@ import base64 import json import logging +from fastapi import HTTPException import os import secrets from datetime import UTC, datetime, timedelta @@ -26,7 +27,6 @@ from sqlmodel import Session, and_, select from app.core.config import settings -from app.core.exceptions import UpstreamError from app.models import APIKey, AuthContext, Organization, Project, User logger = logging.getLogger(__name__) @@ -216,8 +216,8 @@ def encrypt_credentials(credentials: dict[str, Any]) -> str: logger.error( f"[encrypt_credentials] Encryption failed | error: {e}", exc_info=True ) - raise UpstreamError( - "Failed to encrypt credentials. Retry shortly.", provider="kms" + raise HTTPException( + status_code=502, detail="Failed to encrypt credentials. Retry shortly." ) @@ -240,8 +240,8 @@ def decrypt_credentials(encrypted_credentials: str) -> dict[str, Any]: logger.error( f"[decrypt_credentials] Decryption failed | error: {e}", exc_info=True ) - raise UpstreamError( - "Failed to decrypt credentials. Retry shortly.", provider="kms" + raise HTTPException( + status_code=502, detail="Failed to decrypt credentials. Retry shortly." ) diff --git a/backend/app/crud/collection/collection.py b/backend/app/crud/collection/collection.py index a7ae2b124..6de091548 100644 --- a/backend/app/crud/collection/collection.py +++ b/backend/app/crud/collection/collection.py @@ -9,21 +9,18 @@ from sqlalchemy.exc import IntegrityError from app.models import Document, Collection, CollectionUpdate, DocumentCollection -from app.core.exceptions import ConflictError from app.core.util import now from app.crud.document_collection import DocumentCollectionCrud logger = logging.getLogger(__name__) -class CollectionNameConflictError(ConflictError): +class CollectionNameConflictError(Exception): """Raised when a collection name conflicts with an existing active collection.""" def __init__(self, name: str | None) -> None: self.name = name - super().__init__( - f"Collection '{name}' already exists. Choose a different name." - ) + super().__init__(f"Collection name '{name}' already exists") class CollectionCrud: @@ -114,7 +111,7 @@ def read_one_if_delete(self, collection_id: UUID) -> Collection: "[CollectionCrud.read_one_if_delete] Collection already deleted | " f"{{'project_id': '{self.project_id}', 'collection_id': '{collection_id}'}}" ) - raise ConflictError("Collection already deleted") + raise HTTPException(status_code=409, detail="Collection already deleted") return collection diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index 2a8402c46..ca65a8151 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -5,7 +5,6 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, and_, select -from app.core.exceptions import ConflictError, KaapiError from app.core.util import now from app.crud.model_config import validate_blob_model_or_raise from app.models import ( @@ -69,7 +68,7 @@ def create_or_raise( return config, version - except (HTTPException, KaapiError): + except HTTPException: self.session.rollback() raise @@ -79,8 +78,9 @@ def create_or_raise( f"[ConfigCrud.create] Duplicate configuration name | " f"{{'name': '{config_create.name}', 'project_id': {self.project_id}, 'error': '{str(e)}'}}", ) - raise ConflictError( - f"Config with name '{config_create.name}' already exists in this project." + raise HTTPException( + status_code=409, + detail=f"Config with name '{config_create.name}' already exists in this project.", ) except Exception as e: diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index d3a0ac62d..45dabcfdc 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -8,7 +8,6 @@ from sqlalchemy.orm import defer from sqlmodel import Session, and_, select -from app.core.exceptions import ConflictError, KaapiError from app.core.util import now from app.crud.model_config import is_reasoning_model, validate_blob_model_or_raise from app.models import ( @@ -109,7 +108,7 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: return version - except (HTTPException, KaapiError): + except HTTPException: self.session.rollback() raise @@ -119,8 +118,9 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: f"[ConfigVersionCrud.create_or_raise] Version conflict | " f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", ) - raise ConflictError( - "A version with this number already exists for this config." + raise HTTPException( + status_code=409, + detail="A version with this number already exists for this config.", ) except Exception as e: diff --git a/backend/app/crud/credentials.py b/backend/app/crud/credentials.py index 9a345d8b1..38c106601 100644 --- a/backend/app/crud/credentials.py +++ b/backend/app/crud/credentials.py @@ -7,7 +7,6 @@ from fastapi import HTTPException -from app.core.exceptions import ConflictError, KaapiError, NotFoundError from app.core.providers import validate_provider, validate_provider_credentials from app.core.security import decrypt_credentials, encrypt_credentials from app.core.util import now @@ -66,11 +65,13 @@ def set_creds_for_org( exc_info=True, ) if is_duplicate: - raise ConflictError( - f"Credentials for provider '{provider}' already exist for this organization and project combination" + raise HTTPException( + status_code=409, + detail=f"Credentials for provider '{provider}' already exist for this organization and project combination", ) - raise KaapiError( - f"Could not store credentials for provider '{provider}'. Contact Kaapi if it persists." + raise HTTPException( + status_code=500, + detail=f"Could not store credentials for provider '{provider}'. Contact Kaapi if it persists.", ) logger.info( f"[set_creds_for_org] Successfully created credentials | organization_id {organization_id}, project_id {project_id}" @@ -287,7 +288,9 @@ def remove_provider_credential( logger.warning( f"[remove_provider_credential] Credential not found | organization_id {org_id}, provider {provider}, project_id {project_id}" ) - raise NotFoundError(f"No credentials found for provider '{provider}'.") + raise HTTPException( + status_code=404, detail=f"No credentials found for provider '{provider}'." + ) session.commit() logger.info( f"[remove_provider_credential] Successfully deleted credential | provider {provider}, organization_id {org_id}, project_id {project_id}" @@ -324,7 +327,10 @@ def remove_creds_for_org(*, session: Session, org_id: int, project_id: int) -> N logger.warning( f"[remove_creds_for_org] No credentials to delete | organization_id {org_id}, project_id {project_id}" ) - raise NotFoundError("No credentials found for this organization and project.") + raise HTTPException( + status_code=404, + detail="No credentials found for this organization and project.", + ) if rows_deleted < expected_count: logger.error( diff --git a/backend/app/crud/evaluations/batch.py b/backend/app/crud/evaluations/batch.py index 3be1dc3e5..a1c659139 100644 --- a/backend/app/crud/evaluations/batch.py +++ b/backend/app/crud/evaluations/batch.py @@ -8,6 +8,7 @@ """ import logging +from fastapi import HTTPException from typing import Any from langfuse import Langfuse @@ -28,12 +29,6 @@ map_kaapi_to_google_params, map_kaapi_to_openai_params, ) -from app.core.exceptions import ( - InvalidPayloadError, - InvalidValueError, - NotFoundError, - UpstreamError, -) from app.services.llm.providers.registry import LLMProvider from app.utils import get_openai_client @@ -52,7 +47,8 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str, List of dataset items with input and expected_output Raises: - NotFoundError, UpstreamError, InvalidPayloadError + HTTPException: 404 if the dataset is missing, 502 if Langfuse errored + or was unreachable, 422 if the dataset is empty. """ try: dataset = langfuse.get_dataset(dataset_name) @@ -60,33 +56,37 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str, logger.warning( f"[fetch_dataset_items] Dataset not found | dataset={dataset_name} | {e}" ) - raise NotFoundError(f"Dataset '{dataset_name}' not found in Langfuse.") + raise HTTPException( + status_code=404, detail=f"Dataset '{dataset_name}' not found in Langfuse." + ) except LangfuseApiError as e: logger.error( f"[fetch_dataset_items] Langfuse rejected the request | " f"dataset={dataset_name}, code={getattr(e, 'status_code', 'unknown')}", exc_info=True, ) - raise UpstreamError( - f"Langfuse could not return dataset '{dataset_name}' " + raise HTTPException( + status_code=502, + detail=f"Langfuse could not return dataset '{dataset_name}' " f"(code: {getattr(e, 'status_code', 'unknown')}). Retry shortly.", - provider="langfuse", ) except Exception: logger.error( f"[fetch_dataset_items] Failed to reach Langfuse | dataset={dataset_name}", exc_info=True, ) - raise UpstreamError( - f"Could not reach Langfuse to fetch dataset '{dataset_name}'. Retry shortly.", - provider="langfuse", + raise HTTPException( + status_code=502, + detail=f"Could not reach Langfuse to fetch dataset '{dataset_name}'. Retry shortly.", ) if not dataset.items: logger.warning( f"[fetch_dataset_items] Dataset is empty | dataset={dataset_name}" ) - raise InvalidPayloadError(f"Dataset '{dataset_name}' is empty.") + raise HTTPException( + status_code=422, detail=f"Dataset '{dataset_name}' is empty." + ) items = [] for item in dataset.items: @@ -231,8 +231,9 @@ def start_evaluation_batch( dataset_items=dataset_items, openai_params=mapped_params ) if not jsonl_data: - raise InvalidPayloadError( - "Evaluation dataset did not produce any JSONL entries (missing questions?)." + raise HTTPException( + status_code=422, + detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", ) openai_client = get_openai_client( @@ -271,8 +272,9 @@ def start_evaluation_batch( dataset_items=dataset_items, google_params=mapped_params ) if not jsonl_data: - raise InvalidPayloadError( - "Evaluation dataset did not produce any JSONL entries (missing questions?)." + raise HTTPException( + status_code=422, + detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", ) gemini_client = GeminiClient.from_credentials( @@ -302,8 +304,9 @@ def start_evaluation_batch( ) else: - raise InvalidValueError( - f"Unsupported provider for evaluation batches: {provider}" + raise HTTPException( + status_code=400, + detail=f"Unsupported provider for evaluation batches: {provider}", ) eval_run.batch_job_id = batch_job.id diff --git a/backend/app/crud/onboarding.py b/backend/app/crud/onboarding.py index a324a2baa..23a48cfd7 100644 --- a/backend/app/crud/onboarding.py +++ b/backend/app/crud/onboarding.py @@ -3,7 +3,6 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session -from app.core.exceptions import ConflictError from app.core.security import encrypt_credentials, get_password_hash from app.crud import ( api_key_manager, @@ -147,9 +146,10 @@ def onboard_project( f"[onboard_project] Conflicting concurrent onboarding | " f"org={onboard_in.organization_name}, project={onboard_in.project_name} | {e}" ) - raise ConflictError( - f"Project '{onboard_in.project_name}' already exists for " - f"organization '{onboard_in.organization_name}'" + raise HTTPException( + status_code=409, + detail=f"Project '{onboard_in.project_name}' already exists for " + f"organization '{onboard_in.organization_name}'", ) cred_ids = [c.id for c in created_credentials] diff --git a/backend/app/services/evaluations/dataset.py b/backend/app/services/evaluations/dataset.py index bb9abf472..1eec2458c 100644 --- a/backend/app/services/evaluations/dataset.py +++ b/backend/app/services/evaluations/dataset.py @@ -6,7 +6,6 @@ from sqlmodel import Session from app.core.cloud import get_cloud_storage -from app.core.exceptions import UpstreamError from app.crud.evaluations import ( create_evaluation_dataset, upload_csv_to_object_store, @@ -140,9 +139,9 @@ def upload_dataset( f"[upload_dataset] Failed to upload dataset to Langfuse | dataset={dataset_name}", exc_info=True, ) - raise UpstreamError( - f"Langfuse could not accept dataset '{dataset_name}'. Retry shortly.", - provider="langfuse", + raise HTTPException( + status_code=502, + detail=f"Langfuse could not accept dataset '{dataset_name}'. Retry shortly.", ) # Step 5: Store metadata in database diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index dcb56c8f1..459da6e0a 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -21,7 +21,6 @@ from app.core.cloud import get_cloud_storage from app.core.config import settings from app.core.db import engine -from app.core.exceptions import InvalidPayloadError, KaapiError from app.crud.evaluations import ( get_dataset_by_id, resolve_evaluation_config, @@ -299,7 +298,9 @@ def validate_and_start_fast_evaluation( ) total_items = len(dataset_items) if total_items == 0: - raise InvalidPayloadError(f"Dataset '{dataset.name}' returned no items") + raise HTTPException( + status_code=422, detail=f"Dataset '{dataset.name}' returned no items" + ) n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) # total_items isn't on EvaluationRunUpdate; set it directly, then flip to @@ -322,7 +323,7 @@ def validate_and_start_fast_evaluation( f"eval_run_id={eval_run.id} | total_items={total_items} | " f"n_chunks={n_chunks}" ) - except KaapiError as exc: + except HTTPException as exc: log = logger.error if exc.status_code >= 500 else logger.warning log( f"[validate_and_start_fast_evaluation] Failed to start run | " diff --git a/backend/app/services/guardrails/jobs.py b/backend/app/services/guardrails/jobs.py index 724caff96..1ddf26c11 100644 --- a/backend/app/services/guardrails/jobs.py +++ b/backend/app/services/guardrails/jobs.py @@ -1,4 +1,5 @@ import logging +from fastapi import HTTPException from typing import Any from uuid import UUID, uuid4 @@ -8,7 +9,6 @@ from app.celery.utils import start_guardrails_job from app.core.db import engine -from app.core.exceptions import ServiceUnavailableError from app.core.telemetry import log_context from app.crud.jobs import JobCrud from app.models import Job, JobStatus, JobType, JobUpdate @@ -84,8 +84,9 @@ def start_job( job_id=job.id, job_update=JobUpdate(status=JobStatus.FAILED, error_message=str(e)), ) - raise ServiceUnavailableError( - "Could not queue the guardrails job. Retry shortly." + raise HTTPException( + status_code=503, + detail="Could not queue the guardrails job. Retry shortly.", ) logger.info( diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index 514d689d6..cface6301 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -17,7 +17,6 @@ from app.celery.utils import start_llm_chain_job, start_llm_job from app.core.db import engine -from app.core.exceptions import ServiceUnavailableError from app.core.providers import Provider from app.core.langfuse.langfuse import observe_llm_execution from app.core.telemetry import ( @@ -179,8 +178,8 @@ def start_job( ) job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) - raise ServiceUnavailableError( - "Could not queue the LLM call. Retry shortly." + raise HTTPException( + status_code=503, detail="Could not queue the LLM call. Retry shortly." ) _set_traceability_attributes(span, task_id=str(task_id)) @@ -239,8 +238,9 @@ def start_chain_job( ) job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) - raise ServiceUnavailableError( - "Could not queue the LLM chain job. Retry shortly." + raise HTTPException( + status_code=503, + detail="Could not queue the LLM chain job. Retry shortly.", ) _set_traceability_attributes(span, task_id=str(task_id)) diff --git a/backend/app/services/stt_evaluations/audio.py b/backend/app/services/stt_evaluations/audio.py index 13d9aed9f..1246a4a4c 100644 --- a/backend/app/services/stt_evaluations/audio.py +++ b/backend/app/services/stt_evaluations/audio.py @@ -8,7 +8,6 @@ from sqlmodel import Session from app.core.cloud.storage import get_cloud_storage -from app.core.exceptions import KaapiError, UpstreamError from fastapi import HTTPException from app.crud.file import create_file from app.models.file import FileType, AudioUploadResponse @@ -117,7 +116,7 @@ def upload_audio_file( content_type=content_type, ) - except (HTTPException, KaapiError): + except HTTPException: raise except Exception as e: logger.error( @@ -125,7 +124,7 @@ def upload_audio_file( f"project_id: {project_id}, error: {str(e)}", exc_info=True, ) - raise UpstreamError( - "Object storage could not accept the audio file. Retry shortly.", - provider="object-storage", + raise HTTPException( + status_code=502, + detail="Object storage could not accept the audio file. Retry shortly.", ) diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index 7d9a3cc2b..dab5cf6d6 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -7,7 +7,6 @@ from fastapi.testclient import TestClient from sqlmodel import Session, select -from app.core.exceptions import InvalidPayloadError, InvalidValueError from app.core.util import now from app.crud.evaluations.batch import ( build_google_evaluation_jsonl, @@ -1134,7 +1133,7 @@ def test_unsupported_provider_marks_failed( ) -> None: mock_fetch.return_value = _BATCH_DATASET_ITEMS - with pytest.raises(InvalidValueError, match="Unsupported provider"): + with pytest.raises(HTTPException, match="Unsupported provider"): start_evaluation_batch( langfuse=MagicMock(), session=db, @@ -1158,7 +1157,7 @@ def test_empty_jsonl_marks_failed( ] mock_map.return_value = ({"model": "gpt-4o"}, []) - with pytest.raises(InvalidPayloadError, match="did not produce any JSONL"): + with pytest.raises(HTTPException, match="did not produce any JSONL"): start_evaluation_batch( langfuse=MagicMock(), session=db, diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index 2449ba8e5..53afbef66 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,4 +1,3 @@ -import pytest from fastapi import FastAPI from fastapi.testclient import TestClient @@ -8,15 +7,6 @@ _sanitize_validation_errors, register_exception_handlers, ) -from app.core.exceptions import ( - ConflictError, - InvalidPayloadError, - InvalidValueError, - KaapiError, - NotFoundError, - ServiceUnavailableError, - UpstreamError, -) from app.tests.utils.auth import TestAuthContext @@ -166,30 +156,6 @@ def boom() -> None: return TestClient(app, raise_server_exceptions=False) -class TestKaapiErrorHandler: - """Each domain exception surfaces as its own status code.""" - - @pytest.mark.parametrize( - "exc, expected", - [ - (NotFoundError("gone"), 404), - (ConflictError("dupe"), 409), - (InvalidValueError("bad value"), 400), - (InvalidPayloadError("bad shape"), 422), - (UpstreamError("langfuse down", provider="langfuse"), 502), - (ServiceUnavailableError("broker down"), 503), - (KaapiError("our bug"), 500), - ], - ) - def test_status_code_and_body(self, exc: KaapiError, expected: int) -> None: - response = _app_raising(exc).get("/boom") - - assert response.status_code == expected - body = response.json() - assert body["success"] is False - assert body["error"] == exc.detail - - class TestGenericErrorHandler: def test_does_not_leak_exception_text(self) -> None: response = _app_raising(ValueError("secret arn:aws:kms:key/abc")).get("/boom") diff --git a/backend/app/tests/core/test_security.py b/backend/app/tests/core/test_security.py index 5d844479c..8683bb1db 100644 --- a/backend/app/tests/core/test_security.py +++ b/backend/app/tests/core/test_security.py @@ -4,12 +4,12 @@ import boto3 import jwt import pytest +from fastapi import HTTPException from moto import mock_aws from sqlmodel import Session import app.core.security as security from app.core.config import settings -from app.core.exceptions import UpstreamError from app.core.security import ( ALGORITHM, KMS_CIPHERTEXT_PREFIX, @@ -87,11 +87,10 @@ def test_kms_encrypt_failure_raises_502_without_leaking_cause( broken.encrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") monkeypatch.setattr(security, "_kms_client", broken) - with pytest.raises(UpstreamError) as exc: + with pytest.raises(HTTPException) as exc: encrypt_credentials({"api_key": "sk-1"}) assert exc.value.status_code == 502 - assert exc.value.provider == "kms" assert "arn:aws:kms" not in exc.value.detail def test_kms_decrypt_failure_raises_502(self, monkeypatch, kms_key): @@ -100,7 +99,7 @@ def test_kms_decrypt_failure_raises_502(self, monkeypatch, kms_key): broken.decrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") monkeypatch.setattr(security, "_kms_client", broken) - with pytest.raises(UpstreamError) as exc: + with pytest.raises(HTTPException) as exc: decrypt_credentials(encrypted) assert exc.value.status_code == 502 diff --git a/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py index aeb56d6dd..1b044d51b 100644 --- a/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py +++ b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py @@ -7,7 +7,6 @@ from fastapi import HTTPException -from app.core.exceptions import ConflictError from app.core.util import now from app.crud import CollectionCrud from app.models import Collection, ProviderType @@ -32,7 +31,7 @@ def test_already_deleted_raises_409(db: Session) -> None: project = get_project(db) collection = _make_collection(db, project.id, deleted=True) - with pytest.raises(ConflictError) as exc: + with pytest.raises(HTTPException) as exc: CollectionCrud(db, project.id).read_one_if_delete(collection.id) assert exc.value.status_code == 409 diff --git a/backend/app/tests/crud/evaluations/test_batch_errors.py b/backend/app/tests/crud/evaluations/test_batch_errors.py index b91973269..262ed9f93 100644 --- a/backend/app/tests/crud/evaluations/test_batch_errors.py +++ b/backend/app/tests/crud/evaluations/test_batch_errors.py @@ -3,10 +3,10 @@ from unittest.mock import MagicMock import pytest +from fastapi import HTTPException from langfuse.api import NotFoundError as LangfuseNotFoundError from langfuse.api.core import ApiError as LangfuseApiError -from app.core.exceptions import InvalidPayloadError, NotFoundError, UpstreamError from app.crud.evaluations.batch import fetch_dataset_items @@ -16,7 +16,7 @@ def test_dataset_missing_raises_404() -> None: body={"message": "Dataset not found"} ) - with pytest.raises(NotFoundError) as exc: + with pytest.raises(HTTPException) as exc: fetch_dataset_items(langfuse=langfuse, dataset_name="missing") assert exc.value.status_code == 404 @@ -27,18 +27,17 @@ def test_langfuse_api_error_raises_502() -> None: langfuse = MagicMock() langfuse.get_dataset.side_effect = LangfuseApiError(status_code=500, body="boom") - with pytest.raises(UpstreamError) as exc: + with pytest.raises(HTTPException) as exc: fetch_dataset_items(langfuse=langfuse, dataset_name="ds") assert exc.value.status_code == 502 - assert exc.value.provider == "langfuse" def test_langfuse_unreachable_raises_502() -> None: langfuse = MagicMock() langfuse.get_dataset.side_effect = ConnectionError("dns failure") - with pytest.raises(UpstreamError) as exc: + with pytest.raises(HTTPException) as exc: fetch_dataset_items(langfuse=langfuse, dataset_name="ds") assert exc.value.status_code == 502 @@ -48,7 +47,7 @@ def test_empty_dataset_raises_422() -> None: langfuse = MagicMock() langfuse.get_dataset.return_value = MagicMock(items=[]) - with pytest.raises(InvalidPayloadError) as exc: + with pytest.raises(HTTPException) as exc: fetch_dataset_items(langfuse=langfuse, dataset_name="ds") assert exc.value.status_code == 422 diff --git a/backend/app/tests/crud/test_credentials.py b/backend/app/tests/crud/test_credentials.py index 79d9df6fa..90a7770b5 100644 --- a/backend/app/tests/crud/test_credentials.py +++ b/backend/app/tests/crud/test_credentials.py @@ -13,7 +13,6 @@ remove_creds_for_org, ) from app.models import CredsCreate, CredsUpdate -from app.core.exceptions import ConflictError, NotFoundError from app.core.providers import Provider from app.tests.utils.test_data import ( create_test_project, @@ -384,7 +383,7 @@ def test_remove_provider_credential_race_deletes_nothing_raises_404( ) monkeypatch.setattr(db, "exec", lambda *a, **k: MagicMock(rowcount=0)) - with pytest.raises(NotFoundError) as exc: + with pytest.raises(HTTPException) as exc: remove_provider_credential( session=db, org_id=project.organization_id, @@ -417,7 +416,7 @@ def test_duplicate_provider_raises_409(db: Session) -> None: credential={"openai": {"api_key": "another-key"}}, ) - with pytest.raises(ConflictError) as exc: + with pytest.raises(HTTPException) as exc: set_creds_for_org( session=db, creds_add=creds_create, diff --git a/backend/app/tests/crud/test_onboarding.py b/backend/app/tests/crud/test_onboarding.py index 6f9ffd835..f46ec8eae 100644 --- a/backend/app/tests/crud/test_onboarding.py +++ b/backend/app/tests/crud/test_onboarding.py @@ -3,7 +3,6 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from app.core.exceptions import ConflictError, UpstreamError from app.crud.onboarding import onboard_project from app.crud import ( @@ -298,10 +297,12 @@ def test_onboard_kms_failure_raises_502(db: Session, monkeypatch) -> None: """A KMS outage while encrypting credentials is upstream, not our bug.""" monkeypatch.setattr( "app.crud.onboarding.encrypt_credentials", - lambda _: (_ for _ in ()).throw(UpstreamError("kms down", provider="kms")), + lambda _: (_ for _ in ()).throw( + HTTPException(status_code=502, detail="kms down") + ), ) - with pytest.raises(UpstreamError) as exc: + with pytest.raises(HTTPException) as exc: onboard_project( session=db, onboard_in=OnboardingRequest( @@ -327,7 +328,7 @@ def _raise_integrity_error() -> None: monkeypatch.setattr(db, "commit", _raise_integrity_error) - with pytest.raises(ConflictError) as exc: + with pytest.raises(HTTPException) as exc: onboard_project( session=db, onboard_in=OnboardingRequest( diff --git a/backend/app/tests/services/guardrails/test_jobs.py b/backend/app/tests/services/guardrails/test_jobs.py index 648e171e9..26fc8fd75 100644 --- a/backend/app/tests/services/guardrails/test_jobs.py +++ b/backend/app/tests/services/guardrails/test_jobs.py @@ -4,7 +4,6 @@ import pytest -from app.core.exceptions import ServiceUnavailableError from sqlmodel import Session, select from app.models import Job, JobStatus, JobType @@ -149,7 +148,7 @@ def test_start_job_celery_failure_marks_failed_and_raises_503( ) -> None: with patch("app.services.guardrails.jobs.start_guardrails_job") as mock_enqueue: mock_enqueue.side_effect = RuntimeError("broker down") - with pytest.raises(ServiceUnavailableError) as exc: + with pytest.raises(HTTPException) as exc: start_job( db=db, request=_make_request(), diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index fffb40bc4..a706afd0f 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -6,7 +6,6 @@ from gevent import Timeout -from app.core.exceptions import ServiceUnavailableError from sqlmodel import Session, select from app.crud import JobCrud @@ -96,7 +95,7 @@ def test_start_job_celery_scheduling_fails( with patch("app.services.llm.jobs.start_llm_job") as mock_schedule: mock_schedule.side_effect = Exception("Celery connection failed") - with pytest.raises(ServiceUnavailableError) as exc_info: + with pytest.raises(HTTPException) as exc_info: start_job(db, llm_call_request, project.id, project.organization_id) assert exc_info.value.status_code == 503 @@ -2005,7 +2004,7 @@ def test_start_chain_job_celery_failure(self, db: Session, chain_request): mock_job.id = uuid4() mock_job_crud_class.return_value.create.return_value = mock_job - with pytest.raises(ServiceUnavailableError) as exc_info: + with pytest.raises(HTTPException) as exc_info: start_chain_job(db, chain_request, project.id, project.organization_id) assert exc_info.value.status_code == 503 diff --git a/backend/app/tests/services/stt_evaluations/test_audio.py b/backend/app/tests/services/stt_evaluations/test_audio.py index 3c84b717f..48a6ab6b7 100644 --- a/backend/app/tests/services/stt_evaluations/test_audio.py +++ b/backend/app/tests/services/stt_evaluations/test_audio.py @@ -7,7 +7,6 @@ from fastapi import HTTPException -from app.core.exceptions import UpstreamError from app.services.stt_evaluations.audio import ( _resolve_extension, _validate_audio_file, @@ -180,7 +179,7 @@ def test_upload_storage_error(self, mock_get_storage) -> None: mock_storage.put.side_effect = Exception("S3 connection failed") mock_get_storage.return_value = mock_storage - with pytest.raises(UpstreamError) as exc_info: + with pytest.raises(HTTPException) as exc_info: upload_audio_file( session=MagicMock(), file=_make_upload_file(), diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index aebc88ce8..0b4368980 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -6,11 +6,8 @@ All paths relative to `backend/app/`. - `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. - `core/middleware.py` — request-level middleware. -## Domain exceptions -- `core/exceptions.py` — `KaapiError` base plus `NotFoundError` (404), `ConflictError` (409), `InvalidValueError` (400), `InvalidPayloadError` (422), `UpstreamError` (502, takes `provider=`), `ServiceUnavailableError` (503). Handled by `kaapi_error_handler`, which reads `status_code` off the class. -- Never raise a bare `ValueError` from crud/service for a caller-facing failure: nothing catches it, so it becomes a 500 with a generic body. Use one of the above. -- `KaapiError` is the preferred raise in crud/service (it keeps the layer transport-agnostic), but the migration is partial: ~78 `raise HTTPException` sites remain in `crud/` and are still correct as long as the code is right. Prefer `KaapiError` in new code; convert neighbours opportunistically rather than mixing both idioms inside one function. -- Code selection, applied across evaluations, knowledge-base, llm-call, tenancy, and platform: +## Status-code selection +Raise `HTTPException` directly with the right status code from routes, crud, and service. Never raise a bare `ValueError` for a caller-facing failure: nothing catches it, so it becomes a 500 with a generic body. | Condition | Code | |---|---| @@ -22,7 +19,7 @@ All paths relative to `backend/app/`. | Our async infra could not accept the work (Celery broker) | 503 | | Our bug (DB write failed, unexpected exception) | 500 | -- A broad `except Exception` around a DB write must re-raise `HTTPException`/`KaapiError` first, otherwise typed errors get flattened to 500. +- A broad `except Exception` around a DB write must re-raise `HTTPException` first, otherwise a typed 4xx/5xx gets flattened to 500. ## Provider/SDK error convention - `.claude/conventions/error-handling.md` — the standardized pattern for wrapping provider/SDK exceptions at service/crud call sites. Follow it for any new OpenAI/Gemini/Anthropic call site. diff --git a/features/status-code-consistency/PLAN.md b/features/status-code-consistency/PLAN.md new file mode 100644 index 000000000..8b7674f5a --- /dev/null +++ b/features/status-code-consistency/PLAN.md @@ -0,0 +1,195 @@ +# Status Code and Error Message Consistency (issue #834): Implementation Plan + +Source spec: [GitHub issue #834](https://github.com/ProjectTech4DevAI/kaapi-backend/issues/834), "Observability: Improve status codes consistency" + +## Summary + +Kaapi returns HTTP 500 for conditions that are not server bugs: a missing Langfuse dataset, a provider or storage failure, a KMS encrypt that failed, a broker that could not accept a job. Callers cannot tell "you asked for something that does not exist" from "we broke", and ops alerting on 5xx rate is polluted by caller errors. This change introduces a central domain exception taxonomy in `app/core/exceptions.py` with registered handlers, then converts the mis-coded raise sites across eight domains, and completes the log-level and log-prefix audit on the same lines. No schema change, so no Alembic migration. + +Domains in scope: the five the issue names (LLM Call, LLM Chain, Collections, Documents, Evaluations including STT/TTS), plus Onboarding, Config Management, and Credentials added by later scope decision. + +### Note on the issue's two cited line numbers + +Both citations are stale against `main`; the paths also moved under `backend/`. + +- `app/crud/assistants.py:68` is already correct. [backend/app/crud/assistants.py:64-73](backend/app/crud/assistants.py#L64-L73) raises 404 on `openai.NotFoundError` and 502 on `openai.OpenAIError`. No change needed, and `assistants` belongs to the Responses module, which is not among the in-scope domains. +- `app/crud/evaluations/batch.py:47` is now a docstring line, and no `success` boolean exists in that file. The underlying defect survives in a different shape: [backend/app/crud/evaluations/batch.py:49-58](backend/app/crud/evaluations/batch.py#L49-L58) raises a bare `ValueError` when Langfuse cannot return the dataset, which nothing catches on the fast-eval path, so the generic handler turns it into a 500. Production evidence: `app/logs/app.log.3:9975` logs `status_code: 404, body: {'message': 'Dataset not found', 'error': 'LangfuseNotFoundError'}` on a request that returned 500. Step 3 fixes it. + +### The taxonomy this plan applies + +Every code change below is an application of this table. It is derived from `.claude/conventions/route.md` ("Status codes (the ones to get right)") and `.claude/conventions/error-handling.md`. + +| Condition | Code | Log level | +|---|---|---| +| Resource absent, ours or an upstream-held resource we reference | 404 | `warning` | +| State conflict (already exists, already deleted) | 409 | `warning` | +| Payload unparseable or wrong shape (bad CSV, unreadable file) | 422 | `warning` | +| Valid shape, unacceptable value (unsupported transform pair) | 400 | `warning` | +| Upstream returned an error or was unreachable (Langfuse, OpenAI, Gemini, object storage) | 502 | `error` | +| Our async infrastructure could not accept the work (Celery broker, RabbitMQ) | 503 | `error` | +| Genuinely our bug (DB write failed, unexpected exception) | 500 | `error` | + +## Blast Radius + +Primary entities: this is a cross-cutting change to the HTTP error contract, not to any table. Entities whose read/write paths change their failure responses: `EvaluationDataset`, `EvaluationRun`, `STTSample`/`STTResult`, `TTSResult`, `Collection`, `CollectionJob`, `Document`, `DocTransformationJob`, `Job`, `LlmCall`, `LlmChain`, `Config`, `ConfigVersion`, `Credential`, `Organization`, `Project`, `User`, `APIKey`. + +| Surface | Hop | Impact | Decision | +|---|---|---|---| +| `app/core/exception_handlers.py` | 1 | New typed handler for `KaapiError`; `generic_error_handler` stops echoing `str(exc)` | in scope | +| Evaluations routes/services/crud (text, STT, TTS) | 1 | 500 becomes 404 / 422 / 502 / 503 at named sites | in scope | +| Collections crud/routes | 1 | "already deleted" 400 becomes 409; `CollectionNameConflictError` re-based on `ConflictError` | in scope | +| Documents services/crud | 1 | Unsupported file format 400 becomes 422; wrong-module `HTTPException` import corrected | in scope | +| LLM Call, LLM Chain, Guardrails job start | 1 | Celery enqueue failure 500 becomes 503 | in scope | +| Config Management (`crud/config/`) | 1 | Catch-alls swallow `IntegrityError` into 500; `ConfigBlob` validation returns 400 where the request boundary returns 422 | in scope | +| Credentials (`crud/credentials.py`, `routes/credentials.py`) | 1 | Unique-constraint 400 becomes 409; delete-miss 500 becomes 404; bare `ValueError` sites typed | in scope | +| `app/core/security.py` KMS encrypt/decrypt | 1 | `ValueError` on AWS KMS failure surfaces as 500; becomes 502 | in scope | +| Onboarding (`crud/onboarding.py`) | 1 | Unguarded `encrypt_credentials` and unguarded final `session.commit()` both surface as 500 | in scope | +| `app/tests/` status-code assertions | 1 | 8 assertions encode the current wrong codes and must flip | in scope | +| `docs/wiki/cross-cutting/exceptions.md` | 1 | Must document the new taxonomy module (wiki maintenance rule) | in scope | +| `app/api/docs/**/*.md` swagger copy | 2 | 3 files in the touched domains mention status codes and may go stale | in scope | +| Sentry (`app/core/sentry_filters.py`) | 2 | Reclassifying caller errors off 5xx reduces event volume; no filter code change needed | in scope, verify only | +| kaapi-frontend console | 2 | Breaking if it branches on status code; `APIResponse` body shape is unchanged | **out of scope** (user decision) | +| Langfuse | 2 | Trace and score shape unchanged | unaffected | +| Provider Batch APIs (OpenAI, Gemini, Anthropic) | 2 | Batch payload shape unchanged | unaffected | +| Object storage | 2 | Upload failures resurface as 502 instead of 500; no storage code change | in scope, read path only | +| auth, fine_tuning, assessment, response, api_key | 2 | Same defect class present but outside the eight in-scope domains | **out of scope**, follow-up issue (step 13) | + +## Steps + +### 1. Core: add the domain exception taxonomy +- Files: `backend/app/core/exceptions.py` (new) +- Define `KaapiError(Exception)` carrying `status_code: int` and `detail: str`, plus subclasses `NotFoundError` (404), `ConflictError` (409), `InvalidPayloadError` (422), `InvalidValueError` (400), `UpstreamError` (502), `ServiceUnavailableError` (503). +- `UpstreamError` takes a `provider: str` so the message can name who failed. +- Precedent to follow, not duplicate: the four existing ad-hoc exceptions (`CloudStorageError` in `core/cloud/storage.py:34`, `GeminiClientError` in `core/batch/client.py:15`, `CollectionNameConflictError` in `crud/collection/collection.py:18`, `TransformationError` in `services/doctransform/registry.py:9`) stay where they are; only `CollectionNameConflictError` is re-based in step 6. +- Depends on: nothing + +### 2. Core: register the handler and stop the 500 body leak +- Files: `backend/app/core/exception_handlers.py` (change) +- Add `@app.exception_handler(KaapiError)` inside `register_exception_handlers`, returning `JSONResponse(status_code=exc.status_code, content=APIResponse.failure_response(exc.detail).model_dump())`, matching the existing `http_exception_handler` shape at lines 98-108. +- Change `generic_error_handler` (lines 110-117) to log the exception with `exc_info=True` and the correlation id, and return a fixed message instead of `str(exc)`. This is what currently exposes raw `ValueError` text (including Langfuse response bodies) to API clients. +- Handler registration order matters: register the `KaapiError` handler before the bare `Exception` handler. +- Depends on: step 1 + +### 3. Evaluations crud: dataset fetch raises typed errors +- Files: `backend/app/crud/evaluations/batch.py` (change) +- `fetch_dataset_items` (lines 35-70): in the `except Exception` at line 51, branch on the Langfuse error. Not-found raises `NotFoundError(f"Dataset '{dataset_name}' not found")`; any other Langfuse failure raises `UpstreamError(provider="langfuse", ...)`. Log level follows the taxonomy table (`warning` for not-found, `error` with `exc_info=True` for upstream). +- Line 58 empty-dataset `ValueError` becomes `InvalidPayloadError`. +- The three other `raise ValueError` sites in this file get the same treatment; the file has 5 total. +- Depends on: step 1 + +### 4. Evaluations services: upstream and broker failures +- Files: `backend/app/services/evaluations/dataset.py`, `backend/app/services/evaluations/fast.py`, `backend/app/services/evaluations/prompt_improvement.py` (change) +- `dataset.py:131-138`: Langfuse upload failure becomes `UpstreamError` (502) instead of `HTTPException(500)`. +- `fast.py:211-215`: the broad `except` currently swallows step 3's typed errors into a flat 500. Re-raise `KaapiError` untouched before the catch-all, so a missing dataset reaches the caller as 404. The run is still marked `failed` in both branches. +- `prompt_improvement.py:207-210`: Celery enqueue failure becomes `ServiceUnavailableError` (503). +- Leave `crud/evaluations/dataset.py:102-105` at 500. A failed DB write is genuinely our fault and already logs at `error` with `exc_info=True`. +- Depends on: steps 1, 3 + +### 5. STT/TTS evaluations: queue, storage, and empty-dataset codes +- Files: `backend/app/api/routes/stt_evaluations/evaluation.py`, `backend/app/api/routes/tts_evaluations/evaluation.py`, `backend/app/services/stt_evaluations/audio.py` (change) +- `stt_evaluations/evaluation.py:108-111` and `tts_evaluations/evaluation.py:112-115`: "Failed to queue batch submission" becomes 503. +- `stt_evaluations/evaluation.py:65` and `tts_evaluations/evaluation.py:69`: "Dataset has no samples" moves 400 to 422, matching the empty-dataset code chosen in step 3. +- `audio.py:126-129`: object storage upload failure becomes `UpstreamError` (502). The `except HTTPException: raise` guard at line 119 becomes `except (HTTPException, KaapiError): raise`. +- Depends on: steps 1, 3 + +### 6. Collections: conflict semantics +- Files: `backend/app/crud/collection/collection.py`, `backend/app/api/routes/collections.py` (change) +- `collection.py:114`: "Collection already deleted" moves 400 to 409. It is a state conflict, not bad input. +- `collection.py:18`: `CollectionNameConflictError` subclasses `ConflictError` and carries its own detail, so the manual translation at `collections.py:220-224` can be deleted and the handler from step 2 covers it. Keep the `name` attribute; it is read to build the message. +- Depends on: steps 1, 2 + +### 7. Documents: format errors and the wrong-module import +- Files: `backend/app/services/documents/helpers.py`, `backend/app/crud/document/document.py`, `backend/app/crud/document/doc_transformation_job.py` (change) +- `helpers.py:62-64`: unsupported or unreadable source file format moves 400 to 422, matching `services/evaluations/validators.py:84-97`, which already returns 422 for the same class of problem. See Open Questions. +- `helpers.py:67-72` and `helpers.py:76-83` stay at 400. An unsupported transform pair and an unknown transformer are valid-shape, unacceptable-value cases. +- `helpers.py:59`: the docstring says `Raises: HTTPException(400)` and goes stale; update it. +- `document.py:8` and `doc_transformation_job.py:16` import `HTTPException` from `app.core.exception_handlers` rather than `fastapi`. Correct both imports. Behaviour is identical today, but it couples crud to the handler module. +- Depends on: step 1 + +### 8. LLM Call, LLM Chain, Guardrails: broker failures +- Files: `backend/app/services/llm/jobs.py`, `backend/app/services/guardrails/jobs.py` (change) +- `llm/jobs.py:181-183` (`start_job`), `llm/jobs.py:241-244` (`start_chain_job`), `guardrails/jobs.py:87-90` (`start_job`): all three catch a failed Celery enqueue and return 500. All three become `ServiceUnavailableError` (503). The job is still marked `FAILED` first, unchanged. +- Guardrails is included because `docs/wiki/modules/llm-call.md` places `api/routes/guardrails.py` in the LLM Call module. +- Depends on: step 1 + +### 9. Config Management: catch-all scope and validation code +- Files: `backend/app/crud/config/config.py`, `backend/app/crud/config/version.py` (change) +- `config.py:70-80` (`ConfigCrud.create`) and `version.py:110-120` (`ConfigVersionCrud.create_from_partial`) both wrap their DB writes in a bare `except Exception` that returns 500. A duplicate config name raises `IntegrityError` inside that block and is flattened to 500, even though `config.py:191` already returns 409 for the same conflict on another path. Add an `except IntegrityError` branch returning `ConflictError` ahead of the catch-all, and a `except (HTTPException, KaapiError): raise` guard so typed errors raised inside are not swallowed. +- `version.py:78-85`: a `ConfigBlob` Pydantic `ValidationError` returns `HTTPException(400, detail=validation_errors)`. The same `ConfigBlob` failing at the request boundary returns 422 through `validation_error_handler`. Move this to 422 so one config blob has one rejection code regardless of entry path. The list `detail` still routes through `_sanitize_validation_errors` at `exception_handlers.py:103-104`, so the body shape is unchanged. +- Leave the immutability checks at `version.py:201`, `version.py:330`, and `version.py:340` at 400 for now. See Open Questions. +- Depends on: steps 1, 2 + +### 10. Credentials: conflict, not-found, and KMS failures +- Files: `backend/app/crud/credentials.py`, `backend/app/api/routes/credentials.py`, `backend/app/core/security.py` (change) +- `credentials.py:66-69`: the unique-constraint branch (`uq_credential_org_project_provider`) returns 400. `.claude/conventions/route.md` names 409 for exactly this case. Move to `ConflictError`. +- `credentials.py:70-72`: the non-duplicate `IntegrityError` path raises a bare `ValueError` carrying the raw DB error, which reaches the client as a 500 body. Raise a typed error with a sanitized message; the DB text stays in the log only. +- `credentials.py:190`: `raise ValueError("Provider and credential must be provided")` surfaces as 500, while `routes/credentials.py:122` and `routes/credentials.py:263` raise 400 for the identical condition. Make the crud site 400 to match. +- `credentials.py:285-289` (`remove_provider_credential`): `rowcount == 0` means nothing matched, so it is a 404, not the current 500. Change the code and drop the log from `error` to `warning`, since a caller deleting a non-existent credential is not an outage. +- `credentials.py:322-329` (`remove_creds_for_org`): split the branch. Zero rows deleted returns 404; a genuine partial delete (`0 < rows_deleted < expected_count`) keeps 500 and stays at `logger.error`, because it is a real anomaly. +- `security.py:213-216` and `security.py:233-236`: `encrypt_credentials` and `decrypt_credentials` raise `ValueError` when AWS KMS fails. KMS is upstream, so raise `UpstreamError(provider="kms")` (502). Keep the existing behaviour of never surfacing the underlying message, which may carry AWS ARNs, and add the missing `exc_info=True` to both `logger.error` calls. +- Leave `routes/credentials.py:49` at 500. An empty result from `set_creds_for_org` after no exception is a genuine internal anomaly. +- Leave the `validate_provider_credentials` 400s at `credentials.py:36`, `credentials.py:160`, and `credentials.py:208` unchanged. See Open Questions. +- Depends on: steps 1, 2 + +### 11. Onboarding: KMS and the unguarded commit +- Files: `backend/app/crud/onboarding.py` (change) +- `onboarding.py:126`: `encrypt_credentials(values)` is called with no guard, so a KMS failure becomes a 500. Step 10 makes it raise `UpstreamError`, so this site only needs to stop being the reason a 500 escapes; verify no local `except` re-flattens it. +- `onboarding.py:139` (`session.commit()`): the whole multi-step onboarding has no `try`. The pre-checks at `onboarding.py:67-71` and `onboarding.py:100-105` are read-then-write races, so two concurrent onboards of the same org and project name hit an `IntegrityError` at commit and return 500 with raw DB text. Wrap the commit in `except IntegrityError` returning `ConflictError` with the same wording the pre-check uses, and roll back. +- The two existing 409s (`onboarding.py:68`, `onboarding.py:102`) are already correct and stay as they are. +- Depends on: steps 1, 2, 10 + +### 12. Observability: log-level and prefix audit in the in-scope domains +- Files (change): `backend/app/api/routes/stt_evaluations/evaluation.py:98`, `backend/app/api/routes/tts_evaluations/evaluation.py:102`, `backend/app/crud/evaluations/cron_utils.py:325`, `backend/app/crud/evaluations/embeddings.py:325`, `backend/app/crud/evaluations/langfuse.py:197,599`, `backend/app/crud/evaluations/processing.py:328,1027,1078,1201`, `backend/app/crud/stt_evaluations/batch.py:228`, `backend/app/crud/tts_evaluations/batch.py:137`, `backend/app/services/collections/providers/gemini.py:148,173` +- Add `exc_info=True` to these 14 `logger.error` calls that caught a real exception without it, per `.claude/conventions/error-handling.md` ("`exc_info=True` on any path that caught a real exception"). +- Files (change): `backend/app/crud/evaluations/core.py:106,239` and `backend/app/crud/evaluations/embeddings.py:105,111,120,163,174,232,296,331,387` +- Add the missing `[function_name]` bracket prefix required by CLAUDE.md. 12 lines. +- Log levels already match the fault table at the status-code sites this plan touches; verify rather than assume when editing each one. The two credentials delete sites are the exception and are handled in step 10. +- Depends on: steps 3 through 11 (same lines, edit once) + +### 13. Docs: wiki, swagger, and the follow-up issue +- Files: `docs/wiki/cross-cutting/exceptions.md` (change), `backend/app/api/docs/evaluation/improve_prompt.md`, `backend/app/api/docs/evaluation/create_evaluation.md`, `backend/app/api/docs/collections/update.md`, `backend/app/api/docs/onboarding/onboarding.md` (change if their stated codes went stale) +- Add an "Domain exceptions" section to `exceptions.md` pointing at `core/exceptions.py`, listing the taxonomy table above as the source of truth, and noting that the `KaapiError` handler is registered ahead of the generic one. +- The touched module pages (`llm-call.md`, `evaluations.md`, `knowledge-base.md`, `tenancy.md`, `platform.md`) hold no status codes, so they need no edit. +- Open a follow-up issue listing the out-of-scope sites found during the survey: `app/api/routes/auth.py:65,306,357`, `app/api/routes/fine_tuning.py:269`, `app/crud/api_key.py:97`, `app/crud/assessment/dataset.py:72`, `app/services/assessment/dataset.py:293`, `app/services/assessment/utils/export.py:329`, `app/services/response/jobs.py:42`. +- Depends on: steps 1 through 12 + +## Tests + +Existing assertions that encode the current wrong codes and must flip: + +| Test | Now | After | +|---|---|---| +| `app/tests/api/routes/test_evaluation.py:477` | 500 | per step 3/4 | +| `app/tests/api/routes/test_evaluation_fast.py:1271` | 500 | 404 (missing dataset) | +| `app/tests/api/routes/test_tts_evaluation.py:629` | 500 | 503 | +| `app/tests/api/routes/test_improve_prompt.py:517` | 500 | 503 | +| `app/tests/services/llm/test_jobs.py:101,2012` | 500 | 503 | +| `app/tests/services/stt_evaluations/test_audio.py:188` | 500 | 502 | +| `app/tests/services/guardrails/test_jobs.py:159` | 500 | 503 | +| `app/tests/api/routes/test_creds.py:260` (plus its docstring at line 240, which says "fails with 400") | 400 | 409 | + +Out of scope and untouched: `app/tests/api/test_auth.py:49,313,412` and `app/tests/assessment/test_dataset.py:304`. + +Assertions deliberately left alone: `app/tests/crud/test_credentials.py:273` ("Unsupported provider") and `:332` ("Missing required fields for langfuse") stay at 400, matching the Open Questions decision below. + +New coverage: +- `app/tests/core/test_exceptions.py` (new): each `KaapiError` subclass maps to its status code through the registered handler, and `generic_error_handler` no longer returns `str(exc)` in the body. +- Evaluations: Langfuse dataset-not-found returns 404 and Langfuse unreachable returns 502, on both the batch path and the fast path. Mock the Langfuse client boundary (existing patch target `app.services.evaluations.fast.fetch_dataset_items` and `app.crud.evaluations.batch.fetch_dataset_items`, already used at `test_evaluation_fast.py:322` and `test_evaluation.py:1036`). +- Collections: deleting an already-deleted collection returns 409; duplicate name still returns 409 after the handler takes over from the manual translation. +- Documents: unsupported source format returns 422; unsupported transform pair still returns 400. +- Config: creating a config whose name already exists returns 409 rather than 500; a `ConfigBlob` that fails validation during partial-version merge returns 422 with the same body shape the request boundary produces. +- Credentials: duplicate provider credential returns 409; deleting a credential that does not exist returns 404; a KMS failure during create returns 502 and the response body does not contain the AWS error text. +- Onboarding: a KMS failure during onboarding returns 502; a concurrent onboard hitting `IntegrityError` at commit returns 409, and no partial org/project/user rows survive. + +HTTP boundaries mocked: Langfuse client, OpenAI and Gemini SDK clients, object storage (`core/cloud/storage.py`), the AWS KMS client (`get_kms_client` in `core/security.py`), and the Celery `.delay`/`apply_async` call for the 503 paths. + +`app/tests/services/credentials/` already exists and is the home for the KMS-failure cases. + +## Open Questions + +- **Documents 422 versus 400 for an unsupported file format (step 7).** Two live conventions disagree: `services/documents/helpers.py:64` returns 400, `services/evaluations/validators.py:84-97` returns 422 for the same class of problem. This plan picks 422 on the reading that the uploaded file is the unparseable payload, which also makes evaluations the larger surface that does not have to change. Reverse it if the reviewer reads an unsupported extension as a value problem rather than a shape problem, in which case `validators.py` moves to 400 instead and evaluations tests change. +- **503 versus 500 for a failed Celery enqueue (steps 4, 5, 8).** RabbitMQ being unable to accept a job is an availability problem the caller can retry, which 503 signals and 500 does not. Assumed, not stated in the issue. +- **Config immutability violations: 400 or 409 (step 9).** `version.py:201`, `version.py:330`, and `version.py:340` reject a change to an immutable field (`type`, `provider`) with 400. `.claude/conventions/error-handling.md` describes 409 as "request conflicts with current resource state", which is literally what this is. This plan leaves them at 400 to keep the config diff to the two defects that have no defence. Flip all three to 409 if the reviewer wants the taxonomy applied strictly. +- **Credentials validation: 400 or 422 (step 10).** `validate_provider_credentials` rejects both an unknown provider ("Unsupported provider", a value problem, 400) and a well-formed request missing provider-specific keys ("Missing required fields for langfuse", arguably a shape problem, 422) with the same 400. This plan keeps both at 400 rather than splitting one helper's output across two codes. Splitting them is defensible and would change `app/tests/crud/test_credentials.py:332`. +- **Onboarding bypasses credential validation.** `crud/onboarding.py:118-133` builds `Credential` rows inline instead of calling `set_creds_for_org`, so it never runs `validate_provider_credentials`. The same bad payload gets 400 through `POST /credentials` and is silently stored through `POST /onboard`. This is an error-message consistency defect, but fixing it changes onboarding behaviour rather than its status codes, so it is called out here rather than planned. Confirm whether it belongs in this PR or its own. +- **Frontend coordination is out of scope by decision.** Eleven status codes now change on endpoints the console calls, including `POST /credentials` and `POST /onboard`. The `APIResponse` body shape is untouched, so only code-branching logic can break. Worth a line in the PR description even though no investigation is planned. From 73576c23e0633ea164cd156c3e4b46e1d0767de6 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:15:32 +0530 Subject: [PATCH 03/12] chore: add HTTPException import to test files and enhance exception handling in the core --- backend/app/core/exception_handlers.py | 15 +++++++++++++++ backend/app/tests/api/routes/test_evaluation.py | 1 + .../app/tests/services/guardrails/test_jobs.py | 1 + backend/app/tests/services/llm/test_jobs.py | 1 + 4 files changed, 18 insertions(+) diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index ed8665a48..c114e6803 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -2,6 +2,7 @@ import re from collections import defaultdict +import sentry_sdk from asgi_correlation_id import correlation_id from fastapi import FastAPI, Request, HTTPException from fastapi.exceptions import RequestValidationError @@ -108,6 +109,20 @@ async def http_exception_handler( detail = exc.detail if isinstance(detail, list): detail = _sanitize_validation_errors(detail) + + # Tag at the boundary: the status is only known here, and call-site logs + # fire before it. 5xx is a server fault (error event); 4xx is the caller's. + if sentry_sdk.get_client().is_active(): + sentry_sdk.set_tag("http.status_code", str(exc.status_code)) + sentry_sdk.set_tag( + "error.class", "server" if exc.status_code >= 500 else "client" + ) + if exc.status_code >= 500: + logger.error( + f"[http_exception] {request.method} {request.url.path} " + f"-> {exc.status_code}", + exc_info=exc, + ) return JSONResponse( status_code=exc.status_code, content=APIResponse.failure_response(detail).model_dump(), diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index dab5cf6d6..629021836 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -4,6 +4,7 @@ from uuid import uuid4 import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session, select diff --git a/backend/app/tests/services/guardrails/test_jobs.py b/backend/app/tests/services/guardrails/test_jobs.py index 26fc8fd75..a09da6c2b 100644 --- a/backend/app/tests/services/guardrails/test_jobs.py +++ b/backend/app/tests/services/guardrails/test_jobs.py @@ -3,6 +3,7 @@ from uuid import uuid4 import pytest +from fastapi import HTTPException from sqlmodel import Session, select diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index a706afd0f..40007e140 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -1,4 +1,5 @@ import pytest +from fastapi import HTTPException from unittest.mock import patch, MagicMock from uuid import UUID, uuid4 From 486c909e83459621ddf588404b08fb0897cb058e Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:55:31 +0530 Subject: [PATCH 04/12] fix(tests): update exception handling in decrypt_credentials tests to use HTTPException with status code 502 --- backend/app/tests/core/test_security.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/app/tests/core/test_security.py b/backend/app/tests/core/test_security.py index b31d078be..f45f2a6ae 100644 --- a/backend/app/tests/core/test_security.py +++ b/backend/app/tests/core/test_security.py @@ -162,12 +162,14 @@ def test_v2_tampered_ciphertext_raises(self, kms_key): f"{base64.b64encode(bytes(ct)).decode()}" ) - with pytest.raises(ValueError, match="Failed to decrypt credentials"): + with pytest.raises(HTTPException) as exc: decrypt_credentials(tampered) + assert exc.value.status_code == 502 def test_v2_wrong_segment_count_raises(self, kms_key): - with pytest.raises(ValueError, match="Failed to decrypt credentials"): + with pytest.raises(HTTPException) as exc: decrypt_credentials(f"{KMS_ENVELOPE_PREFIX}onlyoneseg") + assert exc.value.status_code == 502 def test_encrypt_fernet_forces_fernet_even_with_kms_active(self, kms_key): creds = {"api_key": "sk-force-fernet"} From f055e4cfbf5b52e00391048d641867f391623f5d Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:08:46 +0530 Subject: [PATCH 05/12] feat(exception-handling): Enhance DB error handling with specific status codes and hints --- backend/app/core/exception_handlers.py | 46 +++++++++++++++++++++++++- docs/wiki/cross-cutting/exceptions.md | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 2243b989d..ed6afb792 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -4,12 +4,15 @@ import sentry_sdk from asgi_correlation_id import correlation_id -from fastapi import FastAPI, Request, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from sqlalchemy.exc import DataError, IntegrityError, OperationalError, SQLAlchemyError from starlette.status import ( + HTTP_409_CONFLICT, HTTP_422_UNPROCESSABLE_CONTENT, HTTP_500_INTERNAL_SERVER_ERROR, + HTTP_503_SERVICE_UNAVAILABLE, ) from app.utils import APIResponse @@ -17,6 +20,26 @@ logger = logging.getLogger(__name__) GENERIC_ERROR_DETAIL = "An unexpected error occurred." +DB_ERROR_DETAIL = "Database error: request could not be completed." + +# Subclasses of SQLAlchemyError, so the first isinstance match wins. +_DB_ERROR_HINTS: tuple[tuple[type[SQLAlchemyError], int, str], ...] = ( + ( + IntegrityError, + HTTP_409_CONFLICT, + "Database error: request conflicts with existing data.", + ), + ( + DataError, + HTTP_422_UNPROCESSABLE_CONTENT, + "Database error: a value is invalid or too long.", + ), + ( + OperationalError, + HTTP_503_SERVICE_UNAVAILABLE, + "Database error: service temporarily unavailable, please retry.", + ), +) _BRANCH_PATTERN = re.compile(r"^[A-Z]|[\[\]()]") @@ -91,6 +114,14 @@ def _sanitize_validation_errors(errors: list[dict]) -> list[dict]: return errors +def _db_error_response(exc: SQLAlchemyError) -> tuple[int, str]: + """Map a DB exception to a status and a client-safe hint; SQL and parameters stay server-side.""" + for exc_type, status_code, hint in _DB_ERROR_HINTS: + if isinstance(exc, exc_type): + return status_code, hint + return HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL + + def register_exception_handlers(app: FastAPI) -> None: @app.exception_handler(RequestValidationError) async def validation_error_handler( @@ -128,6 +159,19 @@ async def http_exception_handler( content=APIResponse.failure_response(detail).model_dump(), ) + @app.exception_handler(SQLAlchemyError) + async def db_error_handler(request: Request, exc: SQLAlchemyError) -> JSONResponse: + logger.error( + f"[db_error_handler] Unhandled DB error | path: {request.url.path}, " + f"type: {type(exc).__name__}, trace_id: {correlation_id.get() or 'N/A'}", + exc_info=exc, + ) + status_code, hint = _db_error_response(exc) + return JSONResponse( + status_code=status_code, + content=APIResponse.failure_response(hint).model_dump(), + ) + @app.exception_handler(Exception) async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index 0b4368980..6c81b0502 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -3,7 +3,7 @@ All paths relative to `backend/app/`. ## Global handlers -- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. +- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. Unhandled `SQLAlchemyError` gets its own handler: a "Database error: ..." hint with 409 (`IntegrityError`), 422 (`DataError`), 503 (`OperationalError`) or 500 (any other); SQL and parameters only go to logs. - `core/middleware.py` — request-level middleware. ## Status-code selection From 162480fefd365f6469cd9f0c1e82562a8eac5d2e Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:31:59 +0530 Subject: [PATCH 06/12] test: Enhance database error handling tests with specific status codes --- .../app/tests/core/test_exception_handlers.py | 63 ++++++++++++++++++- backend/app/tests/crud/config/test_config.py | 47 ++++++++++++++ backend/app/tests/crud/config/test_version.py | 45 +++++++++++++ backend/app/tests/crud/test_credentials.py | 55 ++++++++++++++++ 4 files changed, 209 insertions(+), 1 deletion(-) diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index e89ca16b5..fe51b2970 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,12 +1,19 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from pydantic import BaseModel +from sqlalchemy.exc import ( + DataError, + IntegrityError, + OperationalError, + ProgrammingError, +) from app.core.config import settings from app.core.exception_handlers import ( + DB_ERROR_DETAIL, GENERIC_ERROR_DETAIL, _sanitize_validation_errors, register_exception_handlers, @@ -224,3 +231,57 @@ def test_does_not_leak_exception_text(self) -> None: assert response.status_code == 500 assert response.json()["error"] == GENERIC_ERROR_DETAIL assert "arn:aws:kms" not in response.text + + +class TestDbErrorHandler: + @pytest.mark.parametrize( + ("exc", "status_code"), + [ + (IntegrityError("INSERT", {"input": "9876543210"}, Exception("dup")), 409), + (DataError("INSERT", {}, Exception("too long")), 422), + (OperationalError("SELECT", {}, Exception("conn refused")), 503), + (ProgrammingError("SELECT", {}, Exception("bad sql")), 500), + ], + ) + def test_maps_status_without_leaking_sql( + self, exc: Exception, status_code: int + ) -> None: + response = _app_raising(exc).get("/boom") + + assert response.status_code == status_code + assert response.json()["error"].startswith("Database error:") + assert "INSERT" not in response.text + assert "9876543210" not in response.text + + def test_unmapped_db_error_uses_default_detail(self) -> None: + response = _app_raising(ProgrammingError("SELECT", {}, Exception("x"))).get( + "/boom" + ) + + assert response.json()["error"] == DB_ERROR_DETAIL + + +class TestHttpExceptionHandler: + def test_server_error_is_logged(self) -> None: + with patch("app.core.exception_handlers.logger.error") as log_error: + response = _app_raising( + HTTPException(status_code=502, detail="upstream failed") + ).get("/boom") + + assert response.status_code == 502 + log_error.assert_called_once() + + def test_tags_status_class_when_sentry_active(self) -> None: + client = MagicMock() + client.is_active.return_value = True + with ( + patch( + "app.core.exception_handlers.sentry_sdk.get_client", + return_value=client, + ), + patch("app.core.exception_handlers.sentry_sdk.set_tag") as set_tag, + ): + _app_raising(HTTPException(status_code=404, detail="nope")).get("/boom") + + set_tag.assert_any_call("http.status_code", "404") + set_tag.assert_any_call("error.class", "client") diff --git a/backend/app/tests/crud/config/test_config.py b/backend/app/tests/crud/config/test_config.py index e3c939ba6..bcf9196e1 100644 --- a/backend/app/tests/crud/config/test_config.py +++ b/backend/app/tests/crud/config/test_config.py @@ -2,6 +2,7 @@ import pytest from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session from app.crud.config import ConfigCrud @@ -569,3 +570,49 @@ def test_config_tag_is_immutable(db: Session) -> None: with pytest.raises(ValueError, match="tag is immutable"): config.tag = ConfigTag.ASSESSMENT + + +def test_create_config_concurrent_duplicate_raises_409( + db: Session, example_config_blob: ConfigBlob, monkeypatch +) -> None: + """A unique violation on commit (lost race past the name pre-check) is a 409.""" + project = create_test_project(db) + config_crud = ConfigCrud(session=db, project_id=project.id) + + def _fail_commit() -> None: + raise IntegrityError("INSERT", {}, Exception("duplicate key")) + + monkeypatch.setattr(db, "commit", _fail_commit) + + with pytest.raises(HTTPException) as exc: + config_crud.create_or_raise( + ConfigCreate( + name=f"test-config-{random_lower_string()}", + config_blob=example_config_blob, + ) + ) + + assert exc.value.status_code == 409 + + +def test_create_config_http_exception_propagates( + db: Session, example_config_blob: ConfigBlob, monkeypatch +) -> None: + """A typed HTTPException inside the write keeps its status, not a 500.""" + project = create_test_project(db) + config_crud = ConfigCrud(session=db, project_id=project.id) + + def _fail_add(*_: object, **__: object) -> None: + raise HTTPException(status_code=503, detail="unavailable") + + monkeypatch.setattr(db, "add", _fail_add) + + with pytest.raises(HTTPException) as exc: + config_crud.create_or_raise( + ConfigCreate( + name=f"test-config-{random_lower_string()}", + config_blob=example_config_blob, + ) + ) + + assert exc.value.status_code == 503 diff --git a/backend/app/tests/crud/config/test_version.py b/backend/app/tests/crud/config/test_version.py index ed122b721..ffe6a7b46 100644 --- a/backend/app/tests/crud/config/test_version.py +++ b/backend/app/tests/crud/config/test_version.py @@ -3,6 +3,7 @@ import pytest from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session from app.crud.config import ConfigVersionCrud @@ -650,3 +651,47 @@ def test_does_not_mutate_shared_params_dict(self) -> None: # prior_blob must NOT have been mutated through the shared reference assert prior_blob["completion"]["params"]["temperature"] == 0.7 assert shared_params["temperature"] == 0.7 + + +def test_create_version_concurrent_duplicate_raises_409( + db: Session, example_config_blob: ConfigBlob, monkeypatch +) -> None: + """A version-number collision on commit is a 409.""" + config = create_test_config(db) + version_crud = ConfigVersionCrud( + session=db, project_id=config.project_id, config_id=config.id + ) + + def _fail_commit() -> None: + raise IntegrityError("INSERT", {}, Exception("duplicate key")) + + monkeypatch.setattr(db, "commit", _fail_commit) + + with pytest.raises(HTTPException) as exc: + version_crud.create_or_raise( + ConfigVersionUpdate(config_blob=example_config_blob.model_dump()) + ) + + assert exc.value.status_code == 409 + + +def test_create_version_http_exception_propagates( + db: Session, example_config_blob: ConfigBlob, monkeypatch +) -> None: + """A typed HTTPException inside the write keeps its status, not a 500.""" + config = create_test_config(db) + version_crud = ConfigVersionCrud( + session=db, project_id=config.project_id, config_id=config.id + ) + monkeypatch.setattr( + version_crud, + "_get_next_version", + MagicMock(side_effect=HTTPException(status_code=503, detail="unavailable")), + ) + + with pytest.raises(HTTPException) as exc: + version_crud.create_or_raise( + ConfigVersionUpdate(config_blob=example_config_blob.model_dump()) + ) + + assert exc.value.status_code == 503 diff --git a/backend/app/tests/crud/test_credentials.py b/backend/app/tests/crud/test_credentials.py index d150b5b2c..84fe6cec0 100644 --- a/backend/app/tests/crud/test_credentials.py +++ b/backend/app/tests/crud/test_credentials.py @@ -3,6 +3,7 @@ import pytest from fastapi import HTTPException from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError from sqlmodel import Session from app.crud import ( @@ -472,3 +473,57 @@ def test_duplicate_provider_raises_409(db: Session) -> None: assert exc.value.status_code == 409 assert "already exist" in exc.value.detail + + +def test_non_duplicate_integrity_error_raises_500(db: Session, monkeypatch) -> None: + """An IntegrityError that is not the unique constraint is a server fault.""" + project = create_test_project(db) + + def _fail_commit() -> None: + raise IntegrityError("INSERT", {}, Exception("foreign key violation")) + + monkeypatch.setattr(db, "commit", _fail_commit) + + with pytest.raises(HTTPException) as exc: + set_creds_for_org( + session=db, + creds_add=CredsCreate( + is_active=True, credential={"openai": {"api_key": "sk-test"}} + ), + organization_id=project.organization_id, + project_id=project.id, + ) + + assert exc.value.status_code == 500 + + +def test_update_creds_without_provider_raises_400(db: Session) -> None: + project = create_test_project(db) + + with pytest.raises(HTTPException) as exc: + update_creds_for_org( + session=db, + org_id=project.organization_id, + project_id=project.id, + creds_in=CredsUpdate.model_construct(provider=None, credential=None), + ) + + assert exc.value.status_code == 400 + + +def test_remove_creds_for_org_race_deletes_nothing_raises_404( + db: Session, monkeypatch +) -> None: + """Rows vanish between the existence check and the delete: 404, not 500.""" + _, project = create_test_credential(db) + monkeypatch.setattr( + "app.crud.credentials.get_creds_by_org", lambda **k: [MagicMock()] + ) + monkeypatch.setattr(db, "exec", lambda *a, **k: MagicMock(rowcount=0)) + + with pytest.raises(HTTPException) as exc: + remove_creds_for_org( + session=db, org_id=project.organization_id, project_id=project.id + ) + + assert exc.value.status_code == 404 From 84950d3ee37de4f1862bd29ecd94edf07995c1f5 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:06:36 +0530 Subject: [PATCH 07/12] Refactor error handling for database integrity violations - Updated tests to propagate IntegrityError instead of raising HTTPException for unique constraint violations in config and version creation. - Introduced make_integrity_error utility to simulate IntegrityError with SQLSTATE and constraint details. - Enhanced test coverage for handling foreign key violations and ensuring proper logging behavior for duplicate run names in evaluation runs. - Adjusted exception handling in credentials management to correctly propagate unique constraint violations and other integrity errors. - Improved documentation on exception handling and observability practices related to error logging and status code consistency. --- .claude/conventions/celery.md | 24 +- .../app/api/docs/evaluation/improve_prompt.md | 2 +- backend/app/celery/tasks/job_execution.py | 304 ++++++------------ backend/app/celery/utils.py | 257 ++++++++------- backend/app/core/exception_handlers.py | 82 +++-- backend/app/crud/config/config.py | 27 +- backend/app/crud/config/version.py | 27 +- backend/app/crud/credentials.py | 30 +- backend/app/crud/evaluations/core.py | 4 + backend/app/crud/onboarding.py | 16 +- backend/app/services/evaluations/fast.py | 3 +- .../evaluations/prompt_improvement.py | 4 +- .../tests/api/routes/test_improve_prompt.py | 4 +- .../app/tests/core/test_exception_handlers.py | 112 ++++++- backend/app/tests/crud/config/test_config.py | 33 +- backend/app/tests/crud/config/test_version.py | 11 +- .../app/tests/crud/evaluations/test_core.py | 43 ++- backend/app/tests/crud/test_credentials.py | 17 +- backend/app/tests/crud/test_onboarding.py | 16 +- backend/app/tests/utils/utils.py | 12 + .../kaapi-llm-call-ARCHITECTURE.md | 4 +- .../kaapi-observability-ARCHITECTURE.md | 2 +- docs/wiki/cross-cutting/exceptions.md | 3 +- docs/wiki/cross-cutting/observability.md | 4 +- features/status-code-consistency/PLAN.md | 195 ----------- 25 files changed, 538 insertions(+), 698 deletions(-) delete mode 100644 features/status-code-consistency/PLAN.md diff --git a/.claude/conventions/celery.md b/.claude/conventions/celery.md index f83413f0a..d21d9c414 100644 --- a/.claude/conventions/celery.md +++ b/.claude/conventions/celery.md @@ -4,7 +4,7 @@ Authoritative conventions for Celery tasks in kaapi-backend. Tasks live in `app/ Celery uses RabbitMQ as broker. There is a **single `default` queue** declared with `x-max-priority=10`; tasks are ordered by a per-task `priority` (higher drains first, FIFO within a band). Read `app/celery/tasks/job_execution.py` before writing — it shows the full pattern -(decorator + timeout + OTel propagation + delegation to a service). +(decorator + timeout + delegation to a service). ## Canonical decorator stack @@ -15,20 +15,18 @@ def run_my_job(self, project_id: int, job_id: str, trace_id: str, **kwargs): from app.services.my_domain.jobs import do_the_work # late import to avoid cycles _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: do_the_work( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return do_the_work( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) ``` -`_set_trace`, `_run_with_otel_parent`, and `gevent_timeout` already exist in this module / -`app/celery/utils.py` — reuse them, don't reinvent. +`_set_trace` and `gevent_timeout` already exist in this module / `app/celery/utils.py` — reuse +them, don't reinvent. Enqueue through a `start_*` helper in `app/celery/utils.py`, which calls +plain `task.apply_async(kwargs={...})`. ## Priority choice — be explicit @@ -51,7 +49,7 @@ inherits its priority unless you override it. - **`bind=True`** so you have `self` (the task instance) for retries, IDs, etc. - **Always pass `queue="default"` and an explicit `priority`** — there is only one queue; the priority is what matters. - **Pass `trace_id` explicitly** as a parameter and call `_set_trace(trace_id)` first thing. This wires `asgi_correlation_id` so logs from inside the task match the originating request. -- **Wrap the work in `_run_with_otel_parent(self, lambda: ...)`** so OpenTelemetry parent context propagates from the enqueueing process. +- **Don't propagate trace context by hand.** OTel's `CeleryInstrumentor` already parents the worker's `run/...` span to the enqueue span. Injecting headers on enqueue or attaching an extracted context in the task reparents the task's spans to the API request, and Sentry splits them into separate transactions. - **Delegate to a service.** The task body should be a thin shim over `app/services//`. No DB queries, no external HTTP, no business logic inside the task itself. - **Late-import the service inside the function body** (as the canonical pattern does). Celery workers boot faster and you avoid model-import cycles. - **Idempotency.** Celery will redeliver. Either: diff --git a/backend/app/api/docs/evaluation/improve_prompt.md b/backend/app/api/docs/evaluation/improve_prompt.md index 811e58b86..c81a77750 100644 --- a/backend/app/api/docs/evaluation/improve_prompt.md +++ b/backend/app/api/docs/evaluation/improve_prompt.md @@ -56,4 +56,4 @@ LLM and trace-download failures surface only via the failure callback, not on th - `409 source_config_unavailable` — the run's config or config_version has been deleted. - `422 traces_not_available` — the run has no `score_trace_url`; trace file is required. - `422 invalid_callback_url` — `callback_url` is missing, not `https://`, or resolves to a blocked (private/loopback/metadata) address. -- `500 prompt_improvement_enqueue_failed` — the job could not be queued. +- `503 prompt_improvement_enqueue_failed` — the job could not be queued; retry shortly. diff --git a/backend/app/celery/tasks/job_execution.py b/backend/app/celery/tasks/job_execution.py index 1288876a8..5bd5db464 100644 --- a/backend/app/celery/tasks/job_execution.py +++ b/backend/app/celery/tasks/job_execution.py @@ -13,15 +13,11 @@ """ import logging -from collections.abc import Callable -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING from asgi_correlation_id import correlation_id from celery import Task, current_task from gevent import Timeout -from opentelemetry import context as otel_context -from opentelemetry import trace -from opentelemetry.propagate import extract from app.celery.celery_app import celery_app from app.celery.utils import gevent_timeout @@ -35,8 +31,6 @@ logger = logging.getLogger(__name__) -T = TypeVar("T") - # Sentinel correlation id used when no trace id is propagated from the # enqueueing request. Matches the codebase-wide "N/A" default (see # app/core/logger.py and app/celery/utils.py). @@ -50,60 +44,18 @@ def _set_trace(trace_id: str) -> None: logger.info(f"[_set_trace] Set correlation ID: {trace_id}") -def _extract_parent_context(task_instance: Task) -> otel_context.Context: - """Extract OTel parent context from Celery headers if available.""" - headers = getattr(task_instance.request, "headers", None) or {} - carrier: dict[str, str] = {} - - if isinstance(headers, dict): - for key, value in headers.items(): - if isinstance(value, str): - carrier[str(key)] = value - - nested = headers.get("otel", {}) - if isinstance(nested, dict): - for key, value in nested.items(): - if isinstance(value, str): - carrier[str(key)] = value - - return extract(carrier) - - -def _run_with_otel_parent( - task_instance: Task, fn: Callable[[], T] -) -> T: # noqa: UP047 (black doesn't support PEP 695 generics yet) - """Attach the extracted parent context and execute `fn` under it. - - Needed because otel's Celery instrumentation misses propagation headers - under `task.request.headers`, leaving `run/...` spans unparented. - """ - parent_ctx = _extract_parent_context(task_instance) - parent_span_ctx = trace.get_current_span(parent_ctx).get_span_context() - if not (parent_span_ctx and parent_span_ctx.is_valid): - return fn() - - token = otel_context.attach(parent_ctx) - try: - return fn() - finally: - otel_context.detach(token) - - @celery_app.task(bind=True, queue="default", priority=9) @gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_llm_job") def run_llm_job(self, project_id: int, job_id: str, trace_id: str, **kwargs): from app.services.llm.jobs import execute_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -113,15 +65,12 @@ def run_llm_chain_job(self, project_id: int, job_id: str, trace_id: str, **kwarg from app.services.llm.jobs import execute_chain_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_chain_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_chain_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -131,15 +80,12 @@ def run_response_job(self, project_id: int, job_id: str, trace_id: str, **kwargs from app.services.response.jobs import execute_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -149,15 +95,12 @@ def run_guardrails_job(self, project_id: int, job_id: str, trace_id: str, **kwar from app.services.guardrails.jobs import execute_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -167,15 +110,12 @@ def run_doctransform_job(self, project_id: int, job_id: str, trace_id: str, **kw from app.services.doctransform.job import execute_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -187,15 +127,12 @@ def run_collection_setup_job( from app.services.collections.create_collection import execute_setup_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_setup_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_setup_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -207,15 +144,12 @@ def run_collection_batch_job( from app.services.collections.create_collection import execute_batch_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_batch_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_batch_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -227,15 +161,12 @@ def run_delete_collection_job( from app.services.collections.delete_collection import execute_job _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_job( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_job( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -247,15 +178,12 @@ def run_evaluation_batch_submission( from app.services.evaluations.batch_job import execute_evaluation_batch_submission _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_evaluation_batch_submission( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_evaluation_batch_submission( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -267,14 +195,11 @@ def run_prompt_improvement(self, project_id: int, job_id: str, trace_id: str, ** from app.services.evaluations.prompt_improvement import execute_prompt_improvement _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_prompt_improvement( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - **kwargs, - ), + return execute_prompt_improvement( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + **kwargs, ) @@ -286,15 +211,12 @@ def run_stt_batch_submission( from app.services.stt_evaluations.batch_job import execute_batch_submission _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_batch_submission( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_batch_submission( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -306,15 +228,12 @@ def run_stt_metric_computation( from app.services.stt_evaluations.metric_job import execute_metric_computation _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_metric_computation( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_metric_computation( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -326,15 +245,12 @@ def run_tts_batch_submission( from app.services.tts_evaluations.batch_job import execute_batch_submission _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_batch_submission( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_batch_submission( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -351,13 +267,10 @@ def run_assessment_pipeline( from app.services.assessment.tasks import execute_assessment_pipeline _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_assessment_pipeline( - run_id=run_id, - organization_id=organization_id, - project_id=project_id, - ), + return execute_assessment_pipeline( + run_id=run_id, + organization_id=organization_id, + project_id=project_id, ) @@ -389,13 +302,10 @@ def run_assessment_api_batch( ) _set_trace(trace_id) - result = _run_with_otel_parent( - self, - lambda: run_batch_stage( - execution_id=execution_id, - organization_id=organization_id, - project_id=project_id, - ), + result = run_batch_stage( + execution_id=execution_id, + organization_id=organization_id, + project_id=project_id, ) if result and result.get("requeue"): self.apply_async( @@ -421,15 +331,12 @@ def run_tts_result_processing( ) _set_trace(trace_id) - return _run_with_otel_parent( - self, - lambda: execute_tts_result_processing( - project_id=project_id, - job_id=job_id, - task_id=current_task.request.id, - task_instance=self, - **kwargs, - ), + return execute_tts_result_processing( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, ) @@ -456,11 +363,8 @@ def run_evaluation_fast_chunk( ) with log_context(module=LogModuleEnum.FASTEVALS): - return _run_with_otel_parent( - self, - lambda: execute_fast_evaluation_chunk( - eval_run_id=eval_run_id, chunk_index=chunk_index - ), + return execute_fast_evaluation_chunk( + eval_run_id=eval_run_id, chunk_index=chunk_index ) @@ -484,10 +388,7 @@ def run_evaluation_fast_aggregate( ) with log_context(module=LogModuleEnum.FASTEVALS): - return _run_with_otel_parent( - self, - lambda: execute_fast_evaluation_aggregate(eval_run_id=eval_run_id), - ) + return execute_fast_evaluation_aggregate(eval_run_id=eval_run_id) # Priority 6 (fast-eval tier): same band as run_prompt_improvement — one graph @@ -524,15 +425,12 @@ def run_evaluation_iteration_graph_step( f"iteration_run_id={iteration_run_id} | resume={resume} | " f"task_id={current_task.request.id}" ) - return _run_with_otel_parent( - self, - lambda: execute_evaluation_iteration_graph_step( - iteration_run_id=iteration_run_id, - resume=resume, - organization_id=organization_id, - project_id=project_id, - **kwargs, - ), + return execute_evaluation_iteration_graph_step( + iteration_run_id=iteration_run_id, + resume=resume, + organization_id=organization_id, + project_id=project_id, + **kwargs, ) diff --git a/backend/app/celery/utils.py b/backend/app/celery/utils.py index 3bd205d22..160f3cb91 100644 --- a/backend/app/celery/utils.py +++ b/backend/app/celery/utils.py @@ -9,7 +9,6 @@ from celery.result import AsyncResult from gevent import Timeout -from opentelemetry.propagate import inject from app.celery.celery_app import celery_app @@ -18,22 +17,17 @@ F = TypeVar("F", bound=Callable[..., Any]) -def _enqueue_with_trace_context(task, **kwargs) -> str: - """Publish Celery task with explicit trace context headers.""" - otel_headers: dict[str, str] = {} - inject(otel_headers) - celery_headers = dict(otel_headers) - celery_headers["otel"] = otel_headers - async_result = task.apply_async(kwargs=kwargs, headers=celery_headers) - return async_result.id - - def start_llm_job(project_id: int, job_id: str, trace_id: str = "N/A", **kwargs) -> str: from app.celery.tasks.job_execution import run_llm_job - task_id = _enqueue_with_trace_context( - run_llm_job, project_id=project_id, job_id=job_id, trace_id=trace_id, **kwargs - ) + task_id = run_llm_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info(f"[start_llm_job] Started job {job_id} with Celery task {task_id}") return task_id @@ -43,13 +37,14 @@ def start_llm_chain_job( ) -> str: from app.celery.tasks.job_execution import run_llm_chain_job - task_id = _enqueue_with_trace_context( - run_llm_chain_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_llm_chain_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_llm_chain_job] Started job {job_id} with Celery task {task_id}" ) @@ -61,13 +56,14 @@ def start_guardrails_job( ) -> str: from app.celery.tasks.job_execution import run_guardrails_job - task_id = _enqueue_with_trace_context( - run_guardrails_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_guardrails_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_guardrails_job] Started job {job_id} with Celery task {task_id}" ) @@ -79,13 +75,14 @@ def start_response_job( ) -> str: from app.celery.tasks.job_execution import run_response_job - task_id = _enqueue_with_trace_context( - run_response_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_response_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info(f"[start_response_job] Started job {job_id} with Celery task {task_id}") return task_id @@ -95,13 +92,14 @@ def start_doctransform_job( ) -> str: from app.celery.tasks.job_execution import run_doctransform_job - task_id = _enqueue_with_trace_context( - run_doctransform_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_doctransform_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_doctransform_job] Started job {job_id} with Celery task {task_id}" ) @@ -113,13 +111,14 @@ def start_collection_setup_job( ) -> str: from app.celery.tasks.job_execution import run_collection_setup_job - task_id = _enqueue_with_trace_context( - run_collection_setup_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_collection_setup_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_collection_setup_job] Started job {job_id} with Celery task {task_id}" ) @@ -131,13 +130,14 @@ def start_collection_batch_job( ) -> str: from app.celery.tasks.job_execution import run_collection_batch_job - task_id = _enqueue_with_trace_context( - run_collection_batch_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_collection_batch_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_collection_batch_job] Started job {job_id} with Celery task {task_id}" ) @@ -149,13 +149,14 @@ def start_delete_collection_job( ) -> str: from app.celery.tasks.job_execution import run_delete_collection_job - task_id = _enqueue_with_trace_context( - run_delete_collection_job, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_delete_collection_job.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_delete_collection_job] Started job {job_id} with Celery task {task_id}" ) @@ -167,13 +168,14 @@ def start_evaluation_batch_submission( ) -> str: from app.celery.tasks.job_execution import run_evaluation_batch_submission - task_id = _enqueue_with_trace_context( - run_evaluation_batch_submission, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_evaluation_batch_submission.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_evaluation_batch_submission] Started job {job_id} with Celery task {task_id}" ) @@ -185,13 +187,14 @@ def start_prompt_improvement( ) -> str: from app.celery.tasks.job_execution import run_prompt_improvement - task_id = _enqueue_with_trace_context( - run_prompt_improvement, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_prompt_improvement.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_prompt_improvement] Started job {job_id} with Celery task {task_id}" ) @@ -203,13 +206,14 @@ def start_stt_batch_submission( ) -> str: from app.celery.tasks.job_execution import run_stt_batch_submission - task_id = _enqueue_with_trace_context( - run_stt_batch_submission, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_stt_batch_submission.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_stt_batch_submission] Started job {job_id} with Celery task {task_id}" ) @@ -221,13 +225,14 @@ def start_stt_metric_computation( ) -> str: from app.celery.tasks.job_execution import run_stt_metric_computation - task_id = _enqueue_with_trace_context( - run_stt_metric_computation, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_stt_metric_computation.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_stt_metric_computation] Started job {job_id} with Celery task {task_id}" ) @@ -239,13 +244,14 @@ def start_tts_batch_submission( ) -> str: from app.celery.tasks.job_execution import run_tts_batch_submission - task_id = _enqueue_with_trace_context( - run_tts_batch_submission, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_tts_batch_submission.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_tts_batch_submission] Started job {job_id} with Celery task {task_id}" ) @@ -257,13 +263,14 @@ def start_tts_result_processing( ) -> str: from app.celery.tasks.job_execution import run_tts_result_processing - task_id = _enqueue_with_trace_context( - run_tts_result_processing, - project_id=project_id, - job_id=job_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_tts_result_processing.apply_async( + kwargs={ + "project_id": project_id, + "job_id": job_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_tts_result_processing] Started job {job_id} with Celery task {task_id}" ) @@ -276,12 +283,13 @@ def start_fast_evaluation_chunk( """Enqueue one parallel responses-chunk task for a fast EvaluationRun.""" from app.celery.tasks.job_execution import run_evaluation_fast_chunk - task_id = _enqueue_with_trace_context( - run_evaluation_fast_chunk, - eval_run_id=eval_run_id, - chunk_index=chunk_index, - trace_id=trace_id, - ) + task_id = run_evaluation_fast_chunk.apply_async( + kwargs={ + "eval_run_id": eval_run_id, + "chunk_index": chunk_index, + "trace_id": trace_id, + } + ).id logger.info( f"[start_fast_evaluation_chunk] Enqueued fast eval chunk | " f"eval_run_id={eval_run_id} | chunk_index={chunk_index} | task_id={task_id}" @@ -293,11 +301,9 @@ def start_fast_evaluation_aggregate(eval_run_id: int, trace_id: str = "N/A") -> """Enqueue the fan-in aggregate task once a fast run's chunks are all done.""" from app.celery.tasks.job_execution import run_evaluation_fast_aggregate - task_id = _enqueue_with_trace_context( - run_evaluation_fast_aggregate, - eval_run_id=eval_run_id, - trace_id=trace_id, - ) + task_id = run_evaluation_fast_aggregate.apply_async( + kwargs={"eval_run_id": eval_run_id, "trace_id": trace_id} + ).id logger.info( f"[start_fast_evaluation_aggregate] Enqueued fast eval aggregate | " f"eval_run_id={eval_run_id} | task_id={task_id}" @@ -321,15 +327,16 @@ def start_evaluation_iteration_round( """ from app.celery.tasks.job_execution import run_evaluation_iteration_graph_step - task_id = _enqueue_with_trace_context( - run_evaluation_iteration_graph_step, - iteration_run_id=iteration_run_id, - resume=resume, - organization_id=organization_id, - project_id=project_id, - trace_id=trace_id, - **kwargs, - ) + task_id = run_evaluation_iteration_graph_step.apply_async( + kwargs={ + "iteration_run_id": iteration_run_id, + "resume": resume, + "organization_id": organization_id, + "project_id": project_id, + "trace_id": trace_id, + **kwargs, + } + ).id logger.info( f"[start_evaluation_iteration_round] Enqueued | " f"iteration_run_id={iteration_run_id} | resume={resume} | task_id={task_id}" diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index ed6afb792..384a546c7 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -22,24 +22,30 @@ GENERIC_ERROR_DETAIL = "An unexpected error occurred." DB_ERROR_DETAIL = "Database error: request could not be completed." -# Subclasses of SQLAlchemyError, so the first isinstance match wins. -_DB_ERROR_HINTS: tuple[tuple[type[SQLAlchemyError], int, str], ...] = ( - ( - IntegrityError, +UNIQUE_VIOLATION_SQLSTATE = "23505" +FOREIGN_KEY_VIOLATION_SQLSTATE = "23503" + +# Other IntegrityError codes (e.g. not-null 23502) mean our write was wrong: 500. +_INTEGRITY_ERROR_HINTS: dict[str, tuple[int, str]] = { + UNIQUE_VIOLATION_SQLSTATE: ( HTTP_409_CONFLICT, "Database error: request conflicts with existing data.", ), - ( - DataError, + FOREIGN_KEY_VIOLATION_SQLSTATE: ( HTTP_422_UNPROCESSABLE_CONTENT, - "Database error: a value is invalid or too long.", - ), - ( - OperationalError, - HTTP_503_SERVICE_UNAVAILABLE, - "Database error: service temporarily unavailable, please retry.", + "Database error: request references a record that does not exist.", ), -) +} + +# Unique violations on these constraints get a specific message; values stay out of it. +_UNIQUE_CONSTRAINT_DETAILS: dict[str, str] = { + "uq_config_project_id_name_active": "A config with this name already exists in this project.", + "uq_config_version_config_id_version": "A version with this number already exists for this config.", + "uq_credential_org_project_provider": "Credentials for this provider already exist for this organization and project.", + "ix_organization_name": "An organization with this name already exists.", + "uq_project_name_org_id": "A project with this name already exists in this organization.", + "ix_user_email": "A user with this email already exists.", +} _BRANCH_PATTERN = re.compile(r"^[A-Z]|[\[\]()]") @@ -114,11 +120,35 @@ def _sanitize_validation_errors(errors: list[dict]) -> list[dict]: return errors +def _db_error_constraint(exc: SQLAlchemyError) -> tuple[str | None, str | None]: + """Postgres SQLSTATE and violated constraint name from the driver error, when present.""" + orig = getattr(exc, "orig", None) + diag = getattr(orig, "diag", None) + return getattr(orig, "sqlstate", None), getattr(diag, "constraint_name", None) + + def _db_error_response(exc: SQLAlchemyError) -> tuple[int, str]: """Map a DB exception to a status and a client-safe hint; SQL and parameters stay server-side.""" - for exc_type, status_code, hint in _DB_ERROR_HINTS: - if isinstance(exc, exc_type): - return status_code, hint + if isinstance(exc, IntegrityError): + sqlstate, constraint = _db_error_constraint(exc) + if ( + sqlstate == UNIQUE_VIOLATION_SQLSTATE + and constraint in _UNIQUE_CONSTRAINT_DETAILS + ): + return HTTP_409_CONFLICT, _UNIQUE_CONSTRAINT_DETAILS[constraint] + return _INTEGRITY_ERROR_HINTS.get( + sqlstate, (HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL) + ) + if isinstance(exc, DataError): + return ( + HTTP_422_UNPROCESSABLE_CONTENT, + "Database error: a value is invalid or too long.", + ) + if isinstance(exc, OperationalError): + return ( + HTTP_503_SERVICE_UNAVAILABLE, + "Database error: service temporarily unavailable, please retry.", + ) return HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL @@ -161,12 +191,22 @@ async def http_exception_handler( @app.exception_handler(SQLAlchemyError) async def db_error_handler(request: Request, exc: SQLAlchemyError) -> JSONResponse: - logger.error( + status_code, hint = _db_error_response(exc) + sqlstate, constraint = _db_error_constraint(exc) + if sentry_sdk.get_client().is_active(): + sentry_sdk.set_tag("http.status_code", str(status_code)) + sentry_sdk.set_tag( + "error.class", "server" if status_code >= 500 else "client" + ) + if constraint: + sentry_sdk.set_tag("db.constraint", constraint) + log = logger.error if status_code >= 500 else logger.warning + log( f"[db_error_handler] Unhandled DB error | path: {request.url.path}, " - f"type: {type(exc).__name__}, trace_id: {correlation_id.get() or 'N/A'}", + f"type: {type(exc).__name__}, status: {status_code}, sqlstate: {sqlstate}, " + f"constraint: {constraint}, trace_id: {correlation_id.get() or 'N/A'}", exc_info=exc, ) - status_code, hint = _db_error_response(exc) return JSONResponse( status_code=status_code, content=APIResponse.failure_response(hint).model_dump(), @@ -175,11 +215,11 @@ async def db_error_handler(request: Request, exc: SQLAlchemyError) -> JSONRespon @app.exception_handler(Exception) async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. - sentry_sdk.capture_exception(exc) + # logger.error alone reports it; adding capture_exception makes a second event. logger.error( f"[generic_error_handler] Unhandled exception | " f"path: {request.url.path}, trace_id: {correlation_id.get() or 'N/A'}", - exc_info=True, + exc_info=exc, ) return JSONResponse( status_code=HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index 0ce77ee9c..175b1ec2e 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -2,7 +2,6 @@ from uuid import UUID from fastapi import HTTPException -from sqlalchemy.exc import IntegrityError from sqlmodel import Session, and_, select from app.core.util import now @@ -69,33 +68,11 @@ def create_or_raise( return config, version - except HTTPException: + except Exception: + # Each error type has a central handler (DB, HTTPException, generic). self.session.rollback() raise - except IntegrityError as e: - self.session.rollback() - logger.warning( - f"[ConfigCrud.create] Duplicate configuration name | " - f"{{'name': '{config_create.name}', 'project_id': {self.project_id}, 'error': '{str(e)}'}}", - ) - raise HTTPException( - status_code=409, - detail=f"Config with name '{config_create.name}' already exists in this project.", - ) - - except Exception as e: - self.session.rollback() - logger.error( - f"[ConfigCrud.create] Failed to create configuration | " - f"{{'name': '{config_create.name}', 'project_id': {self.project_id}, 'error': '{str(e)}'}}", - exc_info=True, - ) - raise HTTPException( - status_code=500, - detail="Unexpected error occurred: failed to create config", - ) - def read_one(self, config_id: UUID) -> Config | None: statement = select(Config).where( and_( diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index 373377970..d262b4e6e 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -3,7 +3,6 @@ from uuid import UUID from fastapi import HTTPException -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import defer from sqlmodel import Session, and_, select @@ -105,33 +104,11 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: return version - except HTTPException: + except Exception: + # Each error type has a central handler (DB, HTTPException, generic). self.session.rollback() raise - except IntegrityError as e: - self.session.rollback() - logger.warning( - f"[ConfigVersionCrud.create_or_raise] Version conflict | " - f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", - ) - raise HTTPException( - status_code=409, - detail="A version with this number already exists for this config.", - ) - - except Exception as e: - self.session.rollback() - logger.error( - f"[ConfigVersionCrud.create_or_raise] Failed to create version | " - f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", - exc_info=True, - ) - raise HTTPException( - status_code=500, - detail="Unexpected error occurred: failed to create version", - ) - def _get_latest_version(self) -> ConfigVersion | None: """Get the latest version for the config.""" stmt = ( diff --git a/backend/app/crud/credentials.py b/backend/app/crud/credentials.py index 14e45a6c6..0881472c0 100644 --- a/backend/app/crud/credentials.py +++ b/backend/app/crud/credentials.py @@ -2,7 +2,6 @@ from typing import Any from sqlalchemy import delete -from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from fastapi import HTTPException @@ -39,31 +38,10 @@ def set_creds_for_org( credential=encrypted_credentials, ) credential.inserted_at = now() - try: - session.add(credential) - session.commit() - session.refresh(credential) - created_credentials.append(credential) - except IntegrityError as e: - session.rollback() - is_duplicate = ( - "uq_credential_org_project_provider" in str(e) - or "unique constraint" in str(e).lower() - ) - log = logger.warning if is_duplicate else logger.error - log( - f"[set_creds_for_org] Integrity error while adding credentials | organization_id {organization_id}, project_id {project_id}, provider {provider}: {str(e)}", - exc_info=True, - ) - if is_duplicate: - raise HTTPException( - status_code=409, - detail=f"Credentials for provider '{provider}' already exist for this organization and project combination", - ) - raise HTTPException( - status_code=500, - detail=f"Could not store credentials for provider '{provider}'. Contact Kaapi if it persists.", - ) + session.add(credential) + session.commit() + session.refresh(credential) + created_credentials.append(credential) logger.info( f"[set_creds_for_org] Successfully created credentials | organization_id {organization_id}, project_id {project_id}" ) diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index 0f1eabaed..2fc2fe46e 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -4,6 +4,7 @@ from uuid import UUID from langfuse import Langfuse +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, col, select from app.core.cloud.storage import get_cloud_storage @@ -125,6 +126,9 @@ def create_evaluation_run( try: session.commit() session.refresh(eval_run) + except IntegrityError: + session.rollback() + raise # caller classifies: run_name conflict is a 409, the rest an error except Exception as e: session.rollback() logger.error( diff --git a/backend/app/crud/onboarding.py b/backend/app/crud/onboarding.py index 8befdba44..2370c9674 100644 --- a/backend/app/crud/onboarding.py +++ b/backend/app/crud/onboarding.py @@ -1,6 +1,5 @@ import logging from fastapi import HTTPException -from sqlalchemy.exc import IntegrityError from sqlmodel import Session from app.core.security import encrypt_credentials, get_password_hash @@ -138,20 +137,7 @@ def onboard_project( created_credentials.append(cred_row) - # Pre-checks are read-then-write; a concurrent onboard only collides here. - try: - session.commit() - except IntegrityError as e: - session.rollback() - logger.warning( - f"[onboard_project] Conflicting concurrent onboarding | " - f"org={onboard_in.organization_name}, project={onboard_in.project_name} | {e}" - ) - raise HTTPException( - status_code=409, - detail=f"Project '{onboard_in.project_name}' already exists for " - f"organization '{onboard_in.organization_name}'", - ) + session.commit() return OnboardingResponse( organization_id=organization.id, diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index 58755c187..d960919e5 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -325,8 +325,9 @@ def validate_and_start_fast_evaluation( ) total_items = original_items * max(1, effective_factor) if total_items == 0: + # Fast eval clients depend on the 500 contract for an empty dataset. raise HTTPException( - status_code=422, detail=f"Dataset '{dataset.name}' has no items" + status_code=500, detail=f"Dataset '{dataset.name}' has no items" ) n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 9b92d5c22..0f9a432b9 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -220,8 +220,8 @@ def start_prompt_improvement_job( ), ) raise HTTPException( - status_code=500, - detail="prompt_improvement_enqueue_failed: could not queue the job", + status_code=503, + detail="prompt_improvement_enqueue_failed: could not queue the job. Retry shortly.", ) logger.info( diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index 17027b399..0e2cbc41b 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -502,7 +502,7 @@ def test_invalid_callback_url_rejected_by_ssrf_guard( assert "invalid_callback_url" in resp.json()["error"], resp.text enqueue.assert_not_called() - def test_enqueue_failure_marks_job_failed_and_raises_500( + def test_enqueue_failure_marks_job_failed_and_raises_503( self, db: Session, auth: TestAuthContext, @@ -521,7 +521,7 @@ def test_enqueue_failure_marks_job_failed_and_raises_500( callback_url=_CALLBACK_URL, ) - assert exc.value.status_code == 500 + assert exc.value.status_code == 503 job = db.exec( select(Job) diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index fe51b2970..ee320901c 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,12 +1,17 @@ +import logging +from collections.abc import Iterator from unittest.mock import MagicMock, patch import pytest +import sentry_sdk from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from pydantic import BaseModel +from sentry_sdk.integrations.dedupe import DedupeIntegration +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.types import Event, Hint from sqlalchemy.exc import ( DataError, - IntegrityError, OperationalError, ProgrammingError, ) @@ -19,6 +24,7 @@ register_exception_handlers, ) from app.tests.utils.auth import TestAuthContext +from app.tests.utils.utils import make_integrity_error class TestSanitizeValidationErrors: @@ -146,16 +152,14 @@ def validate(body: Body) -> dict[str, bool]: return TestClient(app, raise_server_exceptions=False) - def test_generic_handler_captures_unhandled_exception( + def test_generic_handler_reports_unhandled_exception( self, client: TestClient ) -> None: - with patch( - "app.core.exception_handlers.sentry_sdk.capture_exception" - ) as capture: + with patch("app.core.exception_handlers.logger.error") as log_error: response = client.get("/boom") assert response.status_code == 500 - capture.assert_called_once() + log_error.assert_called_once() def test_http_exception_handler_does_not_capture(self, client: TestClient) -> None: with patch( @@ -237,7 +241,9 @@ class TestDbErrorHandler: @pytest.mark.parametrize( ("exc", "status_code"), [ - (IntegrityError("INSERT", {"input": "9876543210"}, Exception("dup")), 409), + (make_integrity_error("23505"), 409), + (make_integrity_error("23503"), 422), + (make_integrity_error("23502"), 500), (DataError("INSERT", {}, Exception("too long")), 422), (OperationalError("SELECT", {}, Exception("conn refused")), 503), (ProgrammingError("SELECT", {}, Exception("bad sql")), 500), @@ -260,6 +266,38 @@ def test_unmapped_db_error_uses_default_detail(self) -> None: assert response.json()["error"] == DB_ERROR_DETAIL + @pytest.mark.parametrize( + ("constraint", "detail"), + [ + ("uq_config_project_id_name_active", "A config with this name"), + ("uq_credential_org_project_provider", "Credentials for this provider"), + ("ix_user_email", "A user with this email"), + ], + ) + def test_known_unique_constraint_gets_specific_message( + self, constraint: str, detail: str + ) -> None: + response = _app_raising(make_integrity_error("23505", constraint)).get("/boom") + + assert response.status_code == 409 + assert response.json()["error"].startswith(detail) + + def test_unknown_unique_constraint_gets_generic_conflict(self) -> None: + response = _app_raising(make_integrity_error("23505", "uq_other")).get("/boom") + + assert response.status_code == 409 + assert response.json()["error"].startswith("Database error:") + + def test_client_side_db_error_logs_warning(self) -> None: + with ( + patch("app.core.exception_handlers.logger.warning") as log_warning, + patch("app.core.exception_handlers.logger.error") as log_error, + ): + _app_raising(make_integrity_error("23505")).get("/boom") + + log_warning.assert_called_once() + log_error.assert_not_called() + class TestHttpExceptionHandler: def test_server_error_is_logged(self) -> None: @@ -285,3 +323,63 @@ def test_tags_status_class_when_sentry_active(self) -> None: set_tag.assert_any_call("http.status_code", "404") set_tag.assert_any_call("error.class", "client") + + +@pytest.fixture +def sentry_events() -> Iterator[list[Event]]: + """Real Sentry client whose before_send records events instead of sending them.""" + events: list[Event] = [] + + def _record(event: Event, _hint: Hint) -> None: + events.append(event) + return None + + sentry_sdk.init( + dsn="https://public@sentry.invalid/1", + before_send=_record, + default_integrations=False, + auto_enabling_integrations=False, + integrations=[ + LoggingIntegration(event_level=logging.ERROR), + DedupeIntegration(), + ], + ) + yield events + sentry_sdk.get_global_scope().set_client(None) + + +class TestSentryEvents: + def test_server_http_exception_event_carries_status_tags( + self, sentry_events: list[Event] + ) -> None: + _app_raising(HTTPException(status_code=502, detail="upstream")).get("/boom") + + assert len(sentry_events) == 1 + tags = sentry_events[0]["tags"] + assert tags["http.status_code"] == "502" + assert tags["error.class"] == "server" + + def test_generic_handler_reports_exactly_one_event( + self, sentry_events: list[Event] + ) -> None: + _app_raising(ValueError("boom")).get("/boom") + + assert len(sentry_events) == 1 + + def test_db_server_error_event_carries_db_tags( + self, sentry_events: list[Event] + ) -> None: + _app_raising(make_integrity_error("23502", "llm_call_input_not_null")).get( + "/boom" + ) + + assert len(sentry_events) == 1 + tags = sentry_events[0]["tags"] + assert tags["http.status_code"] == "500" + assert tags["db.constraint"] == "llm_call_input_not_null" + + def test_db_conflict_creates_no_event(self, sentry_events: list[Event]) -> None: + response = _app_raising(make_integrity_error("23505")).get("/boom") + + assert response.status_code == 409 + assert sentry_events == [] diff --git a/backend/app/tests/crud/config/test_config.py b/backend/app/tests/crud/config/test_config.py index bcf9196e1..6c7bdac6f 100644 --- a/backend/app/tests/crud/config/test_config.py +++ b/backend/app/tests/crud/config/test_config.py @@ -15,7 +15,7 @@ from app.models.config.config import ConfigTag from app.models.llm.request import NativeCompletionConfig from app.tests.utils.test_data import create_test_config, create_test_project -from app.tests.utils.utils import random_lower_string +from app.tests.utils.utils import make_integrity_error, random_lower_string @pytest.fixture @@ -572,19 +572,19 @@ def test_config_tag_is_immutable(db: Session) -> None: config.tag = ConfigTag.ASSESSMENT -def test_create_config_concurrent_duplicate_raises_409( +def test_create_config_concurrent_duplicate_propagates( db: Session, example_config_blob: ConfigBlob, monkeypatch ) -> None: - """A unique violation on commit (lost race past the name pre-check) is a 409.""" + """A commit-time unique violation reaches the central DB handler (409).""" project = create_test_project(db) config_crud = ConfigCrud(session=db, project_id=project.id) def _fail_commit() -> None: - raise IntegrityError("INSERT", {}, Exception("duplicate key")) + raise make_integrity_error("23505") monkeypatch.setattr(db, "commit", _fail_commit) - with pytest.raises(HTTPException) as exc: + with pytest.raises(IntegrityError): config_crud.create_or_raise( ConfigCreate( name=f"test-config-{random_lower_string()}", @@ -592,8 +592,6 @@ def _fail_commit() -> None: ) ) - assert exc.value.status_code == 409 - def test_create_config_http_exception_propagates( db: Session, example_config_blob: ConfigBlob, monkeypatch @@ -616,3 +614,24 @@ def _fail_add(*_: object, **__: object) -> None: ) assert exc.value.status_code == 503 + + +def test_create_config_foreign_key_violation_is_not_reported_as_duplicate( + db: Session, example_config_blob: ConfigBlob, monkeypatch +) -> None: + """Only a unique violation is a name conflict; other IntegrityErrors propagate.""" + project = create_test_project(db) + config_crud = ConfigCrud(session=db, project_id=project.id) + + def _fail_commit() -> None: + raise make_integrity_error("23503") + + monkeypatch.setattr(db, "commit", _fail_commit) + + with pytest.raises(IntegrityError): + config_crud.create_or_raise( + ConfigCreate( + name=f"test-config-{random_lower_string()}", + config_blob=example_config_blob, + ) + ) diff --git a/backend/app/tests/crud/config/test_version.py b/backend/app/tests/crud/config/test_version.py index ffe6a7b46..a76431773 100644 --- a/backend/app/tests/crud/config/test_version.py +++ b/backend/app/tests/crud/config/test_version.py @@ -15,6 +15,7 @@ create_test_project, create_test_version, ) +from app.tests.utils.utils import make_integrity_error @pytest.fixture @@ -653,27 +654,25 @@ def test_does_not_mutate_shared_params_dict(self) -> None: assert shared_params["temperature"] == 0.7 -def test_create_version_concurrent_duplicate_raises_409( +def test_create_version_concurrent_duplicate_propagates( db: Session, example_config_blob: ConfigBlob, monkeypatch ) -> None: - """A version-number collision on commit is a 409.""" + """A commit-time unique violation reaches the central DB handler (409).""" config = create_test_config(db) version_crud = ConfigVersionCrud( session=db, project_id=config.project_id, config_id=config.id ) def _fail_commit() -> None: - raise IntegrityError("INSERT", {}, Exception("duplicate key")) + raise make_integrity_error("23505") monkeypatch.setattr(db, "commit", _fail_commit) - with pytest.raises(HTTPException) as exc: + with pytest.raises(IntegrityError): version_crud.create_or_raise( ConfigVersionUpdate(config_blob=example_config_blob.model_dump()) ) - assert exc.value.status_code == 409 - def test_create_version_http_exception_propagates( db: Session, example_config_blob: ConfigBlob, monkeypatch diff --git a/backend/app/tests/crud/evaluations/test_core.py b/backend/app/tests/crud/evaluations/test_core.py index ead0caaa2..ee1cc8db7 100644 --- a/backend/app/tests/crud/evaluations/test_core.py +++ b/backend/app/tests/crud/evaluations/test_core.py @@ -1,6 +1,9 @@ +import logging from datetime import datetime, timedelta from uuid import UUID, uuid4 +import pytest +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.core.util import now @@ -19,7 +22,7 @@ get_user_test_auth_context, ) from app.tests.utils.test_data import create_test_evaluation_dataset -from app.tests.utils.utils import random_lower_string +from app.tests.utils.utils import make_integrity_error, random_lower_string def _create_config(db: Session, project_id: int) -> tuple: @@ -117,6 +120,44 @@ def test_create_evaluation_run_sets_text_type(self, db: Session) -> None: assert eval_run.status == "pending" assert eval_run.run_name == "test_run" + def test_duplicate_run_name_is_left_to_caller_without_error_log( + self, db: Session, monkeypatch, caplog + ) -> None: + """The service turns a run_name collision into a 409; crud must not log it as an error.""" + org = db.exec(select(Organization)).first() + project = db.exec( + select(Project).where(Project.organization_id == org.id) + ).first() + dataset = create_evaluation_dataset( + session=db, + name="test_dataset_dup_run", + dataset_metadata={"original_items_count": 10}, + organization_id=org.id, + project_id=project.id, + ) + config_id, config_version = _create_config(db, project.id) + + def _fail_commit() -> None: + raise make_integrity_error( + "23505", "uq_evaluation_run_org_project_run_name" + ) + + monkeypatch.setattr(db, "commit", _fail_commit) + + with caplog.at_level(logging.WARNING), pytest.raises(IntegrityError): + create_evaluation_run( + session=db, + run_name="dup_run", + dataset_name=dataset.name, + dataset_id=dataset.id, + config_id=config_id, + config_version=config_version, + organization_id=org.id, + project_id=project.id, + ) + + assert not [r for r in caplog.records if r.levelno >= logging.ERROR] + class TestGetEvaluationRunById: """Test fetching evaluation runs by ID.""" diff --git a/backend/app/tests/crud/test_credentials.py b/backend/app/tests/crud/test_credentials.py index 84fe6cec0..0d8e9a71b 100644 --- a/backend/app/tests/crud/test_credentials.py +++ b/backend/app/tests/crud/test_credentials.py @@ -454,8 +454,8 @@ def test_remove_creds_for_org_when_none_exist_raises_404(db: Session) -> None: assert exc.value.status_code == 404 -def test_duplicate_provider_raises_409(db: Session) -> None: - """Unique-constraint violation is a conflict, not bad input.""" +def test_duplicate_provider_propagates_unique_violation(db: Session) -> None: + """The central DB handler turns this into a 409; crud lets it propagate.""" _, project = create_test_credential(db) creds_create = CredsCreate( @@ -463,7 +463,7 @@ def test_duplicate_provider_raises_409(db: Session) -> None: credential={"openai": {"api_key": "another-key"}}, ) - with pytest.raises(HTTPException) as exc: + with pytest.raises(IntegrityError) as exc: set_creds_for_org( session=db, creds_add=creds_create, @@ -471,12 +471,11 @@ def test_duplicate_provider_raises_409(db: Session) -> None: project_id=project.id, ) - assert exc.value.status_code == 409 - assert "already exist" in exc.value.detail + assert exc.value.orig.diag.constraint_name == "uq_credential_org_project_provider" -def test_non_duplicate_integrity_error_raises_500(db: Session, monkeypatch) -> None: - """An IntegrityError that is not the unique constraint is a server fault.""" +def test_integrity_error_propagates_to_db_handler(db: Session, monkeypatch) -> None: + """Classifying DB errors is the central handler's job, not crud's.""" project = create_test_project(db) def _fail_commit() -> None: @@ -484,7 +483,7 @@ def _fail_commit() -> None: monkeypatch.setattr(db, "commit", _fail_commit) - with pytest.raises(HTTPException) as exc: + with pytest.raises(IntegrityError): set_creds_for_org( session=db, creds_add=CredsCreate( @@ -494,8 +493,6 @@ def _fail_commit() -> None: project_id=project.id, ) - assert exc.value.status_code == 500 - def test_update_creds_without_provider_raises_400(db: Session) -> None: project = create_test_project(db) diff --git a/backend/app/tests/crud/test_onboarding.py b/backend/app/tests/crud/test_onboarding.py index f46ec8eae..84ac96ae2 100644 --- a/backend/app/tests/crud/test_onboarding.py +++ b/backend/app/tests/crud/test_onboarding.py @@ -21,7 +21,11 @@ APIKey, Credential, ) -from app.tests.utils.utils import random_lower_string, random_email +from app.tests.utils.utils import ( + make_integrity_error, + random_email, + random_lower_string, +) from app.tests.utils.test_data import create_test_organization, create_test_project from app.tests.utils.user import create_random_user @@ -318,17 +322,17 @@ def test_onboard_kms_failure_raises_502(db: Session, monkeypatch) -> None: assert exc.value.status_code == 502 -def test_onboard_concurrent_commit_conflict_raises_409( +def test_onboard_concurrent_commit_conflict_propagates( db: Session, monkeypatch ) -> None: - """The pre-checks are racy; a collision at commit is a 409, not a 500.""" + """A commit-time unique violation reaches the central DB handler (409).""" def _raise_integrity_error() -> None: - raise IntegrityError("INSERT", {}, Exception("duplicate key")) + raise make_integrity_error("23505") monkeypatch.setattr(db, "commit", _raise_integrity_error) - with pytest.raises(HTTPException) as exc: + with pytest.raises(IntegrityError): onboard_project( session=db, onboard_in=OnboardingRequest( @@ -339,5 +343,3 @@ def _raise_integrity_error() -> None: password=random_lower_string(), ), ) - - assert exc.value.status_code == 409 diff --git a/backend/app/tests/utils/utils.py b/backend/app/tests/utils/utils.py index 8bf7b1971..fd3a73c2b 100644 --- a/backend/app/tests/utils/utils.py +++ b/backend/app/tests/utils/utils.py @@ -1,9 +1,11 @@ import random import string +from types import SimpleNamespace from uuid import UUID from typing import Type, TypeVar from fastapi.testclient import TestClient +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.core.config import settings @@ -14,6 +16,16 @@ T = TypeVar("T") +def make_integrity_error( + sqlstate: str | None, constraint: str | None = None +) -> IntegrityError: + """IntegrityError whose driver error carries psycopg's sqlstate and diag, as Postgres raises it.""" + orig = Exception("integrity violation") + orig.sqlstate = sqlstate # type: ignore[attr-defined] + orig.diag = SimpleNamespace(constraint_name=constraint) # type: ignore[attr-defined] + return IntegrityError("INSERT", {"input": "9876543210"}, orig) + + def random_lower_string() -> str: return "".join(random.choices(string.ascii_lowercase, k=32)) diff --git a/docs/architecture/kaapi-llm-call-ARCHITECTURE.md b/docs/architecture/kaapi-llm-call-ARCHITECTURE.md index 92ce5e065..5034124ab 100644 --- a/docs/architecture/kaapi-llm-call-ARCHITECTURE.md +++ b/docs/architecture/kaapi-llm-call-ARCHITECTURE.md @@ -89,7 +89,7 @@ backend/app/ │ ├── celery/ │ ├── tasks/job_execution.py run_llm_job (high_priority, priority=9, soft timeout) -│ └── utils.py start_llm_job() — enqueue with OTel trace headers +│ └── utils.py start_llm_job() — enqueue; OTel's Celery instrumentation carries the trace > **Worker concurrency model:** Celery currently runs with the **prefork** pool. > Gevent is being evaluated and will replace prefork once validated in production. @@ -518,7 +518,7 @@ erDiagram ## 9. Observability -Three layers, all keyed off the same `trace_id` (the request `correlation_id`, propagated to Celery via injected OTel headers): +Three layers, all keyed off the same `trace_id` (the request `correlation_id`, passed to Celery as the task's `trace_id` argument; the OTel trace itself is carried by the Celery instrumentation): ```mermaid flowchart TD diff --git a/docs/architecture/kaapi-observability-ARCHITECTURE.md b/docs/architecture/kaapi-observability-ARCHITECTURE.md index 25eb4460a..ed6eab4f4 100644 --- a/docs/architecture/kaapi-observability-ARCHITECTURE.md +++ b/docs/architecture/kaapi-observability-ARCHITECTURE.md @@ -112,7 +112,7 @@ flowchart LR F --> Sentry[("Sentry Traces")] ``` -- **One trace from API to worker:** the Sentry Celery integration adds trace headers on enqueue, and the task re-attaches the parent. Self re-enqueuing poll loops start a fresh trace so one trace does not span hours. +- **One trace from API to worker:** OTel's Celery instrumentation carries the trace on enqueue and parents the worker's `run/...` span to `apply_async`; task spans nest under `run/...`. Trace context is never injected or attached by hand: doing so reparented task spans to the API request and split them into separate Sentry transactions. Self re-enqueuing poll loops start a fresh trace so one trace does not span hours. - **Root attributes are kept:** Sentry's processor copies attributes only for child spans; ours copies root-span attributes too. - **Provider HTTP spans are suppressed** inside LLM calls, since the `chat ` span already covers the call. - **Spans are flushed** after every Celery task and in LLM job `finally` blocks, so a recycled worker does not lose them. diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index 6c81b0502..4fe858030 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -3,7 +3,8 @@ All paths relative to `backend/app/`. ## Global handlers -- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. Unhandled `SQLAlchemyError` gets its own handler: a "Database error: ..." hint with 409 (`IntegrityError`), 422 (`DataError`), 503 (`OperationalError`) or 500 (any other); SQL and parameters only go to logs. +- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. Unhandled `SQLAlchemyError` gets its own handler: a "Database error: ..." hint with 409 (unique violation, SQLSTATE 23505), 422 (foreign key 23503, or `DataError`), 503 (`OperationalError`) or 500 (any other); 4xx logs at warning, 5xx at error; SQL and parameters only go to logs. +- DB errors are classified only in `db_error_handler`: crud/services let `SQLAlchemyError` propagate (rollback, then `raise`), never catch `IntegrityError` to build a 409 or match error text. A unique violation on a constraint listed in `_UNIQUE_CONSTRAINT_DETAILS` gets that message; add the constraint there for a specific 409. Sentry events carry `http.status_code`, `error.class` and `db.constraint` tags. Exception: eval `run_name` keeps its own 409 (`run_name_already_exists` is a client contract). - `core/middleware.py` — request-level middleware. ## Status-code selection diff --git a/docs/wiki/cross-cutting/observability.md b/docs/wiki/cross-cutting/observability.md index b60d5c93b..bc27ace54 100644 --- a/docs/wiki/cross-cutting/observability.md +++ b/docs/wiki/cross-cutting/observability.md @@ -15,8 +15,8 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai - GenAI content never reaches Sentry, in either PII mode. Four locks in `core/telemetry/sentry/filters.py`, wired by `init_sentry()` in `core/telemetry/sentry/init.py` for both API and worker: `scrub_genai_content` strips `_GENAI_CONTENT_KEYS` (prompt/completion span+log attributes) from every event, `before_send_log_filter` does the same for logs, `genai_privacy_integrations()` pins Sentry's auto-enabled AI integrations to `include_prompts=False`, and `include_local_variables=False` / `max_request_body_size="never"` keep prompts out of stack frames and request bodies. Langfuse (isolated tracer provider, `core/langfuse/langfuse.py`) stays the only sink for message bodies. - Corollary for log lines on LLM paths: log lengths/ids/types, never prompt or response text — `enable_logs=True` ships every INFO record to Sentry. - Sampling/profiling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_PROFILE_SESSION_SAMPLE_RATE`, `SENTRY_PROFILE_LIFECYCLE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior). -- Trace propagation: `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace. -- Error capture: generic handler in `core/exception_handlers.py` calls `capture_exception`. +- Trace propagation: OTel's `CeleryInstrumentor` parents the worker's `run/...` span to `apply_async`; never inject/extract trace context by hand around tasks (it reparents task spans to the API request and Sentry splits them into separate transactions). `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace. +- Error capture: handlers in `core/exception_handlers.py` report via `logger.error(exc_info=exc)` (5xx only). Don't add `capture_exception` next to it: on sentry-sdk 2.54 Dedupe does not merge the two, so each error becomes two events. - Tenant impact (both in `core/telemetry/context.py`, called from `api/deps.py`): `set_request_log_context` puts org/project in the log context + Sentry tags; `bind_sentry_user` binds user/org/project via `sentry_sdk.set_user` so issues report users/orgs affected. The user binding is scope-only — ids in the log context would stamp per-user cardinality on every INFO record `enable_logs` ships. Per-request identity is the `correlation_id` tag (`core/middleware.py`), not the user binding. - Crons: `@sentry_sdk.monitor` on `/cron/*` endpoints (`api/routes/cron.py`). - Runbook (alerts/dashboards/debugging): `features/sentry-utilization/SENTRY-RUNBOOK.md`. diff --git a/features/status-code-consistency/PLAN.md b/features/status-code-consistency/PLAN.md deleted file mode 100644 index 8b7674f5a..000000000 --- a/features/status-code-consistency/PLAN.md +++ /dev/null @@ -1,195 +0,0 @@ -# Status Code and Error Message Consistency (issue #834): Implementation Plan - -Source spec: [GitHub issue #834](https://github.com/ProjectTech4DevAI/kaapi-backend/issues/834), "Observability: Improve status codes consistency" - -## Summary - -Kaapi returns HTTP 500 for conditions that are not server bugs: a missing Langfuse dataset, a provider or storage failure, a KMS encrypt that failed, a broker that could not accept a job. Callers cannot tell "you asked for something that does not exist" from "we broke", and ops alerting on 5xx rate is polluted by caller errors. This change introduces a central domain exception taxonomy in `app/core/exceptions.py` with registered handlers, then converts the mis-coded raise sites across eight domains, and completes the log-level and log-prefix audit on the same lines. No schema change, so no Alembic migration. - -Domains in scope: the five the issue names (LLM Call, LLM Chain, Collections, Documents, Evaluations including STT/TTS), plus Onboarding, Config Management, and Credentials added by later scope decision. - -### Note on the issue's two cited line numbers - -Both citations are stale against `main`; the paths also moved under `backend/`. - -- `app/crud/assistants.py:68` is already correct. [backend/app/crud/assistants.py:64-73](backend/app/crud/assistants.py#L64-L73) raises 404 on `openai.NotFoundError` and 502 on `openai.OpenAIError`. No change needed, and `assistants` belongs to the Responses module, which is not among the in-scope domains. -- `app/crud/evaluations/batch.py:47` is now a docstring line, and no `success` boolean exists in that file. The underlying defect survives in a different shape: [backend/app/crud/evaluations/batch.py:49-58](backend/app/crud/evaluations/batch.py#L49-L58) raises a bare `ValueError` when Langfuse cannot return the dataset, which nothing catches on the fast-eval path, so the generic handler turns it into a 500. Production evidence: `app/logs/app.log.3:9975` logs `status_code: 404, body: {'message': 'Dataset not found', 'error': 'LangfuseNotFoundError'}` on a request that returned 500. Step 3 fixes it. - -### The taxonomy this plan applies - -Every code change below is an application of this table. It is derived from `.claude/conventions/route.md` ("Status codes (the ones to get right)") and `.claude/conventions/error-handling.md`. - -| Condition | Code | Log level | -|---|---|---| -| Resource absent, ours or an upstream-held resource we reference | 404 | `warning` | -| State conflict (already exists, already deleted) | 409 | `warning` | -| Payload unparseable or wrong shape (bad CSV, unreadable file) | 422 | `warning` | -| Valid shape, unacceptable value (unsupported transform pair) | 400 | `warning` | -| Upstream returned an error or was unreachable (Langfuse, OpenAI, Gemini, object storage) | 502 | `error` | -| Our async infrastructure could not accept the work (Celery broker, RabbitMQ) | 503 | `error` | -| Genuinely our bug (DB write failed, unexpected exception) | 500 | `error` | - -## Blast Radius - -Primary entities: this is a cross-cutting change to the HTTP error contract, not to any table. Entities whose read/write paths change their failure responses: `EvaluationDataset`, `EvaluationRun`, `STTSample`/`STTResult`, `TTSResult`, `Collection`, `CollectionJob`, `Document`, `DocTransformationJob`, `Job`, `LlmCall`, `LlmChain`, `Config`, `ConfigVersion`, `Credential`, `Organization`, `Project`, `User`, `APIKey`. - -| Surface | Hop | Impact | Decision | -|---|---|---|---| -| `app/core/exception_handlers.py` | 1 | New typed handler for `KaapiError`; `generic_error_handler` stops echoing `str(exc)` | in scope | -| Evaluations routes/services/crud (text, STT, TTS) | 1 | 500 becomes 404 / 422 / 502 / 503 at named sites | in scope | -| Collections crud/routes | 1 | "already deleted" 400 becomes 409; `CollectionNameConflictError` re-based on `ConflictError` | in scope | -| Documents services/crud | 1 | Unsupported file format 400 becomes 422; wrong-module `HTTPException` import corrected | in scope | -| LLM Call, LLM Chain, Guardrails job start | 1 | Celery enqueue failure 500 becomes 503 | in scope | -| Config Management (`crud/config/`) | 1 | Catch-alls swallow `IntegrityError` into 500; `ConfigBlob` validation returns 400 where the request boundary returns 422 | in scope | -| Credentials (`crud/credentials.py`, `routes/credentials.py`) | 1 | Unique-constraint 400 becomes 409; delete-miss 500 becomes 404; bare `ValueError` sites typed | in scope | -| `app/core/security.py` KMS encrypt/decrypt | 1 | `ValueError` on AWS KMS failure surfaces as 500; becomes 502 | in scope | -| Onboarding (`crud/onboarding.py`) | 1 | Unguarded `encrypt_credentials` and unguarded final `session.commit()` both surface as 500 | in scope | -| `app/tests/` status-code assertions | 1 | 8 assertions encode the current wrong codes and must flip | in scope | -| `docs/wiki/cross-cutting/exceptions.md` | 1 | Must document the new taxonomy module (wiki maintenance rule) | in scope | -| `app/api/docs/**/*.md` swagger copy | 2 | 3 files in the touched domains mention status codes and may go stale | in scope | -| Sentry (`app/core/sentry_filters.py`) | 2 | Reclassifying caller errors off 5xx reduces event volume; no filter code change needed | in scope, verify only | -| kaapi-frontend console | 2 | Breaking if it branches on status code; `APIResponse` body shape is unchanged | **out of scope** (user decision) | -| Langfuse | 2 | Trace and score shape unchanged | unaffected | -| Provider Batch APIs (OpenAI, Gemini, Anthropic) | 2 | Batch payload shape unchanged | unaffected | -| Object storage | 2 | Upload failures resurface as 502 instead of 500; no storage code change | in scope, read path only | -| auth, fine_tuning, assessment, response, api_key | 2 | Same defect class present but outside the eight in-scope domains | **out of scope**, follow-up issue (step 13) | - -## Steps - -### 1. Core: add the domain exception taxonomy -- Files: `backend/app/core/exceptions.py` (new) -- Define `KaapiError(Exception)` carrying `status_code: int` and `detail: str`, plus subclasses `NotFoundError` (404), `ConflictError` (409), `InvalidPayloadError` (422), `InvalidValueError` (400), `UpstreamError` (502), `ServiceUnavailableError` (503). -- `UpstreamError` takes a `provider: str` so the message can name who failed. -- Precedent to follow, not duplicate: the four existing ad-hoc exceptions (`CloudStorageError` in `core/cloud/storage.py:34`, `GeminiClientError` in `core/batch/client.py:15`, `CollectionNameConflictError` in `crud/collection/collection.py:18`, `TransformationError` in `services/doctransform/registry.py:9`) stay where they are; only `CollectionNameConflictError` is re-based in step 6. -- Depends on: nothing - -### 2. Core: register the handler and stop the 500 body leak -- Files: `backend/app/core/exception_handlers.py` (change) -- Add `@app.exception_handler(KaapiError)` inside `register_exception_handlers`, returning `JSONResponse(status_code=exc.status_code, content=APIResponse.failure_response(exc.detail).model_dump())`, matching the existing `http_exception_handler` shape at lines 98-108. -- Change `generic_error_handler` (lines 110-117) to log the exception with `exc_info=True` and the correlation id, and return a fixed message instead of `str(exc)`. This is what currently exposes raw `ValueError` text (including Langfuse response bodies) to API clients. -- Handler registration order matters: register the `KaapiError` handler before the bare `Exception` handler. -- Depends on: step 1 - -### 3. Evaluations crud: dataset fetch raises typed errors -- Files: `backend/app/crud/evaluations/batch.py` (change) -- `fetch_dataset_items` (lines 35-70): in the `except Exception` at line 51, branch on the Langfuse error. Not-found raises `NotFoundError(f"Dataset '{dataset_name}' not found")`; any other Langfuse failure raises `UpstreamError(provider="langfuse", ...)`. Log level follows the taxonomy table (`warning` for not-found, `error` with `exc_info=True` for upstream). -- Line 58 empty-dataset `ValueError` becomes `InvalidPayloadError`. -- The three other `raise ValueError` sites in this file get the same treatment; the file has 5 total. -- Depends on: step 1 - -### 4. Evaluations services: upstream and broker failures -- Files: `backend/app/services/evaluations/dataset.py`, `backend/app/services/evaluations/fast.py`, `backend/app/services/evaluations/prompt_improvement.py` (change) -- `dataset.py:131-138`: Langfuse upload failure becomes `UpstreamError` (502) instead of `HTTPException(500)`. -- `fast.py:211-215`: the broad `except` currently swallows step 3's typed errors into a flat 500. Re-raise `KaapiError` untouched before the catch-all, so a missing dataset reaches the caller as 404. The run is still marked `failed` in both branches. -- `prompt_improvement.py:207-210`: Celery enqueue failure becomes `ServiceUnavailableError` (503). -- Leave `crud/evaluations/dataset.py:102-105` at 500. A failed DB write is genuinely our fault and already logs at `error` with `exc_info=True`. -- Depends on: steps 1, 3 - -### 5. STT/TTS evaluations: queue, storage, and empty-dataset codes -- Files: `backend/app/api/routes/stt_evaluations/evaluation.py`, `backend/app/api/routes/tts_evaluations/evaluation.py`, `backend/app/services/stt_evaluations/audio.py` (change) -- `stt_evaluations/evaluation.py:108-111` and `tts_evaluations/evaluation.py:112-115`: "Failed to queue batch submission" becomes 503. -- `stt_evaluations/evaluation.py:65` and `tts_evaluations/evaluation.py:69`: "Dataset has no samples" moves 400 to 422, matching the empty-dataset code chosen in step 3. -- `audio.py:126-129`: object storage upload failure becomes `UpstreamError` (502). The `except HTTPException: raise` guard at line 119 becomes `except (HTTPException, KaapiError): raise`. -- Depends on: steps 1, 3 - -### 6. Collections: conflict semantics -- Files: `backend/app/crud/collection/collection.py`, `backend/app/api/routes/collections.py` (change) -- `collection.py:114`: "Collection already deleted" moves 400 to 409. It is a state conflict, not bad input. -- `collection.py:18`: `CollectionNameConflictError` subclasses `ConflictError` and carries its own detail, so the manual translation at `collections.py:220-224` can be deleted and the handler from step 2 covers it. Keep the `name` attribute; it is read to build the message. -- Depends on: steps 1, 2 - -### 7. Documents: format errors and the wrong-module import -- Files: `backend/app/services/documents/helpers.py`, `backend/app/crud/document/document.py`, `backend/app/crud/document/doc_transformation_job.py` (change) -- `helpers.py:62-64`: unsupported or unreadable source file format moves 400 to 422, matching `services/evaluations/validators.py:84-97`, which already returns 422 for the same class of problem. See Open Questions. -- `helpers.py:67-72` and `helpers.py:76-83` stay at 400. An unsupported transform pair and an unknown transformer are valid-shape, unacceptable-value cases. -- `helpers.py:59`: the docstring says `Raises: HTTPException(400)` and goes stale; update it. -- `document.py:8` and `doc_transformation_job.py:16` import `HTTPException` from `app.core.exception_handlers` rather than `fastapi`. Correct both imports. Behaviour is identical today, but it couples crud to the handler module. -- Depends on: step 1 - -### 8. LLM Call, LLM Chain, Guardrails: broker failures -- Files: `backend/app/services/llm/jobs.py`, `backend/app/services/guardrails/jobs.py` (change) -- `llm/jobs.py:181-183` (`start_job`), `llm/jobs.py:241-244` (`start_chain_job`), `guardrails/jobs.py:87-90` (`start_job`): all three catch a failed Celery enqueue and return 500. All three become `ServiceUnavailableError` (503). The job is still marked `FAILED` first, unchanged. -- Guardrails is included because `docs/wiki/modules/llm-call.md` places `api/routes/guardrails.py` in the LLM Call module. -- Depends on: step 1 - -### 9. Config Management: catch-all scope and validation code -- Files: `backend/app/crud/config/config.py`, `backend/app/crud/config/version.py` (change) -- `config.py:70-80` (`ConfigCrud.create`) and `version.py:110-120` (`ConfigVersionCrud.create_from_partial`) both wrap their DB writes in a bare `except Exception` that returns 500. A duplicate config name raises `IntegrityError` inside that block and is flattened to 500, even though `config.py:191` already returns 409 for the same conflict on another path. Add an `except IntegrityError` branch returning `ConflictError` ahead of the catch-all, and a `except (HTTPException, KaapiError): raise` guard so typed errors raised inside are not swallowed. -- `version.py:78-85`: a `ConfigBlob` Pydantic `ValidationError` returns `HTTPException(400, detail=validation_errors)`. The same `ConfigBlob` failing at the request boundary returns 422 through `validation_error_handler`. Move this to 422 so one config blob has one rejection code regardless of entry path. The list `detail` still routes through `_sanitize_validation_errors` at `exception_handlers.py:103-104`, so the body shape is unchanged. -- Leave the immutability checks at `version.py:201`, `version.py:330`, and `version.py:340` at 400 for now. See Open Questions. -- Depends on: steps 1, 2 - -### 10. Credentials: conflict, not-found, and KMS failures -- Files: `backend/app/crud/credentials.py`, `backend/app/api/routes/credentials.py`, `backend/app/core/security.py` (change) -- `credentials.py:66-69`: the unique-constraint branch (`uq_credential_org_project_provider`) returns 400. `.claude/conventions/route.md` names 409 for exactly this case. Move to `ConflictError`. -- `credentials.py:70-72`: the non-duplicate `IntegrityError` path raises a bare `ValueError` carrying the raw DB error, which reaches the client as a 500 body. Raise a typed error with a sanitized message; the DB text stays in the log only. -- `credentials.py:190`: `raise ValueError("Provider and credential must be provided")` surfaces as 500, while `routes/credentials.py:122` and `routes/credentials.py:263` raise 400 for the identical condition. Make the crud site 400 to match. -- `credentials.py:285-289` (`remove_provider_credential`): `rowcount == 0` means nothing matched, so it is a 404, not the current 500. Change the code and drop the log from `error` to `warning`, since a caller deleting a non-existent credential is not an outage. -- `credentials.py:322-329` (`remove_creds_for_org`): split the branch. Zero rows deleted returns 404; a genuine partial delete (`0 < rows_deleted < expected_count`) keeps 500 and stays at `logger.error`, because it is a real anomaly. -- `security.py:213-216` and `security.py:233-236`: `encrypt_credentials` and `decrypt_credentials` raise `ValueError` when AWS KMS fails. KMS is upstream, so raise `UpstreamError(provider="kms")` (502). Keep the existing behaviour of never surfacing the underlying message, which may carry AWS ARNs, and add the missing `exc_info=True` to both `logger.error` calls. -- Leave `routes/credentials.py:49` at 500. An empty result from `set_creds_for_org` after no exception is a genuine internal anomaly. -- Leave the `validate_provider_credentials` 400s at `credentials.py:36`, `credentials.py:160`, and `credentials.py:208` unchanged. See Open Questions. -- Depends on: steps 1, 2 - -### 11. Onboarding: KMS and the unguarded commit -- Files: `backend/app/crud/onboarding.py` (change) -- `onboarding.py:126`: `encrypt_credentials(values)` is called with no guard, so a KMS failure becomes a 500. Step 10 makes it raise `UpstreamError`, so this site only needs to stop being the reason a 500 escapes; verify no local `except` re-flattens it. -- `onboarding.py:139` (`session.commit()`): the whole multi-step onboarding has no `try`. The pre-checks at `onboarding.py:67-71` and `onboarding.py:100-105` are read-then-write races, so two concurrent onboards of the same org and project name hit an `IntegrityError` at commit and return 500 with raw DB text. Wrap the commit in `except IntegrityError` returning `ConflictError` with the same wording the pre-check uses, and roll back. -- The two existing 409s (`onboarding.py:68`, `onboarding.py:102`) are already correct and stay as they are. -- Depends on: steps 1, 2, 10 - -### 12. Observability: log-level and prefix audit in the in-scope domains -- Files (change): `backend/app/api/routes/stt_evaluations/evaluation.py:98`, `backend/app/api/routes/tts_evaluations/evaluation.py:102`, `backend/app/crud/evaluations/cron_utils.py:325`, `backend/app/crud/evaluations/embeddings.py:325`, `backend/app/crud/evaluations/langfuse.py:197,599`, `backend/app/crud/evaluations/processing.py:328,1027,1078,1201`, `backend/app/crud/stt_evaluations/batch.py:228`, `backend/app/crud/tts_evaluations/batch.py:137`, `backend/app/services/collections/providers/gemini.py:148,173` -- Add `exc_info=True` to these 14 `logger.error` calls that caught a real exception without it, per `.claude/conventions/error-handling.md` ("`exc_info=True` on any path that caught a real exception"). -- Files (change): `backend/app/crud/evaluations/core.py:106,239` and `backend/app/crud/evaluations/embeddings.py:105,111,120,163,174,232,296,331,387` -- Add the missing `[function_name]` bracket prefix required by CLAUDE.md. 12 lines. -- Log levels already match the fault table at the status-code sites this plan touches; verify rather than assume when editing each one. The two credentials delete sites are the exception and are handled in step 10. -- Depends on: steps 3 through 11 (same lines, edit once) - -### 13. Docs: wiki, swagger, and the follow-up issue -- Files: `docs/wiki/cross-cutting/exceptions.md` (change), `backend/app/api/docs/evaluation/improve_prompt.md`, `backend/app/api/docs/evaluation/create_evaluation.md`, `backend/app/api/docs/collections/update.md`, `backend/app/api/docs/onboarding/onboarding.md` (change if their stated codes went stale) -- Add an "Domain exceptions" section to `exceptions.md` pointing at `core/exceptions.py`, listing the taxonomy table above as the source of truth, and noting that the `KaapiError` handler is registered ahead of the generic one. -- The touched module pages (`llm-call.md`, `evaluations.md`, `knowledge-base.md`, `tenancy.md`, `platform.md`) hold no status codes, so they need no edit. -- Open a follow-up issue listing the out-of-scope sites found during the survey: `app/api/routes/auth.py:65,306,357`, `app/api/routes/fine_tuning.py:269`, `app/crud/api_key.py:97`, `app/crud/assessment/dataset.py:72`, `app/services/assessment/dataset.py:293`, `app/services/assessment/utils/export.py:329`, `app/services/response/jobs.py:42`. -- Depends on: steps 1 through 12 - -## Tests - -Existing assertions that encode the current wrong codes and must flip: - -| Test | Now | After | -|---|---|---| -| `app/tests/api/routes/test_evaluation.py:477` | 500 | per step 3/4 | -| `app/tests/api/routes/test_evaluation_fast.py:1271` | 500 | 404 (missing dataset) | -| `app/tests/api/routes/test_tts_evaluation.py:629` | 500 | 503 | -| `app/tests/api/routes/test_improve_prompt.py:517` | 500 | 503 | -| `app/tests/services/llm/test_jobs.py:101,2012` | 500 | 503 | -| `app/tests/services/stt_evaluations/test_audio.py:188` | 500 | 502 | -| `app/tests/services/guardrails/test_jobs.py:159` | 500 | 503 | -| `app/tests/api/routes/test_creds.py:260` (plus its docstring at line 240, which says "fails with 400") | 400 | 409 | - -Out of scope and untouched: `app/tests/api/test_auth.py:49,313,412` and `app/tests/assessment/test_dataset.py:304`. - -Assertions deliberately left alone: `app/tests/crud/test_credentials.py:273` ("Unsupported provider") and `:332` ("Missing required fields for langfuse") stay at 400, matching the Open Questions decision below. - -New coverage: -- `app/tests/core/test_exceptions.py` (new): each `KaapiError` subclass maps to its status code through the registered handler, and `generic_error_handler` no longer returns `str(exc)` in the body. -- Evaluations: Langfuse dataset-not-found returns 404 and Langfuse unreachable returns 502, on both the batch path and the fast path. Mock the Langfuse client boundary (existing patch target `app.services.evaluations.fast.fetch_dataset_items` and `app.crud.evaluations.batch.fetch_dataset_items`, already used at `test_evaluation_fast.py:322` and `test_evaluation.py:1036`). -- Collections: deleting an already-deleted collection returns 409; duplicate name still returns 409 after the handler takes over from the manual translation. -- Documents: unsupported source format returns 422; unsupported transform pair still returns 400. -- Config: creating a config whose name already exists returns 409 rather than 500; a `ConfigBlob` that fails validation during partial-version merge returns 422 with the same body shape the request boundary produces. -- Credentials: duplicate provider credential returns 409; deleting a credential that does not exist returns 404; a KMS failure during create returns 502 and the response body does not contain the AWS error text. -- Onboarding: a KMS failure during onboarding returns 502; a concurrent onboard hitting `IntegrityError` at commit returns 409, and no partial org/project/user rows survive. - -HTTP boundaries mocked: Langfuse client, OpenAI and Gemini SDK clients, object storage (`core/cloud/storage.py`), the AWS KMS client (`get_kms_client` in `core/security.py`), and the Celery `.delay`/`apply_async` call for the 503 paths. - -`app/tests/services/credentials/` already exists and is the home for the KMS-failure cases. - -## Open Questions - -- **Documents 422 versus 400 for an unsupported file format (step 7).** Two live conventions disagree: `services/documents/helpers.py:64` returns 400, `services/evaluations/validators.py:84-97` returns 422 for the same class of problem. This plan picks 422 on the reading that the uploaded file is the unparseable payload, which also makes evaluations the larger surface that does not have to change. Reverse it if the reviewer reads an unsupported extension as a value problem rather than a shape problem, in which case `validators.py` moves to 400 instead and evaluations tests change. -- **503 versus 500 for a failed Celery enqueue (steps 4, 5, 8).** RabbitMQ being unable to accept a job is an availability problem the caller can retry, which 503 signals and 500 does not. Assumed, not stated in the issue. -- **Config immutability violations: 400 or 409 (step 9).** `version.py:201`, `version.py:330`, and `version.py:340` reject a change to an immutable field (`type`, `provider`) with 400. `.claude/conventions/error-handling.md` describes 409 as "request conflicts with current resource state", which is literally what this is. This plan leaves them at 400 to keep the config diff to the two defects that have no defence. Flip all three to 409 if the reviewer wants the taxonomy applied strictly. -- **Credentials validation: 400 or 422 (step 10).** `validate_provider_credentials` rejects both an unknown provider ("Unsupported provider", a value problem, 400) and a well-formed request missing provider-specific keys ("Missing required fields for langfuse", arguably a shape problem, 422) with the same 400. This plan keeps both at 400 rather than splitting one helper's output across two codes. Splitting them is defensible and would change `app/tests/crud/test_credentials.py:332`. -- **Onboarding bypasses credential validation.** `crud/onboarding.py:118-133` builds `Credential` rows inline instead of calling `set_creds_for_org`, so it never runs `validate_provider_credentials`. The same bad payload gets 400 through `POST /credentials` and is silently stored through `POST /onboard`. This is an error-message consistency defect, but fixing it changes onboarding behaviour rather than its status codes, so it is called out here rather than planned. Confirm whether it belongs in this PR or its own. -- **Frontend coordination is out of scope by decision.** Eleven status codes now change on endpoints the console calls, including `POST /credentials` and `POST /onboard`. The `APIResponse` body shape is untouched, so only code-branching logic can break. Worth a line in the PR description even though no investigation is planned. From 0b6517a46e0c60ad32113e1348cca82d3ed44261 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:30:53 +0530 Subject: [PATCH 08/12] refactor: Improve status code consistency and error messaging in exception handlers --- backend/app/core/config.py | 2 - backend/app/core/exception_handlers.py | 45 ++++--------------- backend/app/core/telemetry/llm.py | 11 +++-- backend/app/core/telemetry/sentry/init.py | 4 +- backend/app/core/telemetry/tracing.py | 7 --- .../core/telemetry/test_telemetry_llm.py | 2 +- .../core/telemetry/test_telemetry_tracing.py | 30 +------------ backend/app/tests/core/test_config.py | 29 ------------ .../app/tests/core/test_exception_handlers.py | 26 +++-------- backend/pyproject.toml | 1 - backend/uv.lock | 17 ------- .../kaapi-observability-ARCHITECTURE.md | 8 ++-- docs/wiki/cross-cutting/exceptions.md | 4 +- docs/wiki/cross-cutting/observability.md | 4 +- 14 files changed, 33 insertions(+), 157 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 7bf84205b..b6d983021 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -162,8 +162,6 @@ def AWS_S3_BUCKET(self) -> str: SENTRY_TRACES_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0) SENTRY_SEND_DEFAULT_PII: bool = False SENTRY_ERROR_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0) - SENTRY_PROFILE_SESSION_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0) - SENTRY_PROFILE_LIFECYCLE: Literal["manual", "trace"] = "trace" # Threshold Request Rate per minute THRESHOLD_LLM_CALL_RATE: int = 15 diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 384a546c7..9a04c4c17 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -7,7 +7,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from sqlalchemy.exc import DataError, IntegrityError, OperationalError, SQLAlchemyError +from sqlalchemy.exc import DataError, OperationalError, SQLAlchemyError from starlette.status import ( HTTP_409_CONFLICT, HTTP_422_UNPROCESSABLE_CONTENT, @@ -20,33 +20,11 @@ logger = logging.getLogger(__name__) GENERIC_ERROR_DETAIL = "An unexpected error occurred." -DB_ERROR_DETAIL = "Database error: request could not be completed." +DB_ERROR_DETAIL = "The request could not be completed." UNIQUE_VIOLATION_SQLSTATE = "23505" FOREIGN_KEY_VIOLATION_SQLSTATE = "23503" -# Other IntegrityError codes (e.g. not-null 23502) mean our write was wrong: 500. -_INTEGRITY_ERROR_HINTS: dict[str, tuple[int, str]] = { - UNIQUE_VIOLATION_SQLSTATE: ( - HTTP_409_CONFLICT, - "Database error: request conflicts with existing data.", - ), - FOREIGN_KEY_VIOLATION_SQLSTATE: ( - HTTP_422_UNPROCESSABLE_CONTENT, - "Database error: request references a record that does not exist.", - ), -} - -# Unique violations on these constraints get a specific message; values stay out of it. -_UNIQUE_CONSTRAINT_DETAILS: dict[str, str] = { - "uq_config_project_id_name_active": "A config with this name already exists in this project.", - "uq_config_version_config_id_version": "A version with this number already exists for this config.", - "uq_credential_org_project_provider": "Credentials for this provider already exist for this organization and project.", - "ix_organization_name": "An organization with this name already exists.", - "uq_project_name_org_id": "A project with this name already exists in this organization.", - "ix_user_email": "A user with this email already exists.", -} - _BRANCH_PATTERN = re.compile(r"^[A-Z]|[\[\]()]") @@ -129,25 +107,20 @@ def _db_error_constraint(exc: SQLAlchemyError) -> tuple[str | None, str | None]: def _db_error_response(exc: SQLAlchemyError) -> tuple[int, str]: """Map a DB exception to a status and a client-safe hint; SQL and parameters stay server-side.""" - if isinstance(exc, IntegrityError): - sqlstate, constraint = _db_error_constraint(exc) - if ( - sqlstate == UNIQUE_VIOLATION_SQLSTATE - and constraint in _UNIQUE_CONSTRAINT_DETAILS - ): - return HTTP_409_CONFLICT, _UNIQUE_CONSTRAINT_DETAILS[constraint] - return _INTEGRITY_ERROR_HINTS.get( - sqlstate, (HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL) - ) + sqlstate, _ = _db_error_constraint(exc) + if sqlstate == UNIQUE_VIOLATION_SQLSTATE: + return HTTP_409_CONFLICT, "Record already exists." + if sqlstate == FOREIGN_KEY_VIOLATION_SQLSTATE: + return HTTP_422_UNPROCESSABLE_CONTENT, "Referenced record not found." if isinstance(exc, DataError): return ( HTTP_422_UNPROCESSABLE_CONTENT, - "Database error: a value is invalid or too long.", + "A value in the request is invalid or too long.", ) if isinstance(exc, OperationalError): return ( HTTP_503_SERVICE_UNAVAILABLE, - "Database error: service temporarily unavailable, please retry.", + "Service temporarily unavailable. Please retry.", ) return HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL diff --git a/backend/app/core/telemetry/llm.py b/backend/app/core/telemetry/llm.py index 32423f847..0ea7ee37b 100644 --- a/backend/app/core/telemetry/llm.py +++ b/backend/app/core/telemetry/llm.py @@ -127,12 +127,15 @@ def set_gen_ai_response_attributes( """Set OTel GenAI response attributes (usage, model) on `span`.""" usage = response.usage if usage: + reasoning_tokens = usage.reasoning_tokens or 0 + output_tokens = usage.output_tokens + if reasoning_tokens and usage.input_tokens + output_tokens < usage.total_tokens: + output_tokens += reasoning_tokens span.set_attribute("gen_ai.usage.input_tokens", usage.input_tokens) - span.set_attribute("gen_ai.usage.output_tokens", usage.output_tokens) + span.set_attribute("gen_ai.usage.output_tokens", output_tokens) span.set_attribute("gen_ai.usage.total_tokens", usage.total_tokens) - reasoning_tokens = getattr(usage, "reasoning_tokens", None) - if reasoning_tokens is not None: - span.set_attribute("gen_ai.usage.output_tokens.reasoning", reasoning_tokens) + if reasoning_tokens: + span.set_attribute("gen_ai.usage.reasoning.output_tokens", reasoning_tokens) if response.response and response.response.model: span.set_attribute("gen_ai.response.model", response.response.model) diff --git a/backend/app/core/telemetry/sentry/init.py b/backend/app/core/telemetry/sentry/init.py index c33a631ba..138e08cde 100644 --- a/backend/app/core/telemetry/sentry/init.py +++ b/backend/app/core/telemetry/sentry/init.py @@ -1,4 +1,4 @@ -"""sentry_sdk.init() with shared sampling, profiling, PII and before_send filters.""" +"""sentry_sdk.init() with shared sampling, PII and before_send filters.""" import logging @@ -32,8 +32,6 @@ def init_sentry( instrumenter="otel", traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE, sample_rate=settings.SENTRY_ERROR_SAMPLE_RATE, - profile_session_sample_rate=settings.SENTRY_PROFILE_SESSION_SAMPLE_RATE, - profile_lifecycle=settings.SENTRY_PROFILE_LIFECYCLE, send_default_pii=settings.SENTRY_SEND_DEFAULT_PII, enable_logs=True, include_local_variables=False, diff --git a/backend/app/core/telemetry/tracing.py b/backend/app/core/telemetry/tracing.py index 7bd9a2982..ae54488ef 100644 --- a/backend/app/core/telemetry/tracing.py +++ b/backend/app/core/telemetry/tracing.py @@ -150,13 +150,6 @@ def setup_telemetry(service_name: str | None = None) -> None: except Exception: logger.exception("[setup_telemetry] Failed to instrument Celery") - try: - from opentelemetry.instrumentation.redis import RedisInstrumentor - - RedisInstrumentor().instrument() - except Exception: - logger.exception("[setup_telemetry] Failed to instrument Redis") - try: from opentelemetry.instrumentation.botocore import BotocoreInstrumentor diff --git a/backend/app/tests/core/telemetry/test_telemetry_llm.py b/backend/app/tests/core/telemetry/test_telemetry_llm.py index 16aa496fa..20ee05d87 100644 --- a/backend/app/tests/core/telemetry/test_telemetry_llm.py +++ b/backend/app/tests/core/telemetry/test_telemetry_llm.py @@ -101,7 +101,7 @@ def test_response_attributes_with_usage_and_model(self) -> None: telemetry.set_gen_ai_response_attributes(span, response=response) set_calls = {c.args[0]: c.args[1] for c in span.set_attribute.call_args_list} assert set_calls["gen_ai.usage.total_tokens"] == 3 - assert set_calls["gen_ai.usage.output_tokens.reasoning"] == 4 + assert set_calls["gen_ai.usage.reasoning.output_tokens"] == 4 assert set_calls["gen_ai.response.model"] == "gpt-4o-2024" def test_response_attributes_without_usage(self) -> None: diff --git a/backend/app/tests/core/telemetry/test_telemetry_tracing.py b/backend/app/tests/core/telemetry/test_telemetry_tracing.py index 3fcf0e7d5..811f0e7fd 100644 --- a/backend/app/tests/core/telemetry/test_telemetry_tracing.py +++ b/backend/app/tests/core/telemetry/test_telemetry_tracing.py @@ -12,33 +12,7 @@ class TestSetupTelemetryInstrumentors: - def test_enabled_calls_instrument_on_both(self) -> None: - redis, botocore = MagicMock(), MagicMock() - with ( - patch.object(telemetry.settings, "SENTRY_DSN", None), - patch.object(telemetry, "TracerProvider", MagicMock()), - patch.object(telemetry.trace, "set_tracer_provider", MagicMock()), - patch.object(telemetry, "LoggingInstrumentor", MagicMock()), - patch.object(telemetry, "HTTPXClientInstrumentor", MagicMock()), - patch.object(telemetry, "RequestsInstrumentor", MagicMock()), - patch( - "opentelemetry.instrumentation.celery.CeleryInstrumentor", MagicMock() - ), - patch("opentelemetry.instrumentation.redis.RedisInstrumentor", redis), - patch( - "opentelemetry.instrumentation.botocore.BotocoreInstrumentor", botocore - ), - ): - telemetry.setup_telemetry() - - redis.return_value.instrument.assert_called_once() - botocore.return_value.instrument.assert_called_once() - - def test_redis_failure_does_not_propagate_and_botocore_still_instruments( - self, - ) -> None: - redis = MagicMock() - redis.return_value.instrument.side_effect = RuntimeError("boom") + def test_enabled_instruments_botocore(self) -> None: botocore = MagicMock() with ( patch.object(telemetry.settings, "SENTRY_DSN", None), @@ -50,7 +24,6 @@ def test_redis_failure_does_not_propagate_and_botocore_still_instruments( patch( "opentelemetry.instrumentation.celery.CeleryInstrumentor", MagicMock() ), - patch("opentelemetry.instrumentation.redis.RedisInstrumentor", redis), patch( "opentelemetry.instrumentation.botocore.BotocoreInstrumentor", botocore ), @@ -265,7 +238,6 @@ def test_registers_processor_and_propagator(self) -> None: patch( "opentelemetry.instrumentation.celery.CeleryInstrumentor", MagicMock() ), - patch("opentelemetry.instrumentation.redis.RedisInstrumentor", MagicMock()), patch( "opentelemetry.instrumentation.botocore.BotocoreInstrumentor", MagicMock(), diff --git a/backend/app/tests/core/test_config.py b/backend/app/tests/core/test_config.py index 4be4e932d..bd279d6f5 100644 --- a/backend/app/tests/core/test_config.py +++ b/backend/app/tests/core/test_config.py @@ -1,5 +1,4 @@ import pytest -from pydantic import TypeAdapter, ValidationError from app.core.config import Settings @@ -11,37 +10,9 @@ class TestSentryConfigDefaults: ("SENTRY_TRACES_SAMPLE_RATE", 1.0), ("SENTRY_SEND_DEFAULT_PII", False), ("SENTRY_ERROR_SAMPLE_RATE", 1.0), - ("SENTRY_PROFILE_SESSION_SAMPLE_RATE", 1.0), - ("SENTRY_PROFILE_LIFECYCLE", "trace"), ], ) def test_declared_default( self, field: str, expected: float | str | bool | None ) -> None: assert Settings.model_fields[field].default == expected - - -class TestSentryProfilingValidation: - @staticmethod - def _adapter(field: str) -> TypeAdapter[object]: - return TypeAdapter(Settings.model_fields[field].rebuild_annotation()) - - @pytest.mark.parametrize("value", [0.0, 0.25, 1.0]) - def test_session_sample_rate_accepts_unit_interval(self, value: float) -> None: - assert ( - self._adapter("SENTRY_PROFILE_SESSION_SAMPLE_RATE").validate_python(value) - == value - ) - - @pytest.mark.parametrize("value", [-0.1, 1.5]) - def test_session_sample_rate_rejects_out_of_range(self, value: float) -> None: - with pytest.raises(ValidationError): - self._adapter("SENTRY_PROFILE_SESSION_SAMPLE_RATE").validate_python(value) - - @pytest.mark.parametrize("value", ["manual", "trace"]) - def test_lifecycle_accepts_supported_modes(self, value: str) -> None: - assert self._adapter("SENTRY_PROFILE_LIFECYCLE").validate_python(value) == value - - def test_lifecycle_rejects_unknown_mode(self) -> None: - with pytest.raises(ValidationError): - self._adapter("SENTRY_PROFILE_LIFECYCLE").validate_python("always") diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index ee320901c..8a6eb9dd9 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -255,7 +255,6 @@ def test_maps_status_without_leaking_sql( response = _app_raising(exc).get("/boom") assert response.status_code == status_code - assert response.json()["error"].startswith("Database error:") assert "INSERT" not in response.text assert "9876543210" not in response.text @@ -266,27 +265,14 @@ def test_unmapped_db_error_uses_default_detail(self) -> None: assert response.json()["error"] == DB_ERROR_DETAIL - @pytest.mark.parametrize( - ("constraint", "detail"), - [ - ("uq_config_project_id_name_active", "A config with this name"), - ("uq_credential_org_project_provider", "Credentials for this provider"), - ("ix_user_email", "A user with this email"), - ], - ) - def test_known_unique_constraint_gets_specific_message( - self, constraint: str, detail: str - ) -> None: - response = _app_raising(make_integrity_error("23505", constraint)).get("/boom") - - assert response.status_code == 409 - assert response.json()["error"].startswith(detail) - - def test_unknown_unique_constraint_gets_generic_conflict(self) -> None: - response = _app_raising(make_integrity_error("23505", "uq_other")).get("/boom") + def test_constraint_name_stays_out_of_client_message(self) -> None: + response = _app_raising( + make_integrity_error("23505", "uq_credential_org_project_provider") + ).get("/boom") assert response.status_code == 409 - assert response.json()["error"].startswith("Database error:") + assert response.json()["error"] == "Record already exists." + assert "uq_credential" not in response.text def test_client_side_db_error_logs_warning(self) -> None: with ( diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0ca77e4fd..0f6fe54c7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "opentelemetry-instrumentation-requests>=0.51b0", "opentelemetry-instrumentation-logging>=0.51b0", "opentelemetry-instrumentation-sqlalchemy>=0.51b0", - "opentelemetry-instrumentation-redis>=0.51b0", "opentelemetry-instrumentation-botocore>=0.51b0", "pyjwt>=2.13.0,<3.0.0", "boto3>=1.37.20", diff --git a/backend/uv.lock b/backend/uv.lock index 1f5c8ed49..328573904 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -287,7 +287,6 @@ dependencies = [ { name = "opentelemetry-instrumentation-fastapi" }, { name = "opentelemetry-instrumentation-httpx" }, { name = "opentelemetry-instrumentation-logging" }, - { name = "opentelemetry-instrumentation-redis" }, { name = "opentelemetry-instrumentation-requests" }, { name = "opentelemetry-instrumentation-sqlalchemy" }, { name = "opentelemetry-sdk" }, @@ -362,7 +361,6 @@ requires-dist = [ { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.51b0" }, { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.51b0" }, { name = "opentelemetry-instrumentation-logging", specifier = ">=0.51b0" }, - { name = "opentelemetry-instrumentation-redis", specifier = ">=0.51b0" }, { name = "opentelemetry-instrumentation-requests", specifier = ">=0.51b0" }, { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.51b0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, @@ -2768,21 +2766,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/8e/7577914681d77b180f8d6dcbac435be8e4ca6add6315da2d01ac4289eaa3/opentelemetry_instrumentation_logging-0.65b0-py3-none-any.whl", hash = "sha256:68365b31755c844f1e85f07dcd217839ff92f2d278a214bdf02d4dc806f9d915", size = 15727, upload-time = "2026-07-16T15:25:18.774Z" }, ] -[[package]] -name = "opentelemetry-instrumentation-redis" -version = "0.65b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/e5/b369897a9ff902eb028f1f999c9f9a4602f0b30fe1e97298d2c15fe5b8b0/opentelemetry_instrumentation_redis-0.65b0.tar.gz", hash = "sha256:f16409f189092984ff922f26939e7be79509365f5c9d202308c13fddf1368147", size = 17218, upload-time = "2026-07-16T15:26:17.243Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/2a/5b874bf8824e7cb995b22e1e61e1b5bc42810023d81612c44edbfd2c48a3/opentelemetry_instrumentation_redis-0.65b0-py3-none-any.whl", hash = "sha256:7d135f61db9d72416e1f382012fbc13de6f5e726491bebd0344a9c42a60e6769", size = 14658, upload-time = "2026-07-16T15:25:29.146Z" }, -] - [[package]] name = "opentelemetry-instrumentation-requests" version = "0.65b0" diff --git a/docs/architecture/kaapi-observability-ARCHITECTURE.md b/docs/architecture/kaapi-observability-ARCHITECTURE.md index ed6eab4f4..2849bd7e3 100644 --- a/docs/architecture/kaapi-observability-ARCHITECTURE.md +++ b/docs/architecture/kaapi-observability-ARCHITECTURE.md @@ -2,7 +2,7 @@ ## Overview -Kaapi collects **traces, metrics and logs** from both processes, the FastAPI web app and the Celery worker, and sends all of them to **one Sentry project**. Errors, cron check-ins and profiles go to the same place, so one request can be followed from the HTTP call to the Celery task, the provider call and the callback. +Kaapi collects **traces, metrics and logs** from both processes, the FastAPI web app and the Celery worker, and sends all of them to **one Sentry project**. Errors and cron check-ins go to the same place, so one request can be followed from the HTTP call to the Celery task, the provider call and the callback. - **OpenTelemetry creates spans, Sentry stores them** (`instrumenter="otel"`). Sentry's own span integrations are off, so nothing is recorded twice. - **One package, `core/telemetry/`**, holds all of it. API and worker share one `init_sentry()`. @@ -30,7 +30,7 @@ flowchart LR FLT["sentry/filters.py\nscrub · drop"] end - Sentry[("Sentry project\nIssues · Traces · Logs · Metrics\nProfiles · Crons · Uptime")] + Sentry[("Sentry project\nIssues · Traces · Logs · Metrics\nCrons · Uptime")] LF[("Langfuse\nmessage bodies only")] CW[("CloudWatch\nstdout text")] @@ -69,14 +69,14 @@ core/telemetry/ - **One switch:** `OTEL_ENABLED` is checked only where instrumentation starts (`main.py`, `celery_app.py`, `core/db.py`). The telemetry functions themselves have no flags. - **No double recording:** Sentry's FastAPI, Starlette, SQLAlchemy, httpx (and, on the API, Celery) integrations are disabled because OTel already covers them. The worker keeps Sentry's Celery integration for task-failure Issues. -- **Sampling and profiling** come from settings, each validated to 0–1: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_ERROR_SAMPLE_RATE`, `SENTRY_PROFILE_SESSION_SAMPLE_RATE` (continuous profiling, tied to traces). +- **Sampling** comes from settings, each validated to 0–1: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_ERROR_SAMPLE_RATE`. Profiling is off: Sentry rate-limits profile chunks once the quota runs out, and the SDK then backs off and drops transactions too. - **Environments:** production and staging report to separate Sentry projects (`kaapi-production`, `kaapi-staging`). --- ## 1. Tracing -**What is traced:** HTTP requests, DB queries, Celery tasks, Redis, S3/KMS (botocore), outbound HTTP, and every LLM call. These feed Sentry's ready-made views: **AI Insights** (LLM calls), **Queries** (DB), **Queues** (Celery), **Caches** (Redis) and **Outbound API Requests**. +**What is traced:** HTTP requests, DB queries, Celery tasks, S3/KMS (botocore), outbound HTTP, and every LLM call. These feed Sentry's ready-made views: **AI Insights** (LLM calls), **Queries** (DB), **Queues** (Celery) and **Outbound API Requests**. Redis is not instrumented: its spans were only Celery result-backend bookkeeping and rate-limit counters. The broker (RabbitMQ) shows only as Celery's `apply_async` span, which links the API request to the worker trace. **Waterfall of one `/llm/call`**, as it appears in Sentry: diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index 4fe858030..fccf264aa 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -3,8 +3,8 @@ All paths relative to `backend/app/`. ## Global handlers -- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. Unhandled `SQLAlchemyError` gets its own handler: a "Database error: ..." hint with 409 (unique violation, SQLSTATE 23505), 422 (foreign key 23503, or `DataError`), 503 (`OperationalError`) or 500 (any other); 4xx logs at warning, 5xx at error; SQL and parameters only go to logs. -- DB errors are classified only in `db_error_handler`: crud/services let `SQLAlchemyError` propagate (rollback, then `raise`), never catch `IntegrityError` to build a 409 or match error text. A unique violation on a constraint listed in `_UNIQUE_CONSTRAINT_DETAILS` gets that message; add the constraint there for a specific 409. Sentry events carry `http.status_code`, `error.class` and `db.constraint` tags. Exception: eval `run_name` keeps its own 409 (`run_name_already_exists` is a client contract). +- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. Unhandled `SQLAlchemyError` gets its own handler: a short generic message with 409 (unique violation, SQLSTATE 23505), 422 (foreign key 23503, or `DataError`), 503 (`OperationalError`) or 500 (any other); 4xx logs at warning, 5xx at error; SQL and parameters only go to logs. +- DB errors are classified only in `db_error_handler`: crud/services let `SQLAlchemyError` propagate (rollback, then `raise`), never catch `IntegrityError` to build a 409 or match error text. Clients get a short generic message per status; the SQLSTATE and constraint name go only to the log and the `db.constraint` Sentry tag. Sentry events carry `http.status_code`, `error.class` and `db.constraint` tags. Exception: eval `run_name` keeps its own 409 (`run_name_already_exists` is a client contract). - `core/middleware.py` — request-level middleware. ## Status-code selection diff --git a/docs/wiki/cross-cutting/observability.md b/docs/wiki/cross-cutting/observability.md index bc27ace54..c54570557 100644 --- a/docs/wiki/cross-cutting/observability.md +++ b/docs/wiki/cross-cutting/observability.md @@ -14,7 +14,7 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai - `core/telemetry/sentry/filters.py` — `before_send_transaction_filter` (drops probe transactions, scrubs genai content; DB spans are kept so Sentry Queries has data) and `before_send_error_filter` (drops probe events, always drops the request body, scrubs headers/cookies/query when `SENTRY_SEND_DEFAULT_PII` off, redacts LLM task kwargs). - GenAI content never reaches Sentry, in either PII mode. Four locks in `core/telemetry/sentry/filters.py`, wired by `init_sentry()` in `core/telemetry/sentry/init.py` for both API and worker: `scrub_genai_content` strips `_GENAI_CONTENT_KEYS` (prompt/completion span+log attributes) from every event, `before_send_log_filter` does the same for logs, `genai_privacy_integrations()` pins Sentry's auto-enabled AI integrations to `include_prompts=False`, and `include_local_variables=False` / `max_request_body_size="never"` keep prompts out of stack frames and request bodies. Langfuse (isolated tracer provider, `core/langfuse/langfuse.py`) stays the only sink for message bodies. - Corollary for log lines on LLM paths: log lengths/ids/types, never prompt or response text — `enable_logs=True` ships every INFO record to Sentry. -- Sampling/profiling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_PROFILE_SESSION_SAMPLE_RATE`, `SENTRY_PROFILE_LIFECYCLE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior). +- Sampling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior). - Trace propagation: OTel's `CeleryInstrumentor` parents the worker's `run/...` span to `apply_async`; never inject/extract trace context by hand around tasks (it reparents task spans to the API request and Sentry splits them into separate transactions). `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace. - Error capture: handlers in `core/exception_handlers.py` report via `logger.error(exc_info=exc)` (5xx only). Don't add `capture_exception` next to it: on sentry-sdk 2.54 Dedupe does not merge the two, so each error becomes two events. - Tenant impact (both in `core/telemetry/context.py`, called from `api/deps.py`): `set_request_log_context` puts org/project in the log context + Sentry tags; `bind_sentry_user` binds user/org/project via `sentry_sdk.set_user` so issues report users/orgs affected. The user binding is scope-only — ids in the log context would stamp per-user cardinality on every INFO record `enable_logs` ships. Per-request identity is the `correlation_id` tag (`core/middleware.py`), not the user binding. @@ -28,5 +28,5 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai - Sentry Logs: `module_name: [knowledgebase] app.api.routes.collection_job.get_collection_job`, `logger.name: app.api.routes.collection_job`, `code.line.number: 58`; filter with `module_name:[knowledgebase]*`. - CloudWatch (stdout format puts `module_name` where the logger name was): `2026-10-11 10:16:57,137 - [kaapi-backend] - [a1b2c3] - INFO - [knowledgebase] app.api.routes.collection_job.get_collection_job - [get_collection_job] Fetched job | job_id=8f2c`; Logs Insights `filter @message like "[knowledgebase]"`. - `OTEL_ENABLED` is checked only at the entry points (`main.py`, `celery/celery_app.py`, `core/db.py`), which decide whether to call `setup_telemetry` / `instrument_app` / `instrument_db_engine`; the telemetry functions themselves are unconditional. `LogContextFilter` is installed by `configure_logging`, so log context works with OTel off. -- OTel auto-instrumentation (in `setup_telemetry`): FastAPI, SQLAlchemy, httpx, requests, logging, Celery (Queues insight), Redis (Caches insight), botocore (S3/KMS spans). +- OTel auto-instrumentation (in `setup_telemetry`): FastAPI, SQLAlchemy, httpx, requests, logging, Celery (Queues insight), botocore (S3/KMS spans). Redis is deliberately not instrumented (only Celery result bookkeeping and rate-limit counters); keep Celery's `apply_async` span, it links API and worker traces. - `core/rate_monitor.py` — provider rate tracking From df82ec45158efaf5b3a780c5a2bd714444acbcb7 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:41:49 +0530 Subject: [PATCH 09/12] fix: standardize HTTP status codes across evaluation endpoints and improve error handling --- .../app/api/docs/evaluation/improve_prompt.md | 2 +- .../api/routes/stt_evaluations/evaluation.py | 9 ++- .../api/routes/tts_evaluations/evaluation.py | 9 ++- backend/app/crud/evaluations/batch.py | 56 ++++--------------- backend/app/crud/evaluations/core.py | 20 ++----- backend/app/crud/evaluations/cron_utils.py | 3 +- backend/app/crud/evaluations/embeddings.py | 55 ++++++------------ backend/app/crud/evaluations/langfuse.py | 6 +- backend/app/crud/evaluations/processing.py | 6 +- backend/app/crud/stt_evaluations/batch.py | 3 +- backend/app/crud/tts_evaluations/batch.py | 3 +- backend/app/services/evaluations/dataset.py | 7 +-- backend/app/services/evaluations/fast.py | 34 +++-------- .../evaluations/prompt_improvement.py | 4 +- backend/app/services/stt_evaluations/audio.py | 9 ++- .../app/tests/api/routes/test_evaluation.py | 13 ++--- .../tests/api/routes/test_improve_prompt.py | 4 +- .../tests/api/routes/test_tts_evaluation.py | 4 +- .../crud/evaluations/test_batch_errors.py | 53 ------------------ .../app/tests/crud/evaluations/test_core.py | 43 +------------- .../services/stt_evaluations/test_audio.py | 7 +-- 21 files changed, 80 insertions(+), 270 deletions(-) delete mode 100644 backend/app/tests/crud/evaluations/test_batch_errors.py diff --git a/backend/app/api/docs/evaluation/improve_prompt.md b/backend/app/api/docs/evaluation/improve_prompt.md index c81a77750..811e58b86 100644 --- a/backend/app/api/docs/evaluation/improve_prompt.md +++ b/backend/app/api/docs/evaluation/improve_prompt.md @@ -56,4 +56,4 @@ LLM and trace-download failures surface only via the failure callback, not on th - `409 source_config_unavailable` — the run's config or config_version has been deleted. - `422 traces_not_available` — the run has no `score_trace_url`; trace file is required. - `422 invalid_callback_url` — `callback_url` is missing, not `https://`, or resolves to a blocked (private/loopback/metadata) address. -- `503 prompt_improvement_enqueue_failed` — the job could not be queued; retry shortly. +- `500 prompt_improvement_enqueue_failed` — the job could not be queued. diff --git a/backend/app/api/routes/stt_evaluations/evaluation.py b/backend/app/api/routes/stt_evaluations/evaluation.py index c5dc34283..6b96f4414 100644 --- a/backend/app/api/routes/stt_evaluations/evaluation.py +++ b/backend/app/api/routes/stt_evaluations/evaluation.py @@ -62,7 +62,7 @@ def start_stt_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=422, detail="Dataset has no samples") + raise HTTPException(status_code=400, detail="Dataset has no samples") # Use language_id from the dataset language_id = dataset.language_id @@ -97,8 +97,7 @@ def start_stt_evaluation( except Exception as e: logger.error( f"[start_stt_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}", - exc_info=True, + f"run_id: {run.id}, error: {str(e)}" ) update_stt_run( session=session, @@ -107,8 +106,8 @@ def start_stt_evaluation( error_message=f"Failed to queue batch submission: {str(e)}", ) raise HTTPException( - status_code=503, - detail="Could not queue the batch submission. Retry shortly.", + status_code=500, + detail=f"Failed to queue batch submission: {e}", ) return APIResponse.success_response( diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index e27d39666..6751d8219 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -66,7 +66,7 @@ def start_tts_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=422, detail="Dataset has no samples") + raise HTTPException(status_code=400, detail="Dataset has no samples") language_id = dataset.language_id @@ -101,8 +101,7 @@ def start_tts_evaluation( except Exception as e: logger.error( f"[start_tts_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}", - exc_info=True, + f"run_id: {run.id}, error: {str(e)}" ) update_tts_run( session=session, @@ -111,8 +110,8 @@ def start_tts_evaluation( error_message=f"Failed to queue batch submission: {str(e)}", ) raise HTTPException( - status_code=503, - detail="Could not queue the batch submission. Retry shortly.", + status_code=500, + detail=f"Failed to queue batch submission: {e}", ) return APIResponse.success_response( diff --git a/backend/app/crud/evaluations/batch.py b/backend/app/crud/evaluations/batch.py index a1c659139..7b8892644 100644 --- a/backend/app/crud/evaluations/batch.py +++ b/backend/app/crud/evaluations/batch.py @@ -8,12 +8,9 @@ """ import logging -from fastapi import HTTPException from typing import Any from langfuse import Langfuse -from langfuse.api import NotFoundError as LangfuseNotFoundError -from langfuse.api.core import ApiError as LangfuseApiError from sqlmodel import Session from app.core.batch import ( @@ -47,46 +44,18 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str, List of dataset items with input and expected_output Raises: - HTTPException: 404 if the dataset is missing, 502 if Langfuse errored - or was unreachable, 422 if the dataset is empty. + ValueError: If dataset not found or empty """ try: dataset = langfuse.get_dataset(dataset_name) - except LangfuseNotFoundError as e: + except Exception as e: logger.warning( - f"[fetch_dataset_items] Dataset not found | dataset={dataset_name} | {e}" - ) - raise HTTPException( - status_code=404, detail=f"Dataset '{dataset_name}' not found in Langfuse." - ) - except LangfuseApiError as e: - logger.error( - f"[fetch_dataset_items] Langfuse rejected the request | " - f"dataset={dataset_name}, code={getattr(e, 'status_code', 'unknown')}", - exc_info=True, - ) - raise HTTPException( - status_code=502, - detail=f"Langfuse could not return dataset '{dataset_name}' " - f"(code: {getattr(e, 'status_code', 'unknown')}). Retry shortly.", - ) - except Exception: - logger.error( - f"[fetch_dataset_items] Failed to reach Langfuse | dataset={dataset_name}", - exc_info=True, - ) - raise HTTPException( - status_code=502, - detail=f"Could not reach Langfuse to fetch dataset '{dataset_name}'. Retry shortly.", + f"[fetch_dataset_items] Failed to fetch dataset | dataset={dataset_name} | {e}" ) + raise ValueError(f"Dataset '{dataset_name}' not found: {e}") if not dataset.items: - logger.warning( - f"[fetch_dataset_items] Dataset is empty | dataset={dataset_name}" - ) - raise HTTPException( - status_code=422, detail=f"Dataset '{dataset_name}' is empty." - ) + raise ValueError(f"Dataset '{dataset_name}' is empty") items = [] for item in dataset.items: @@ -231,9 +200,8 @@ def start_evaluation_batch( dataset_items=dataset_items, openai_params=mapped_params ) if not jsonl_data: - raise HTTPException( - status_code=422, - detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", + raise ValueError( + "Evaluation dataset did not produce any JSONL entries (missing questions?)." ) openai_client = get_openai_client( @@ -272,9 +240,8 @@ def start_evaluation_batch( dataset_items=dataset_items, google_params=mapped_params ) if not jsonl_data: - raise HTTPException( - status_code=422, - detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", + raise ValueError( + "Evaluation dataset did not produce any JSONL entries (missing questions?)." ) gemini_client = GeminiClient.from_credentials( @@ -304,10 +271,7 @@ def start_evaluation_batch( ) else: - raise HTTPException( - status_code=400, - detail=f"Unsupported provider for evaluation batches: {provider}", - ) + raise ValueError(f"Unsupported provider for evaluation batches: {provider}") eval_run.batch_job_id = batch_job.id eval_run.status = "processing" diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index 222e3f3b9..b875eae25 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -4,7 +4,6 @@ from uuid import UUID from langfuse import Langfuse -from sqlalchemy.exc import IntegrityError from sqlmodel import Session, col, select from app.core.cloud.storage import get_cloud_storage @@ -18,8 +17,8 @@ EvaluationScore, ) from app.models import EvaluationRun, EvaluationRunUpdate -from app.models.config.config import ConfigTag from app.models.evaluation import RunModeEnum, TraceData +from app.models.config.config import ConfigTag from app.models.llm.request import ConfigBlob, LLMCallConfig from app.models.stt_evaluation import EvaluationType from app.services.llm.jobs import resolve_config_blob @@ -129,15 +128,9 @@ def create_evaluation_run( try: session.commit() session.refresh(eval_run) - except IntegrityError: - session.rollback() - raise # caller classifies: run_name conflict is a 409, the rest an error except Exception as e: session.rollback() - logger.error( - f"[create_evaluation_run] Failed to create EvaluationRun: {e}", - exc_info=True, - ) + logger.error(f"Failed to create EvaluationRun: {e}", exc_info=True) raise logger.info( @@ -220,12 +213,12 @@ def get_evaluation_run_by_id( if eval_run: logger.info( - f"[get_evaluation_run_by_id] Found evaluation run {evaluation_id}: status={eval_run.status}, " + f"Found evaluation run {evaluation_id}: status={eval_run.status}, " f"batch_job_id={eval_run.batch_job_id}" ) else: logger.warning( - f"[get_evaluation_run_by_id] Evaluation run {evaluation_id} not found or not accessible " + f"Evaluation run {evaluation_id} not found or not accessible " f"for org_id={organization_id}, project_id={project_id}" ) @@ -273,10 +266,7 @@ def update_evaluation_run( session.refresh(eval_run) except Exception as e: session.rollback() - logger.error( - f"[update_evaluation_run] Failed to update EvaluationRun: {e}", - exc_info=True, - ) + logger.error(f"Failed to update EvaluationRun: {e}", exc_info=True) raise if should_notify: diff --git a/backend/app/crud/evaluations/cron_utils.py b/backend/app/crud/evaluations/cron_utils.py index b272e1f83..6aa38dd11 100644 --- a/backend/app/crud/evaluations/cron_utils.py +++ b/backend/app/crud/evaluations/cron_utils.py @@ -324,8 +324,7 @@ async def poll_all_pending_evaluations_by_type( except Exception as client_err: logger.error( f"[{fn_name}] Failed to get Gemini client | " - f"org_id={org_id} | project_id={project_id} | error={client_err}", - exc_info=True, + f"org_id={org_id} | project_id={project_id} | error={client_err}" ) for run in project_runs: update_run_fn( diff --git a/backend/app/crud/evaluations/embeddings.py b/backend/app/crud/evaluations/embeddings.py index 552235ab1..5d1458271 100644 --- a/backend/app/crud/evaluations/embeddings.py +++ b/backend/app/crud/evaluations/embeddings.py @@ -102,15 +102,13 @@ def build_embedding_jsonl( ground_truth = result.get("ground_truth", "") if not item_id: - logger.warning("[build_embedding_jsonl] Skipping result with no item_id") + logger.warning("Skipping result with no item_id") continue # Get trace_id from mapping trace_id = trace_id_mapping.get(item_id) if not trace_id: - logger.warning( - f"[build_embedding_jsonl] Skipping item {item_id} - no trace_id found" - ) + logger.warning(f"Skipping item {item_id} - no trace_id found") skipped.append( {"item_id": item_id, "trace_id": None, "reason": "missing_trace_id"} ) @@ -119,9 +117,7 @@ def build_embedding_jsonl( # Empty output/ground_truth can't be embedded; record the reason. if not generated_output or not ground_truth: reason = "empty_output" if not generated_output else "empty_ground_truth" - logger.warning( - f"[build_embedding_jsonl] Skipping item {item_id} - {reason}" - ) + logger.warning(f"Skipping item {item_id} - {reason}") skipped.append({"item_id": item_id, "trace_id": trace_id, "reason": reason}) continue @@ -144,7 +140,7 @@ def build_embedding_jsonl( jsonl_data.append(batch_request) logger.info( - f"[build_embedding_jsonl] Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" + f"Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" ) return jsonl_data, skipped @@ -164,9 +160,7 @@ def parse_embedding_results( - failed_trace_ids: trace_ids whose embedding could not be parsed (flagged embedding_failed downstream). Lines with no trace_id are dropped silently. """ - logger.info( - f"[parse_embedding_results] Parsing embedding results from {len(raw_results)} lines" - ) + logger.info(f"Parsing embedding results from {len(raw_results)} lines") embedding_pairs = [] failed_trace_ids: list[str] = [] @@ -177,9 +171,7 @@ def parse_embedding_results( # Extract BATCH_KEY (which is now the Langfuse trace_id) trace_id = response.get(BATCH_KEY) if not trace_id: - logger.warning( - f"[parse_embedding_results] Line {line_num}: No {BATCH_KEY} found, skipping" - ) + logger.warning(f"Line {line_num}: No {BATCH_KEY} found, skipping") continue # Handle errors in batch processing @@ -197,7 +189,7 @@ def parse_embedding_results( if len(embedding_data) < 2: logger.warning( - f"[parse_embedding_results] Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" + f"Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" ) failed_trace_ids.append(trace_id) continue @@ -222,7 +214,7 @@ def parse_embedding_results( if output_embedding is None or ground_truth_embedding is None: logger.warning( - f"[parse_embedding_results] Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " + f"Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " f"ground_truth={ground_truth_embedding is not None})" ) failed_trace_ids.append(trace_id) @@ -237,16 +229,13 @@ def parse_embedding_results( ) except Exception as e: - logger.error( - f"[parse_embedding_results] Line {line_num}: Unexpected error: {e}", - exc_info=True, - ) + logger.error(f"Line {line_num}: Unexpected error: {e}", exc_info=True) if trace_id: failed_trace_ids.append(trace_id) continue logger.info( - f"[parse_embedding_results] Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " + f"Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " f"| failed={len(failed_trace_ids)}" ) return embedding_pairs, failed_trace_ids @@ -304,9 +293,7 @@ def calculate_average_similarity( "per_item_scores": [...] # Individual scores with trace_ids } """ - logger.info( - f"[calculate_average_similarity] Calculating similarity for {len(embedding_pairs)} pairs" - ) + logger.info(f"Calculating similarity for {len(embedding_pairs)} pairs") if not embedding_pairs: return { @@ -336,15 +323,12 @@ def calculate_average_similarity( except Exception as e: logger.error( - f"[calculate_average_similarity] Error calculating similarity for trace {pair.get('trace_id')}: {e}", - exc_info=True, + f"Error calculating similarity for trace {pair.get('trace_id')}: {e}" ) continue if not similarities: - logger.warning( - "[calculate_average_similarity] No valid similarities calculated" - ) + logger.warning("No valid similarities calculated") return { "cosine_similarity_avg": 0.0, "cosine_similarity_std": 0.0, @@ -363,7 +347,7 @@ def calculate_average_similarity( } logger.info( - f"[calculate_average_similarity] Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " + f"Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " f"std={stats['cosine_similarity_std']:.3f}" ) @@ -400,9 +384,7 @@ def start_embedding_batch( Exception: If any step fails """ try: - logger.info( - f"[start_embedding_batch] Starting embedding batch for evaluation run {eval_run.id}" - ) + logger.info(f"Starting embedding batch for evaluation run {eval_run.id}") # Use default embedding model embedding_model = EMBEDDING_MODEL @@ -458,7 +440,7 @@ def start_embedding_batch( session.refresh(eval_run) logger.info( - f"[start_embedding_batch] Successfully started embedding batch: batch_job_id={batch_job.id}, " + f"Successfully started embedding batch: batch_job_id={batch_job.id}, " f"provider_batch_id={batch_job.provider_batch_id} " f"for evaluation run {eval_run.id} with {batch_job.total_items} items" ) @@ -466,9 +448,6 @@ def start_embedding_batch( return eval_run except Exception as e: - logger.error( - f"[start_embedding_batch] Failed to start embedding batch: {e}", - exc_info=True, - ) + logger.error(f"Failed to start embedding batch: {e}", exc_info=True) # Don't update eval_run status here - let caller decide raise diff --git a/backend/app/crud/evaluations/langfuse.py b/backend/app/crud/evaluations/langfuse.py index f06761c24..793c89726 100644 --- a/backend/app/crud/evaluations/langfuse.py +++ b/backend/app/crud/evaluations/langfuse.py @@ -205,8 +205,7 @@ def _create_single_trace(result: dict[str, Any]) -> tuple[str, str] | None: if getattr(e, "status_code", None) == 429: logger.error( f"[create_langfuse_dataset_run] Langfuse rate limit (429) | " - f"item_id={item_id}", - exc_info=True, + f"item_id={item_id}" ) else: logger.error( @@ -616,8 +615,7 @@ def _fetch_single_trace(trace_id: str) -> TraceData | None: f"[fetch_trace_scores_from_langfuse] Circuit breaker triggered | " f"consecutive_failures={consecutive_failures} | " f"total_failures={total_failures} | " - f"total_traces={len(trace_ids)}", - exc_info=True, + f"total_traces={len(trace_ids)}" ) raise RuntimeError( f"Langfuse API unavailable: {consecutive_failures} consecutive " diff --git a/backend/app/crud/evaluations/processing.py b/backend/app/crud/evaluations/processing.py index 288ea70d1..a84354989 100644 --- a/backend/app/crud/evaluations/processing.py +++ b/backend/app/crud/evaluations/processing.py @@ -327,8 +327,7 @@ def parse_evaluation_output( except Exception as e: logger.error( - f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}", - exc_info=True, + f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}" ) continue @@ -1207,8 +1206,7 @@ async def poll_all_pending_evaluations(session: Session) -> dict[str, Any]: ) except HTTPException as http_exc: logger.error( - f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}", - exc_info=True, + f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}" ) # Mark all runs in this project as failed due to client configuration error for eval_run in project_runs: diff --git a/backend/app/crud/stt_evaluations/batch.py b/backend/app/crud/stt_evaluations/batch.py index 0a5aaf013..c9ffe61fa 100644 --- a/backend/app/crud/stt_evaluations/batch.py +++ b/backend/app/crud/stt_evaluations/batch.py @@ -227,8 +227,7 @@ def _upload_to_gemini(sample: STTSample) -> _UploadResult: except Exception as e: logger.error( f"[start_stt_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}", - exc_info=True, + f"model: {model}, error: {str(e)}" ) if not batch_jobs: diff --git a/backend/app/crud/tts_evaluations/batch.py b/backend/app/crud/tts_evaluations/batch.py index fff0757a6..3a9abb543 100644 --- a/backend/app/crud/tts_evaluations/batch.py +++ b/backend/app/crud/tts_evaluations/batch.py @@ -136,8 +136,7 @@ def start_tts_evaluation_batch( except Exception as e: logger.error( f"[start_tts_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}", - exc_info=True, + f"model: {model}, error: {str(e)}" ) pending = get_pending_results_for_run( session=session, run_id=run.id, provider=model diff --git a/backend/app/services/evaluations/dataset.py b/backend/app/services/evaluations/dataset.py index d08df7a21..0872b39b7 100644 --- a/backend/app/services/evaluations/dataset.py +++ b/backend/app/services/evaluations/dataset.py @@ -146,14 +146,13 @@ def upload_dataset( f"dataset={dataset_name} | id={langfuse_dataset_id}" ) - except Exception: + except Exception as e: logger.error( - f"[upload_dataset] Failed to upload dataset to Langfuse | dataset={dataset_name}", + f"[upload_dataset] Failed to upload dataset to Langfuse | {e}", exc_info=True, ) raise HTTPException( - status_code=502, - detail=f"Langfuse could not accept dataset '{dataset_name}'. Retry shortly.", + status_code=500, detail=f"Failed to upload dataset to Langfuse: {e}" ) # Step 5: Store metadata in database diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index d960919e5..57ed337f3 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -58,17 +58,6 @@ PROMPT_TEMPLATE_INPUT_PLACEHOLDER = "{{input}}" -def _fail_run(*, session: Session, eval_run: EvaluationRun, reason: str) -> None: - update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate( - status="failed", - error_message=f"Failed to start fast eval: {reason}", - ), - ) - - def is_dataset_fast_eligible(*, original_items_count: int) -> bool: """A dataset is eligible for fast mode when its unique-row count is within cap.""" return original_items_count <= settings.EVAL_FAST_MAX_UNIQUE_ROWS @@ -325,10 +314,7 @@ def validate_and_start_fast_evaluation( ) total_items = original_items * max(1, effective_factor) if total_items == 0: - # Fast eval clients depend on the 500 contract for an empty dataset. - raise HTTPException( - status_code=500, detail=f"Dataset '{dataset.name}' has no items" - ) + raise ValueError(f"Dataset '{dataset.name}' has no items") n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) # v2 markers (is_judge_run, callback_url, duplication_factor) land in one INSERT. @@ -363,22 +349,20 @@ def validate_and_start_fast_evaluation( f"eval_run_id={eval_run.id} | total_items={total_items} | " f"n_chunks={n_chunks}" ) - except HTTPException as exc: - log = logger.error if exc.status_code >= 500 else logger.warning - log( - f"[validate_and_start_fast_evaluation] Failed to start run | " - f"eval_run_id={eval_run.id} | code={exc.status_code}", - exc_info=True, - ) - _fail_run(session=session, eval_run=eval_run, reason=exc.detail) - raise except Exception as exc: logger.error( f"[validate_and_start_fast_evaluation] Failed to start run | " f"eval_run_id={eval_run.id} | error={exc}", exc_info=True, ) - _fail_run(session=session, eval_run=eval_run, reason=str(exc)) + update_evaluation_run( + session=session, + eval_run=eval_run, + update=EvaluationRunUpdate( + status="failed", + error_message=f"Failed to start fast eval: {exc}", + ), + ) raise HTTPException( status_code=500, detail="Failed to start fast evaluation", diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index f10bf5d0e..c6d2f0030 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -221,8 +221,8 @@ def start_prompt_improvement_job( ), ) raise HTTPException( - status_code=503, - detail="prompt_improvement_enqueue_failed: could not queue the job. Retry shortly.", + status_code=500, + detail="prompt_improvement_enqueue_failed: could not queue the job", ) logger.info( diff --git a/backend/app/services/stt_evaluations/audio.py b/backend/app/services/stt_evaluations/audio.py index 1246a4a4c..8f890d7e9 100644 --- a/backend/app/services/stt_evaluations/audio.py +++ b/backend/app/services/stt_evaluations/audio.py @@ -8,7 +8,7 @@ from sqlmodel import Session from app.core.cloud.storage import get_cloud_storage -from fastapi import HTTPException +from app.core.exception_handlers import HTTPException from app.crud.file import create_file from app.models.file import FileType, AudioUploadResponse from app.services.stt_evaluations.constants import ( @@ -121,10 +121,9 @@ def upload_audio_file( except Exception as e: logger.error( f"[upload_audio_file] Failed to upload audio | " - f"project_id: {project_id}, error: {str(e)}", - exc_info=True, + f"project_id: {project_id}, error: {str(e)}" ) raise HTTPException( - status_code=502, - detail="Object storage could not accept the audio file. Retry shortly.", + status_code=500, + detail="Failed to upload audio file. Please try again later.", ) diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index 07aecb291..51e3c4686 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -4,7 +4,6 @@ from uuid import uuid4 import pytest -from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session, select @@ -440,13 +439,13 @@ def test_upload_with_duplication_factor_boundary_minimum( class TestDatasetUploadErrors: """Test error handling.""" - def test_upload_returns_502_when_langfuse_upload_raises( + def test_upload_returns_500_when_langfuse_upload_raises( self, client: TestClient, user_api_key_header: dict[str, str], valid_csv_content: str, ) -> None: - """Tracing enabled but the Langfuse upload fails → 502.""" + """Tracing enabled but the Langfuse upload fails → 500.""" with ( patch("app.core.cloud.get_cloud_storage") as _mock_storage, patch( @@ -475,12 +474,12 @@ def test_upload_returns_502_when_langfuse_upload_raises( headers=user_api_key_header, ) - assert response.status_code == 502, response.text + assert response.status_code == 500, response.text response_data = response.json() error_str = response_data.get( "detail", response_data.get("error", str(response_data)) ) - assert "Langfuse could not accept dataset" in str(error_str) + assert "Failed to upload dataset to Langfuse" in str(error_str) def test_upload_invalid_csv_format( self, client: TestClient, user_api_key_header: dict[str, str] @@ -1134,7 +1133,7 @@ def test_unsupported_provider_marks_failed( ) -> None: mock_fetch.return_value = _BATCH_DATASET_ITEMS - with pytest.raises(HTTPException, match="Unsupported provider"): + with pytest.raises(ValueError, match="Unsupported provider"): start_evaluation_batch( langfuse=MagicMock(), session=db, @@ -1158,7 +1157,7 @@ def test_empty_jsonl_marks_failed( ] mock_map.return_value = ({"model": "gpt-4o"}, []) - with pytest.raises(HTTPException, match="did not produce any JSONL"): + with pytest.raises(ValueError, match="did not produce any JSONL"): start_evaluation_batch( langfuse=MagicMock(), session=db, diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index 0e2cbc41b..17027b399 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -502,7 +502,7 @@ def test_invalid_callback_url_rejected_by_ssrf_guard( assert "invalid_callback_url" in resp.json()["error"], resp.text enqueue.assert_not_called() - def test_enqueue_failure_marks_job_failed_and_raises_503( + def test_enqueue_failure_marks_job_failed_and_raises_500( self, db: Session, auth: TestAuthContext, @@ -521,7 +521,7 @@ def test_enqueue_failure_marks_job_failed_and_raises_503( callback_url=_CALLBACK_URL, ) - assert exc.value.status_code == 503 + assert exc.value.status_code == 500 job = db.exec( select(Job) diff --git a/backend/app/tests/api/routes/test_tts_evaluation.py b/backend/app/tests/api/routes/test_tts_evaluation.py index 0e3957c53..ae9ade1c8 100644 --- a/backend/app/tests/api/routes/test_tts_evaluation.py +++ b/backend/app/tests/api/routes/test_tts_evaluation.py @@ -626,7 +626,7 @@ def test_start_evaluation_celery_failure( headers=user_api_key_header, ) - assert response.status_code == 503 + assert response.status_code == 500 assert "queue" in response.json()["error"].lower() def test_start_evaluation_invalid_dataset( @@ -674,7 +674,7 @@ def test_start_evaluation_empty_dataset( headers=user_api_key_header, ) - assert response.status_code == 422 + assert response.status_code == 400 assert "no samples" in response.json()["error"].lower() def test_start_evaluation_unsupported_model( diff --git a/backend/app/tests/crud/evaluations/test_batch_errors.py b/backend/app/tests/crud/evaluations/test_batch_errors.py deleted file mode 100644 index 262ed9f93..000000000 --- a/backend/app/tests/crud/evaluations/test_batch_errors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Langfuse failures map to distinct status codes rather than a flat 500.""" - -from unittest.mock import MagicMock - -import pytest -from fastapi import HTTPException -from langfuse.api import NotFoundError as LangfuseNotFoundError -from langfuse.api.core import ApiError as LangfuseApiError - -from app.crud.evaluations.batch import fetch_dataset_items - - -def test_dataset_missing_raises_404() -> None: - langfuse = MagicMock() - langfuse.get_dataset.side_effect = LangfuseNotFoundError( - body={"message": "Dataset not found"} - ) - - with pytest.raises(HTTPException) as exc: - fetch_dataset_items(langfuse=langfuse, dataset_name="missing") - - assert exc.value.status_code == 404 - assert "missing" in exc.value.detail - - -def test_langfuse_api_error_raises_502() -> None: - langfuse = MagicMock() - langfuse.get_dataset.side_effect = LangfuseApiError(status_code=500, body="boom") - - with pytest.raises(HTTPException) as exc: - fetch_dataset_items(langfuse=langfuse, dataset_name="ds") - - assert exc.value.status_code == 502 - - -def test_langfuse_unreachable_raises_502() -> None: - langfuse = MagicMock() - langfuse.get_dataset.side_effect = ConnectionError("dns failure") - - with pytest.raises(HTTPException) as exc: - fetch_dataset_items(langfuse=langfuse, dataset_name="ds") - - assert exc.value.status_code == 502 - - -def test_empty_dataset_raises_422() -> None: - langfuse = MagicMock() - langfuse.get_dataset.return_value = MagicMock(items=[]) - - with pytest.raises(HTTPException) as exc: - fetch_dataset_items(langfuse=langfuse, dataset_name="ds") - - assert exc.value.status_code == 422 diff --git a/backend/app/tests/crud/evaluations/test_core.py b/backend/app/tests/crud/evaluations/test_core.py index ee1cc8db7..ead0caaa2 100644 --- a/backend/app/tests/crud/evaluations/test_core.py +++ b/backend/app/tests/crud/evaluations/test_core.py @@ -1,9 +1,6 @@ -import logging from datetime import datetime, timedelta from uuid import UUID, uuid4 -import pytest -from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.core.util import now @@ -22,7 +19,7 @@ get_user_test_auth_context, ) from app.tests.utils.test_data import create_test_evaluation_dataset -from app.tests.utils.utils import make_integrity_error, random_lower_string +from app.tests.utils.utils import random_lower_string def _create_config(db: Session, project_id: int) -> tuple: @@ -120,44 +117,6 @@ def test_create_evaluation_run_sets_text_type(self, db: Session) -> None: assert eval_run.status == "pending" assert eval_run.run_name == "test_run" - def test_duplicate_run_name_is_left_to_caller_without_error_log( - self, db: Session, monkeypatch, caplog - ) -> None: - """The service turns a run_name collision into a 409; crud must not log it as an error.""" - org = db.exec(select(Organization)).first() - project = db.exec( - select(Project).where(Project.organization_id == org.id) - ).first() - dataset = create_evaluation_dataset( - session=db, - name="test_dataset_dup_run", - dataset_metadata={"original_items_count": 10}, - organization_id=org.id, - project_id=project.id, - ) - config_id, config_version = _create_config(db, project.id) - - def _fail_commit() -> None: - raise make_integrity_error( - "23505", "uq_evaluation_run_org_project_run_name" - ) - - monkeypatch.setattr(db, "commit", _fail_commit) - - with caplog.at_level(logging.WARNING), pytest.raises(IntegrityError): - create_evaluation_run( - session=db, - run_name="dup_run", - dataset_name=dataset.name, - dataset_id=dataset.id, - config_id=config_id, - config_version=config_version, - organization_id=org.id, - project_id=project.id, - ) - - assert not [r for r in caplog.records if r.levelno >= logging.ERROR] - class TestGetEvaluationRunById: """Test fetching evaluation runs by ID.""" diff --git a/backend/app/tests/services/stt_evaluations/test_audio.py b/backend/app/tests/services/stt_evaluations/test_audio.py index 48a6ab6b7..a1c1d2342 100644 --- a/backend/app/tests/services/stt_evaluations/test_audio.py +++ b/backend/app/tests/services/stt_evaluations/test_audio.py @@ -5,8 +5,7 @@ import pytest from fastapi import UploadFile -from fastapi import HTTPException - +from app.core.exception_handlers import HTTPException from app.services.stt_evaluations.audio import ( _resolve_extension, _validate_audio_file, @@ -186,8 +185,8 @@ def test_upload_storage_error(self, mock_get_storage) -> None: organization_id=1, project_id=1, ) - assert exc_info.value.status_code == 502 - assert "Object storage could not accept" in str(exc_info.value.detail) + assert exc_info.value.status_code == 500 + assert "Failed to upload audio file" in str(exc_info.value.detail) @patch("app.services.stt_evaluations.audio.create_file") @patch("app.services.stt_evaluations.audio.get_cloud_storage") From a151abb50e724da657b55b6c76765690295a89d9 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:20:45 +0530 Subject: [PATCH 10/12] fix: improve observability by tagging Sentry tasks with tenant information and update exception handling for HTTP exceptions --- backend/app/api/deps.py | 4 +- backend/app/celery/celery_app.py | 15 ++++++- backend/app/core/exception_handlers.py | 44 ++++++++++++------- backend/app/services/llm/jobs.py | 3 +- .../app/tests/core/test_exception_handlers.py | 19 ++------ backend/app/utils.py | 2 +- docs/wiki/cross-cutting/observability.md | 1 + 7 files changed, 49 insertions(+), 39 deletions(-) diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index a84c3c33d..a35dc37e1 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -50,9 +50,9 @@ def _set_tenant_span_attributes(auth_context: AuthContext) -> None: if span.is_recording(): span.set_attribute("user.id", str(auth_context.user.id)) if auth_context.organization: - span.set_attribute("tenant.org_id", auth_context.organization.id) + span.set_attribute("tenant.org_id", str(auth_context.organization.id)) if auth_context.project: - span.set_attribute("tenant.project_id", auth_context.project.id) + span.set_attribute("tenant.project_id", str(auth_context.project.id)) org_id = auth_context.organization.id if auth_context.organization else None project_id = auth_context.project.id if auth_context.project else None diff --git a/backend/app/celery/celery_app.py b/backend/app/celery/celery_app.py index a98d133f8..5d816c699 100644 --- a/backend/app/celery/celery_app.py +++ b/backend/app/celery/celery_app.py @@ -46,9 +46,20 @@ def _initialize_worker_observability() -> None: ) @task_prerun.connect(weak=False) - def _tag_sentry_task(task: "celery.Task", **_: object) -> None: # type: ignore[name-defined] - """Tag the task's Sentry scope with its name so Issues can be filtered by task.""" + def _tag_sentry_task( + task: "celery.Task", # type: ignore[name-defined] + kwargs: dict[str, object] | None = None, + **_: object, + ) -> None: + """Tag the task's Sentry scope with its name and tenant so Issues can be filtered by both.""" sentry_sdk.set_tag("celery.task_name", task.name) + task_kwargs = kwargs or {} + for tag, key in ( + ("tenant.project_id", "project_id"), + ("tenant.org_id", "organization_id"), + ): + if task_kwargs.get(key) is not None: + sentry_sdk.set_tag(tag, str(task_kwargs[key])) _sentry_initialized = True diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 9a04c4c17..71a507f56 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -4,9 +4,10 @@ import sentry_sdk from asgi_correlation_id import correlation_id -from fastapi import FastAPI, HTTPException, Request +from fastapi import FastAPI, HTTPException, Request # noqa: F401 (re-exported) from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException from sqlalchemy.exc import DataError, OperationalError, SQLAlchemyError from starlette.status import ( HTTP_409_CONFLICT, @@ -125,6 +126,23 @@ def _db_error_response(exc: SQLAlchemyError) -> tuple[int, str]: return HTTP_500_INTERNAL_SERVER_ERROR, DB_ERROR_DETAIL +def _tag_error_scope(request: Request, status_code: int) -> None: + """Tag the Sentry scope with route, method and status before the error is logged. + + The middleware tags the route only after the response, too late for an event raised here. + """ + if not sentry_sdk.get_client().is_active(): + return + scope = sentry_sdk.get_current_scope() + route = getattr(request.scope.get("route"), "path", None) + if route: + scope.set_tag("http.route", route) + scope.set_transaction_name(f"{request.method} {route}") + scope.set_tag("http.method", request.method) + scope.set_tag("http.status_code", str(status_code)) + scope.set_tag("error.class", "server" if status_code >= 500 else "client") + + def register_exception_handlers(app: FastAPI) -> None: @app.exception_handler(RequestValidationError) async def validation_error_handler( @@ -136,21 +154,15 @@ async def validation_error_handler( content=APIResponse.failure_response(errors).model_dump(), ) - @app.exception_handler(HTTPException) + @app.exception_handler(StarletteHTTPException) async def http_exception_handler( - request: Request, exc: HTTPException + request: Request, exc: StarletteHTTPException ) -> JSONResponse: detail = exc.detail if isinstance(detail, list): detail = _sanitize_validation_errors(detail) - # Tag at the boundary: the status is only known here, and call-site logs - # fire before it. 5xx is a server fault (error event); 4xx is the caller's. - if sentry_sdk.get_client().is_active(): - sentry_sdk.set_tag("http.status_code", str(exc.status_code)) - sentry_sdk.set_tag( - "error.class", "server" if exc.status_code >= 500 else "client" - ) + _tag_error_scope(request, exc.status_code) if exc.status_code >= 500: logger.error( f"[http_exception] {request.method} {request.url.path} " @@ -160,19 +172,16 @@ async def http_exception_handler( return JSONResponse( status_code=exc.status_code, content=APIResponse.failure_response(detail).model_dump(), + headers=exc.headers, ) @app.exception_handler(SQLAlchemyError) async def db_error_handler(request: Request, exc: SQLAlchemyError) -> JSONResponse: status_code, hint = _db_error_response(exc) sqlstate, constraint = _db_error_constraint(exc) - if sentry_sdk.get_client().is_active(): - sentry_sdk.set_tag("http.status_code", str(status_code)) - sentry_sdk.set_tag( - "error.class", "server" if status_code >= 500 else "client" - ) - if constraint: - sentry_sdk.set_tag("db.constraint", constraint) + _tag_error_scope(request, status_code) + if constraint and sentry_sdk.get_client().is_active(): + sentry_sdk.set_tag("db.constraint", constraint) log = logger.error if status_code >= 500 else logger.warning log( f"[db_error_handler] Unhandled DB error | path: {request.url.path}, " @@ -189,6 +198,7 @@ async def db_error_handler(request: Request, exc: SQLAlchemyError) -> JSONRespon async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. # logger.error alone reports it; adding capture_exception makes a second event. + _tag_error_scope(request, HTTP_500_INTERNAL_SERVER_ERROR) logger.error( f"[generic_error_handler] Unhandled exception | " f"path: {request.url.path}, trace_id: {correlation_id.get() or 'N/A'}", diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index a4db185c3..09c46ce49 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -1433,7 +1433,8 @@ def execute_llm_call( except (Timeout, SoftTimeLimitExceeded): raise except HTTPException as e: - logger.error( + log = logger.error if e.status_code >= 500 else logger.warning + log( f"[execute_llm_call] HTTP error: {e.detail} | job_id={job_id}", exc_info=True, ) diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index 8a6eb9dd9..dfca4f09d 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,6 +1,6 @@ import logging from collections.abc import Iterator -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest import sentry_sdk @@ -295,21 +295,6 @@ def test_server_error_is_logged(self) -> None: assert response.status_code == 502 log_error.assert_called_once() - def test_tags_status_class_when_sentry_active(self) -> None: - client = MagicMock() - client.is_active.return_value = True - with ( - patch( - "app.core.exception_handlers.sentry_sdk.get_client", - return_value=client, - ), - patch("app.core.exception_handlers.sentry_sdk.set_tag") as set_tag, - ): - _app_raising(HTTPException(status_code=404, detail="nope")).get("/boom") - - set_tag.assert_any_call("http.status_code", "404") - set_tag.assert_any_call("error.class", "client") - @pytest.fixture def sentry_events() -> Iterator[list[Event]]: @@ -344,6 +329,8 @@ def test_server_http_exception_event_carries_status_tags( tags = sentry_events[0]["tags"] assert tags["http.status_code"] == "502" assert tags["error.class"] == "server" + assert tags["http.route"] == "/boom" + assert sentry_events[0]["transaction"] == "GET /boom" def test_generic_handler_reports_exactly_one_event( self, sentry_events: list[Event] diff --git a/backend/app/utils.py b/backend/app/utils.py index 1d215fc9f..c8692ec96 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -608,7 +608,7 @@ def send_callback( except requests.RequestException as e: status = e.response.status_code if e.response is not None else None - logger.error( + logger.warning( f"[send_callback] Callback failed | url={callback_url} | status={status} " f"| duration_ms={(time.perf_counter() - started_at) * 1000:.2f} " f"| bytes={len(raw_body)} | payload={log_payload} | error={e}", diff --git a/docs/wiki/cross-cutting/observability.md b/docs/wiki/cross-cutting/observability.md index a247cc6a5..43b1dd95a 100644 --- a/docs/wiki/cross-cutting/observability.md +++ b/docs/wiki/cross-cutting/observability.md @@ -17,6 +17,7 @@ OTel-first, Sentry as sole in-process sink (`instrumenter="otel"`). Init at `mai - Sampling/PII: `SENTRY_TRACES_SAMPLE_RATE`, `SENTRY_SEND_DEFAULT_PII`, `SENTRY_ERROR_SAMPLE_RATE` (config.py; defaults preserve current behavior). - Trace propagation: OTel's `CeleryInstrumentor` parents the worker's `run/...` span to `apply_async`; never inject/extract trace context by hand around tasks (it reparents task spans to the API request and Sentry splits them into separate transactions). `CeleryIntegration(propagate_traces=True)` links API to worker as one trace; poll-loop re-enqueues pass `SENTRY_NO_PROPAGATE_HEADERS` (`celery/tasks/job_execution.py`) to start a fresh trace. - Error capture: handlers in `core/exception_handlers.py` report via `logger.error(exc_info=exc)` (5xx only). Don't add `capture_exception` next to it: on sentry-sdk 2.54 Dedupe does not merge the two, so each error becomes two events. +- Celery tasks: `task_prerun` in `celery/celery_app.py` tags each task's Sentry scope with `celery.task_name` and, from the task kwargs, `tenant.project_id` / `tenant.org_id` (Sentry isolates the scope per task, so tags don't leak between tasks). - Tenant impact (both in `core/telemetry/context.py`, called from `api/deps.py`): `set_request_log_context` puts org/project in the log context + Sentry tags; `bind_sentry_user` binds user/org/project via `sentry_sdk.set_user` so issues report users/orgs affected. The user binding is scope-only — ids in the log context would stamp per-user cardinality on every INFO record `enable_logs` ships. Per-request identity is the `correlation_id` tag (`core/middleware.py`), not the user binding. - Crons: `@sentry_sdk.monitor` on `/cron/*` endpoints (`api/routes/cron.py`), e.g. `llm-call-retention-cron-job` (daily 09:00 IST). - Runbook (alerts/dashboards/debugging): `features/sentry-utilization/SENTRY-RUNBOOK.md`. From 8df56ca77d55319ad43afa81edaf7f8f8f8a5079 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:59:55 +0530 Subject: [PATCH 11/12] fix: add additional GenAI content keys for improved data handling --- backend/app/core/telemetry/sentry/filters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/app/core/telemetry/sentry/filters.py b/backend/app/core/telemetry/sentry/filters.py index 00f37a69f..65d2b3fdc 100644 --- a/backend/app/core/telemetry/sentry/filters.py +++ b/backend/app/core/telemetry/sentry/filters.py @@ -16,6 +16,8 @@ _GENAI_CONTENT_KEYS: frozenset[str] = frozenset( { "gen_ai.request.messages", + "gen_ai.input.messages", + "gen_ai.output.messages", "gen_ai.response.text", "gen_ai.response.tool_calls", "gen_ai.choice", From a8f5863e630c02f0ea6f9984b702ee642229e550 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:57:53 +0530 Subject: [PATCH 12/12] test: cover Celery enqueue helpers, task wrappers and tenant tagging --- backend/app/tests/api/test_deps.py | 23 +++- backend/app/tests/celery/test_celery_app.py | 27 ++++ .../app/tests/celery/test_job_execution.py | 89 +++++++++++++ backend/app/tests/celery/test_utils.py | 117 ++++++++++++++++++ 4 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 backend/app/tests/celery/test_utils.py diff --git a/backend/app/tests/api/test_deps.py b/backend/app/tests/api/test_deps.py index 881df45e8..cd2494b7a 100644 --- a/backend/app/tests/api/test_deps.py +++ b/backend/app/tests/api/test_deps.py @@ -1,11 +1,11 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session -from app.api.deps import get_auth_context +from app.api.deps import _set_tenant_span_attributes, get_auth_context from app.core.config import settings from app.core.security import create_access_token, create_refresh_token from app.models import ( @@ -270,3 +270,22 @@ def test_get_auth_context_jwt_with_org_and_project( assert auth_context.organization.id == user_api_key.organization.id assert auth_context.project is not None assert auth_context.project.id == user_api_key.project.id + + +def test_tenant_attributes_set_on_recording_span() -> None: + span = MagicMock() + span.is_recording.return_value = True + auth_context = MagicMock() + auth_context.user.id = 5 + auth_context.organization.id = 2 + auth_context.project.id = 3 + + with ( + patch("app.api.deps.trace.get_current_span", return_value=span), + patch("app.api.deps.set_request_log_context"), + patch("app.api.deps.bind_sentry_user"), + ): + _set_tenant_span_attributes(auth_context) + + span.set_attribute.assert_any_call("tenant.org_id", "2") + span.set_attribute.assert_any_call("tenant.project_id", "3") diff --git a/backend/app/tests/celery/test_celery_app.py b/backend/app/tests/celery/test_celery_app.py index f40275440..512345c82 100644 --- a/backend/app/tests/celery/test_celery_app.py +++ b/backend/app/tests/celery/test_celery_app.py @@ -90,3 +90,30 @@ def test_tags_sentry_scope_with_task_name_on_prerun(self) -> None: sentry.set_tag.assert_called_once_with( "celery.task_name", "app.celery.tasks.job_execution.run_llm_job" ) + + def test_tags_sentry_scope_with_tenant_from_task_kwargs(self) -> None: + hooks: list = [] + sentry = MagicMock() + with ( + patch.object(celery_app, "_sentry_initialized", False), + patch.object(celery_app, "_telemetry_initialized", True), + patch.object( + celery_app.settings, "SENTRY_DSN", "https://k@o.ingest.sentry.io/1" + ), + patch("app.core.telemetry.init_sentry", MagicMock()), + patch.object( + celery_app.task_prerun, + "connect", + MagicMock(side_effect=lambda **_: hooks.append), + ), + patch.object(celery_app, "configure_logging", MagicMock()), + patch.dict("sys.modules", {"sentry_sdk": sentry}), + ): + celery_app._initialize_worker_observability() + hooks[0]( + task=SimpleNamespace(name="run_llm_job"), + kwargs={"project_id": 3, "organization_id": 2}, + ) + + sentry.set_tag.assert_any_call("tenant.project_id", "3") + sentry.set_tag.assert_any_call("tenant.org_id", "2") diff --git a/backend/app/tests/celery/test_job_execution.py b/backend/app/tests/celery/test_job_execution.py index c18eb32e8..12e94ee60 100644 --- a/backend/app/tests/celery/test_job_execution.py +++ b/backend/app/tests/celery/test_job_execution.py @@ -13,6 +13,44 @@ ("run_llm_job", "app.services.llm.jobs.execute_job"), ("run_llm_chain_job", "app.services.llm.jobs.execute_chain_job"), ("run_response_job", "app.services.response.jobs.execute_job"), + ("run_guardrails_job", "app.services.guardrails.jobs.execute_job"), + ("run_doctransform_job", "app.services.doctransform.job.execute_job"), + ( + "run_collection_setup_job", + "app.services.collections.create_collection.execute_setup_job", + ), + ( + "run_collection_batch_job", + "app.services.collections.create_collection.execute_batch_job", + ), + ( + "run_delete_collection_job", + "app.services.collections.delete_collection.execute_job", + ), + ( + "run_evaluation_batch_submission", + "app.services.evaluations.batch_job.execute_evaluation_batch_submission", + ), + ( + "run_prompt_improvement", + "app.services.evaluations.prompt_improvement.execute_prompt_improvement", + ), + ( + "run_stt_batch_submission", + "app.services.stt_evaluations.batch_job.execute_batch_submission", + ), + ( + "run_stt_metric_computation", + "app.services.stt_evaluations.metric_job.execute_metric_computation", + ), + ( + "run_tts_batch_submission", + "app.services.tts_evaluations.batch_job.execute_batch_submission", + ), + ( + "run_tts_result_processing", + "app.services.tts_evaluations.batch_result_processing.execute_tts_result_processing", + ), ], ) class TestJobWrappers: @@ -27,6 +65,57 @@ def test_returns_service_result(self, task_name, service_target): assert result is SENTINEL +@pytest.mark.parametrize( + ("task_name", "service_target", "task_kwargs"), + [ + ( + "run_evaluation_fast_chunk", + "app.services.evaluations.fast.execute_fast_evaluation_chunk", + {"eval_run_id": 7, "chunk_index": 2}, + ), + ( + "run_evaluation_fast_aggregate", + "app.services.evaluations.fast.execute_fast_evaluation_aggregate", + {"eval_run_id": 7}, + ), + ( + "run_evaluation_iteration_graph_step", + "app.services.evaluations.iteration_graph.execute_evaluation_iteration_graph_step", + { + "iteration_run_id": 3, + "resume": True, + "organization_id": 1, + "project_id": 2, + }, + ), + ], +) +def test_evaluation_task_sets_trace_and_returns_service_result( + task_name, service_target, task_kwargs +): + task = getattr(job_execution, task_name) + with patch(service_target, return_value=SENTINEL): + result = task(trace_id="trace-1", **task_kwargs) + + assert result is SENTINEL + assert job_execution.correlation_id.get() == "trace-1" + + +def test_assessment_pipeline_returns_service_result(): + with ( + patch.object(job_execution, "_set_trace", lambda trace_id: None), + patch( + "app.services.assessment.tasks.execute_assessment_pipeline", + return_value=SENTINEL, + ), + ): + result = job_execution.run_assessment_pipeline( + run_id=1, organization_id=2, project_id=3, trace_id="trace-1" + ) + + assert result is SENTINEL + + class TestAssessmentApiBatchReenqueue: def _run(self, requeue: bool): with ( diff --git a/backend/app/tests/celery/test_utils.py b/backend/app/tests/celery/test_utils.py new file mode 100644 index 000000000..db82366c2 --- /dev/null +++ b/backend/app/tests/celery/test_utils.py @@ -0,0 +1,117 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.celery import utils +from app.celery.tasks import job_execution + +JOB_KWARGS = {"project_id": 1, "job_id": "job-1", "trace_id": "trace-1"} + + +@pytest.mark.parametrize( + ("starter", "task_name", "call_kwargs", "expected_kwargs"), + [ + ("start_llm_job", "run_llm_job", JOB_KWARGS, JOB_KWARGS), + ("start_llm_chain_job", "run_llm_chain_job", JOB_KWARGS, JOB_KWARGS), + ("start_guardrails_job", "run_guardrails_job", JOB_KWARGS, JOB_KWARGS), + ("start_response_job", "run_response_job", JOB_KWARGS, JOB_KWARGS), + ("start_doctransform_job", "run_doctransform_job", JOB_KWARGS, JOB_KWARGS), + ( + "start_collection_setup_job", + "run_collection_setup_job", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_collection_batch_job", + "run_collection_batch_job", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_delete_collection_job", + "run_delete_collection_job", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_evaluation_batch_submission", + "run_evaluation_batch_submission", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_prompt_improvement", + "run_prompt_improvement", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_stt_batch_submission", + "run_stt_batch_submission", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_stt_metric_computation", + "run_stt_metric_computation", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_tts_batch_submission", + "run_tts_batch_submission", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_tts_result_processing", + "run_tts_result_processing", + JOB_KWARGS, + JOB_KWARGS, + ), + ( + "start_fast_evaluation_chunk", + "run_evaluation_fast_chunk", + {"eval_run_id": 7, "chunk_index": 2}, + {"eval_run_id": 7, "chunk_index": 2, "trace_id": "N/A"}, + ), + ( + "start_fast_evaluation_aggregate", + "run_evaluation_fast_aggregate", + {"eval_run_id": 7}, + {"eval_run_id": 7, "trace_id": "N/A"}, + ), + ( + "start_evaluation_iteration_round", + "run_evaluation_iteration_graph_step", + { + "iteration_run_id": 3, + "resume": True, + "organization_id": 1, + "project_id": 2, + }, + { + "iteration_run_id": 3, + "resume": True, + "organization_id": 1, + "project_id": 2, + "trace_id": "N/A", + }, + ), + ], +) +def test_start_helper_enqueues_task_and_returns_its_id( + starter: str, + task_name: str, + call_kwargs: dict[str, object], + expected_kwargs: dict[str, object], +) -> None: + task = getattr(job_execution, task_name) + with patch.object( + task, "apply_async", return_value=MagicMock(id="task-1") + ) as apply_async: + task_id = getattr(utils, starter)(**call_kwargs) + + assert task_id == "task-1" + apply_async.assert_called_once_with(kwargs=expected_kwargs)