From f836b24a9392601f90d92b34534f1aa5411b171a Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 6 Oct 2026 09:34:59 +0530 Subject: [PATCH 1/3] feat: add sarvam and elevenlabs to evals --- .../docs/tts_evaluation/start_evaluation.md | 9 +- .../api/routes/tts_evaluations/evaluation.py | 9 +- backend/app/celery/tasks/job_execution.py | 26 +- backend/app/celery/utils.py | 19 + backend/app/core/batch/operations.py | 5 + backend/app/crud/language.py | 6 + backend/app/crud/tts_evaluations/__init__.py | 21 +- backend/app/crud/tts_evaluations/batch.py | 83 ++-- backend/app/crud/tts_evaluations/cron.py | 29 +- backend/app/crud/tts_evaluations/result.py | 81 +++- backend/app/crud/tts_evaluations/run.py | 57 +++ backend/app/models/tts_evaluation.py | 17 +- .../app/services/tts_evaluations/batch_job.py | 381 +++++++++++---- .../batch_result_processing.py | 403 ++++++++-------- .../app/services/tts_evaluations/constants.py | 123 ++++- .../app/services/tts_evaluations/dataset.py | 41 +- .../app/services/tts_evaluations/language.py | 61 +++ .../tts_evaluations/sync_generation.py | 449 ++++++++++++++++++ .../services/tts_evaluations/synthesizers.py | 403 ++++++++++++++++ .../tests/api/routes/test_tts_evaluation.py | 34 ++ backend/app/tests/celery/test_tts_tasks.py | 84 ++++ .../app/tests/core/batch/test_operations.py | 102 ++++ backend/app/tests/crud/test_language.py | 17 + .../tests/crud/tts_evaluations/__init__.py | 0 .../tests/crud/tts_evaluations/test_batch.py | 180 +++++++ .../tests/crud/tts_evaluations/test_cron.py | 191 ++++++++ .../tests/crud/tts_evaluations/test_result.py | 120 +++++ .../tests/crud/tts_evaluations/test_run.py | 85 ++++ .../app/tests/models/test_tts_evaluation.py | 52 ++ .../services/tts_evaluations/__init__.py | 0 .../tts_evaluations/test_batch_job.py | 357 ++++++++++++++ .../test_batch_result_processing.py | 289 +++++++++++ .../tts_evaluations/test_constants.py | 62 +++ .../services/tts_evaluations/test_dataset.py | 113 +++++ .../services/tts_evaluations/test_language.py | 75 +++ .../tts_evaluations/test_sync_generation.py | 347 ++++++++++++++ .../tts_evaluations/test_synthesizers.py | 368 ++++++++++++++ backend/app/tests/utils/tts_evaluation.py | 167 +++++++ docs/wiki/modules/evaluations.md | 5 +- 39 files changed, 4510 insertions(+), 361 deletions(-) create mode 100644 backend/app/services/tts_evaluations/language.py create mode 100644 backend/app/services/tts_evaluations/sync_generation.py create mode 100644 backend/app/services/tts_evaluations/synthesizers.py create mode 100644 backend/app/tests/celery/test_tts_tasks.py create mode 100644 backend/app/tests/core/batch/test_operations.py create mode 100644 backend/app/tests/crud/test_language.py create mode 100644 backend/app/tests/crud/tts_evaluations/__init__.py create mode 100644 backend/app/tests/crud/tts_evaluations/test_batch.py create mode 100644 backend/app/tests/crud/tts_evaluations/test_cron.py create mode 100644 backend/app/tests/crud/tts_evaluations/test_result.py create mode 100644 backend/app/tests/crud/tts_evaluations/test_run.py create mode 100644 backend/app/tests/models/test_tts_evaluation.py create mode 100644 backend/app/tests/services/tts_evaluations/__init__.py create mode 100644 backend/app/tests/services/tts_evaluations/test_batch_job.py create mode 100644 backend/app/tests/services/tts_evaluations/test_batch_result_processing.py create mode 100644 backend/app/tests/services/tts_evaluations/test_constants.py create mode 100644 backend/app/tests/services/tts_evaluations/test_dataset.py create mode 100644 backend/app/tests/services/tts_evaluations/test_language.py create mode 100644 backend/app/tests/services/tts_evaluations/test_sync_generation.py create mode 100644 backend/app/tests/services/tts_evaluations/test_synthesizers.py create mode 100644 backend/app/tests/utils/tts_evaluation.py diff --git a/backend/app/api/docs/tts_evaluation/start_evaluation.md b/backend/app/api/docs/tts_evaluation/start_evaluation.md index 40a7c3ff2..8a2107ca5 100644 --- a/backend/app/api/docs/tts_evaluation/start_evaluation.md +++ b/backend/app/api/docs/tts_evaluation/start_evaluation.md @@ -9,7 +9,10 @@ Optional fields: The evaluation will: 1. Process each text sample through the specified TTS models -2. Generate speech audio using Gemini Batch API -3. Store WAV audio files in S3 for human review +2. Generate speech audio: Gemini models via the Gemini Batch API; Sarvam and ElevenLabs models via direct API calls in background workers +3. Store 24 kHz mono WAV audio files in S3 for human review -**Supported models:** `gemini-2.5-pro-preview-tts` +Synthesis uses the dataset's language; datasets without a language are synthesized as `en-IN`. +The response's `run_metadata.voices` lists the voice used per model. + +**Supported models:** `gemini-2.5-pro-preview-tts`, `bulbul:v3` (Sarvam), `eleven_v3`, `eleven_v4` (ElevenLabs) diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index 6751d8219..7ef45790c 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -25,6 +25,7 @@ from app.services.tts_evaluations.constants import ( DEFAULT_STYLE_PROMPT, DEFAULT_VOICE_NAME, + get_tts_model_spec, ) from app.utils import APIResponse, load_description @@ -83,7 +84,7 @@ def start_tts_evaluation( total_items=sample_count * len(run_create.models), ) - # Offload batch submission (result creation, JSONL, Gemini upload) to Celery worker + # Offload submission (result creation, Gemini batch, sync-model fan-out) to Celery worker trace_id = correlation_id.get() or "N/A" try: celery_task_id = start_tts_batch_submission( @@ -114,11 +115,17 @@ def start_tts_evaluation( detail=f"Failed to queue batch submission: {e}", ) + voices: dict[str, str] = {} + for model in run_create.models: + voices[model] = get_tts_model_spec(model).default_voice + return APIResponse.success_response( data=TTSEvaluationRunPublic.from_model( run, run_metadata={ + # Legacy key kept for existing clients; `voices` is per model. "voice_name": DEFAULT_VOICE_NAME, + "voices": voices, "style_prompt": DEFAULT_STYLE_PROMPT, }, ) diff --git a/backend/app/celery/tasks/job_execution.py b/backend/app/celery/tasks/job_execution.py index 483384364..d33ea4503 100644 --- a/backend/app/celery/tasks/job_execution.py +++ b/backend/app/celery/tasks/job_execution.py @@ -6,7 +6,8 @@ 9 LLM call + LLM chain (run_llm_job, run_llm_chain_job, run_response_job) 6 Fast evaluation (run_evaluation_fast_chunk, run_evaluation_fast_aggregate, run_prompt_improvement, run_evaluation_iteration_graph_step) - 2 Everything else (doctransform, collections, STT/TTS evaluation, assessment) + 2 Everything else (doctransform, collections, STT/TTS evaluation incl. + run_tts_sync_chunk, assessment) 1 Notifications (send_eval_completion_notification) Higher priority drains first; within the same priority, delivery is FIFO. @@ -429,6 +430,29 @@ def run_tts_result_processing( ) +@celery_app.task(bind=True, queue="default", priority=2) +@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_tts_sync_chunk") +def run_tts_sync_chunk(self, project_id: int, job_id: str, trace_id: str, **kwargs): + """Synthesize one chunk of a TTS run's sync-model results. + + Idempotent: only results still PENDING are synthesized and written, so a + redelivered chunk skips rows an earlier attempt already finished. + """ + from app.services.tts_evaluations.sync_generation import execute_tts_sync_chunk + + _set_trace(trace_id) + return _run_with_otel_parent( + self, + lambda: execute_tts_sync_chunk( + project_id=project_id, + job_id=job_id, + task_id=current_task.request.id, + task_instance=self, + **kwargs, + ), + ) + + @celery_app.task(bind=True, queue="default", priority=6) @gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_evaluation_fast_chunk") def run_evaluation_fast_chunk( diff --git a/backend/app/celery/utils.py b/backend/app/celery/utils.py index 3bd205d22..98c416da3 100644 --- a/backend/app/celery/utils.py +++ b/backend/app/celery/utils.py @@ -270,6 +270,25 @@ def start_tts_result_processing( return task_id +def start_tts_sync_chunk( + project_id: int, job_id: str, trace_id: str = "N/A", **kwargs: Any +) -> str: + """Enqueue one synchronous TTS synthesis chunk (one model, a slice of result IDs).""" + from app.celery.tasks.job_execution import run_tts_sync_chunk + + task_id = _enqueue_with_trace_context( + run_tts_sync_chunk, + project_id=project_id, + job_id=job_id, + trace_id=trace_id, + **kwargs, + ) + logger.info( + f"[start_tts_sync_chunk] Started job {job_id} with Celery task {task_id}" + ) + return task_id + + def start_fast_evaluation_chunk( eval_run_id: int, chunk_index: int, trace_id: str = "N/A" ) -> str: diff --git a/backend/app/core/batch/operations.py b/backend/app/core/batch/operations.py index d8db4151b..04f1e76e6 100644 --- a/backend/app/core/batch/operations.py +++ b/backend/app/core/batch/operations.py @@ -49,6 +49,11 @@ def start_batch_job( batch_job = create_batch_job(session=session, batch_job_create=batch_job_create) + # create_batch_job's refresh() re-opens a transaction; end it so no pooled + # connection sits idle-in-transaction during the (slow) upload + create. + # Nothing is pending at this point, so this only releases the connection. + session.commit() + try: batch_result = provider.create_batch(jsonl_data=jsonl_data, config=config) diff --git a/backend/app/crud/language.py b/backend/app/crud/language.py index 765e50542..4d6fc3004 100644 --- a/backend/app/crud/language.py +++ b/backend/app/crud/language.py @@ -36,3 +36,9 @@ def get_language_by_locale(session: Session, locale: str) -> Optional[Language]: """Retrieve a language by its locale code.""" statement = select(Language).where(Language.locale == locale) return session.exec(statement).first() + + +def get_language_locale_by_id(session: Session, language_id: int) -> str | None: + """Return a language's ISO 639-1 locale, or None if the ID doesn't exist.""" + statement = select(Language.locale).where(Language.id == language_id) + return session.exec(statement).first() diff --git a/backend/app/crud/tts_evaluations/__init__.py b/backend/app/crud/tts_evaluations/__init__.py index fcced112f..bb476b7d9 100644 --- a/backend/app/crud/tts_evaluations/__init__.py +++ b/backend/app/crud/tts_evaluations/__init__.py @@ -1,27 +1,31 @@ """TTS Evaluation CRUD operations.""" -from .batch import start_tts_evaluation_batch +from .batch import TTSBatchSubmissionError, start_tts_evaluation_batch from .cron import poll_all_pending_tts_evaluations from .dataset import ( create_tts_dataset, get_tts_dataset_by_id, list_tts_datasets, ) +from .result import ( + bulk_update_pending_tts_results, + create_tts_results, + get_results_by_run_id, + get_tts_result_by_id, + list_pending_tts_results_by_ids, + update_tts_human_feedback, +) from .run import ( create_tts_run, + finalize_tts_run_status, get_tts_run_by_id, list_tts_runs, update_tts_run, ) -from .result import ( - create_tts_results, - get_tts_result_by_id, - get_results_by_run_id, - update_tts_human_feedback, -) __all__ = [ # Batch + "TTSBatchSubmissionError", "start_tts_evaluation_batch", # Cron "poll_all_pending_tts_evaluations", @@ -31,12 +35,15 @@ "list_tts_datasets", # Run "create_tts_run", + "finalize_tts_run_status", "get_tts_run_by_id", "list_tts_runs", "update_tts_run", # Result + "bulk_update_pending_tts_results", "create_tts_results", "get_tts_result_by_id", "get_results_by_run_id", + "list_pending_tts_results_by_ids", "update_tts_human_feedback", ] diff --git a/backend/app/crud/tts_evaluations/batch.py b/backend/app/crud/tts_evaluations/batch.py index 3a9abb543..639cc8cb5 100644 --- a/backend/app/crud/tts_evaluations/batch.py +++ b/backend/app/crud/tts_evaluations/batch.py @@ -11,33 +11,38 @@ create_tts_batch_requests, start_batch_job, ) -from app.models.batch_job import BatchJobType from app.crud.tts_evaluations.result import ( get_pending_results_for_run, update_tts_result, ) from app.crud.tts_evaluations.run import update_tts_run from app.models import EvaluationRun +from app.models.batch_job import BatchJobType from app.models.job import JobStatus from app.models.tts_evaluation import TTSResult from app.services.tts_evaluations.constants import ( DEFAULT_STYLE_PROMPT, - DEFAULT_TTS_MODEL, - DEFAULT_VOICE_NAME, + TTSExecutionModeEnum, + get_tts_model_spec, ) logger = logging.getLogger(__name__) +class TTSBatchSubmissionError(Exception): + """Raised when no batch-mode model could be submitted to its provider.""" + + def start_tts_evaluation_batch( *, session: Session, run: EvaluationRun, results: list[TTSResult], + models: list[str], org_id: int, project_id: int, ) -> dict[str, Any]: - """Submit Gemini batch jobs for TTS evaluation. + """Submit Gemini batch jobs for the batch-mode models of a TTS evaluation. Submits one batch job per model. Each batch job is tracked via its config containing evaluation_run_id and tts_provider. @@ -45,7 +50,8 @@ def start_tts_evaluation_batch( Args: session: Database session run: The evaluation run record - results: List of TTSResult records (contains sample_text) + results: TTSResult records (loaded); rows for non-batch models are ignored + models: Batch-mode models to submit (sync models must not be passed) org_id: Organization ID project_id: Project ID @@ -53,47 +59,50 @@ def start_tts_evaluation_batch( dict: Result with batch job information per model Raises: - Exception: If batch submission fails for all models + ValueError: If a non-batch model is passed + TTSBatchSubmissionError: If batch submission fails for every model """ - models = run.providers or [DEFAULT_TTS_MODEL] + for model in models: + if get_tts_model_spec(model).execution_mode != TTSExecutionModeEnum.BATCH: + raise ValueError(f"Model '{model}' is not a batch-mode TTS model") + + # Capture plain values up front: start_batch_job commits, which expires every + # ORM instance in the session, and touching them later would issue a reload + # per row (and reopen a transaction right before the next network call). + run_id = run.id + texts_by_model: dict[str, list[str]] = {} + keys_by_model: dict[str, list[str]] = {} + for result in results: + texts_by_model.setdefault(result.provider, []).append(result.sample_text) + keys_by_model.setdefault(result.provider, []).append(str(result.id)) logger.info( f"[start_tts_evaluation_batch] Starting batch submission | " - f"run_id: {run.id}, result_count: {len(results)}, " + f"run_id: {run_id}, result_count: {len(results)}, " f"models: {models}" ) - # Initialize Gemini client gemini_client = GeminiClient.from_credentials( session=session, org_id=org_id, project_id=project_id, ) - # Collect unique sample texts and their result IDs per model - # Group results by model to build per-model batch requests - results_by_model: dict[str, list[TTSResult]] = {} - for result in results: - results_by_model.setdefault(result.provider, []).append(result) - - # Submit one batch job per model batch_jobs: dict[str, Any] = {} first_batch_job_id: int | None = None + submitted_count = 0 for model in models: - model_results = results_by_model.get(model, []) - if not model_results: + texts = texts_by_model.get(model, []) + if not texts: continue - texts = [r.sample_text for r in model_results] - keys = [str(r.id) for r in model_results] - - # Create JSONL batch requests for TTS + voice_name = get_tts_model_spec(model).default_voice jsonl_data = create_tts_batch_requests( texts=texts, - voice_name=DEFAULT_VOICE_NAME, + voice_name=voice_name, style_prompt=DEFAULT_STYLE_PROMPT, - keys=keys, + keys=keys_by_model[model], ) model_path = f"models/{model}" @@ -105,7 +114,7 @@ def start_tts_evaluation_batch( batch_job = start_batch_job( session=session, provider=batch_provider, - provider_name="google-aistudio", + provider_name=get_tts_model_spec(model).provider.value, job_type=BatchJobType.TTS_EVALUATION, organization_id=org_id, project_id=project_id, @@ -113,8 +122,8 @@ def start_tts_evaluation_batch( config={ "model": model, "tts_provider": model, - "evaluation_run_id": run.id, - "voice_name": DEFAULT_VOICE_NAME, + "evaluation_run_id": run_id, + "voice_name": voice_name, "style_prompt": DEFAULT_STYLE_PROMPT, }, ) @@ -123,23 +132,25 @@ def start_tts_evaluation_batch( "batch_job_id": batch_job.id, "provider_batch_id": batch_job.provider_batch_id, } + submitted_count += len(texts) if first_batch_job_id is None: first_batch_job_id = batch_job.id logger.info( f"[start_tts_evaluation_batch] Batch job created | " - f"run_id: {run.id}, model: {model}, " + f"run_id: {run_id}, model: {model}, " f"batch_job_id: {batch_job.id}" ) except Exception as e: logger.error( f"[start_tts_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}" + f"run_id: {run_id}, model: {model}, error: {str(e)}", + exc_info=True, ) pending = get_pending_results_for_run( - session=session, run_id=run.id, provider=model + session=session, run_id=run_id, provider=model ) for result in pending: update_tts_result( @@ -151,25 +162,25 @@ def start_tts_evaluation_batch( session.commit() if not batch_jobs: - raise Exception("Batch submission failed for all models") + raise TTSBatchSubmissionError("Batch submission failed for all models") # Link first batch job to the evaluation run (for pending run detection) update_tts_run( session=session, - run_id=run.id, + run_id=run_id, status="processing", batch_job_id=first_batch_job_id, ) logger.info( f"[start_tts_evaluation_batch] Batch submission complete | " - f"run_id: {run.id}, models_submitted: {list(batch_jobs.keys())}, " - f"result_count: {len(results)}" + f"run_id: {run_id}, models_submitted: {list(batch_jobs.keys())}, " + f"result_count: {submitted_count}" ) return { "success": True, - "run_id": run.id, + "run_id": run_id, "batch_jobs": batch_jobs, - "result_count": len(results), + "result_count": submitted_count, } diff --git a/backend/app/crud/tts_evaluations/cron.py b/backend/app/crud/tts_evaluations/cron.py index 2f3c67aaf..5bd988064 100644 --- a/backend/app/crud/tts_evaluations/cron.py +++ b/backend/app/crud/tts_evaluations/cron.py @@ -21,14 +21,10 @@ poll_all_pending_evaluations_by_type, poll_batch_jobs, ) -from app.crud.tts_evaluations.result import ( - count_results_by_status, - get_pending_results_for_run, -) -from app.crud.tts_evaluations.run import update_tts_run +from app.crud.tts_evaluations.result import get_pending_results_for_run +from app.crud.tts_evaluations.run import finalize_tts_run_status, update_tts_run from app.models import EvaluationRun from app.models.batch_job import BatchJob, BatchJobType -from app.models.job import JobStatus from app.models.stt_evaluation import EvaluationType logger = logging.getLogger(__name__) @@ -187,23 +183,12 @@ async def _on_already_succeeded(batch_job: BatchJob, provider_name: str) -> bool ) # All batch jobs are done and no dispatching needed - finalize the run - status_counts = count_results_by_status(session=session, run_id=run.id) - pending = status_counts.get(JobStatus.PENDING.value, 0) - failed_count = status_counts.get(JobStatus.FAILED.value, 0) - - final_status = "completed" if pending == 0 else "processing" - error_message = None - if result.any_failed: - error_message = "; ".join(result.errors) - elif failed_count > 0: - error_message = f"{failed_count} synthesis(es) failed" - - update_tts_run( - session=session, - run_id=run.id, - status=final_status, - error_message=error_message, + batch_error = "; ".join(result.errors) if result.any_failed else None + finalized_run = finalize_tts_run_status( + session=session, run_id=run.id, error_message=batch_error ) + final_status = finalized_run.status if finalized_run else run.status + error_message = finalized_run.error_message if finalized_run else batch_error action = "completed" if not result.any_failed else "failed" diff --git a/backend/app/crud/tts_evaluations/result.py b/backend/app/crud/tts_evaluations/result.py index 6ebf872ab..67a4b6d61 100644 --- a/backend/app/crud/tts_evaluations/result.py +++ b/backend/app/crud/tts_evaluations/result.py @@ -3,13 +3,13 @@ import logging from typing import Any -from sqlmodel import Session, func, select +from sqlmodel import Session, col, func, select from app.core.cloud.storage import CloudStorage from app.core.exception_handlers import HTTPException from app.core.util import now from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResult, TTSResultPublic +from app.models.tts_evaluation import TTSResult, TTSResultPublic, TTSResultUpdate logger = logging.getLogger(__name__) @@ -285,6 +285,83 @@ def get_pending_results_for_run( return list(session.exec(statement).all()) +def list_pending_tts_results_by_ids( + *, + session: Session, + run_id: int, + result_ids: list[int], +) -> list[TTSResult]: + """List the still-PENDING results of a run among the given IDs. + + Filtering on PENDING is what makes a redelivered sync chunk a no-op for rows + an earlier attempt already finished. + """ + if not result_ids: + return [] + + statement = ( + select(TTSResult) + .where( + TTSResult.evaluation_run_id == run_id, + TTSResult.status == JobStatus.PENDING.value, + col(TTSResult.id).in_(result_ids), + ) + .order_by(TTSResult.id) + ) + + return list(session.exec(statement).all()) + + +def bulk_update_pending_tts_results( + *, + session: Session, + updates: list[TTSResultUpdate], +) -> int: + """Apply terminal outcomes to results that are still PENDING; commit once. + + Rows already moved out of PENDING (by a concurrent or earlier attempt) are + left untouched so a late writer can never clobber a finished result. + + Returns: + int: Number of rows updated + """ + if not updates: + return 0 + + updates_by_id: dict[int, TTSResultUpdate] = {} + for update in updates: + updates_by_id[update.result_id] = update + + statement = select(TTSResult).where( + col(TTSResult.id).in_(list(updates_by_id.keys())), + TTSResult.status == JobStatus.PENDING.value, + ) + rows = session.exec(statement).all() + + timestamp = now() + for row in rows: + update = updates_by_id[row.id] + row.status = update.status.value + if update.object_store_url is not None: + row.object_store_url = update.object_store_url + if update.metadata is not None: + row.metadata_ = update.metadata + if update.error_message is not None: + row.error_message = update.error_message + row.updated_at = timestamp + session.add(row) + + session.commit() + + skipped = len(updates_by_id) - len(rows) + logger.info( + f"[bulk_update_pending_tts_results] Results updated | " + f"updated: {len(rows)}, skipped_not_pending: {skipped}" + ) + + return len(rows) + + def count_results_by_status( *, session: Session, diff --git a/backend/app/crud/tts_evaluations/run.py b/backend/app/crud/tts_evaluations/run.py index 881491d29..cd53e0575 100644 --- a/backend/app/crud/tts_evaluations/run.py +++ b/backend/app/crud/tts_evaluations/run.py @@ -6,7 +6,9 @@ from sqlmodel import Session, func, select from app.core.util import now +from app.crud.tts_evaluations.result import count_results_by_status from app.models import EvaluationRun +from app.models.job import JobStatus from app.models.stt_evaluation import EvaluationType from app.models.tts_evaluation import TTSEvaluationRunPublic @@ -219,3 +221,58 @@ def update_tts_run( ) return run + + +def finalize_tts_run_status( + *, + session: Session, + run_id: int, + error_message: str | None = None, +) -> EvaluationRun | None: + """Recompute a TTS run's status from its result rows and persist it. + + The run row is locked first so concurrent finalizers (sync chunks, Gemini + result processing, cron) serialize: whichever commits last counts every + result the others already committed, so a run can't be stranded in + `processing` after its final result lands. + + Args: + session: Database session + run_id: Run ID + error_message: Overrides the derived "N synthesis(es) failed" message + + Returns: + EvaluationRun | None: Updated run, or None if it doesn't exist + """ + statement = ( + select(EvaluationRun).where(EvaluationRun.id == run_id).with_for_update() + ) + run = session.exec(statement).one_or_none() + + if not run: + logger.warning(f"[finalize_tts_run_status] Run not found | run_id: {run_id}") + return None + + status_counts = count_results_by_status(session=session, run_id=run_id) + pending = status_counts.get(JobStatus.PENDING.value, 0) + failed = status_counts.get(JobStatus.FAILED.value, 0) + + run.status = "completed" if pending == 0 else "processing" + + if error_message is None and failed > 0: + error_message = f"{failed} synthesis(es) failed" + if error_message is not None: + run.error_message = error_message + + run.updated_at = now() + + session.add(run) + session.commit() + session.refresh(run) + + logger.info( + f"[finalize_tts_run_status] Run finalized | run_id: {run_id}, " + f"status: {run.status}, pending: {pending}, failed: {failed}" + ) + + return run diff --git a/backend/app/models/tts_evaluation.py b/backend/app/models/tts_evaluation.py index a0bbdb054..e3e3959e7 100644 --- a/backend/app/models/tts_evaluation.py +++ b/backend/app/models/tts_evaluation.py @@ -13,7 +13,10 @@ from app.core.util import now from app.models.job import JobStatus -from app.services.tts_evaluations.constants import SUPPORTED_TTS_MODELS +from app.services.tts_evaluations.constants import ( + DEFAULT_TTS_MODEL, + SUPPORTED_TTS_MODELS, +) if TYPE_CHECKING: from app.models import EvaluationDataset, EvaluationRun @@ -141,6 +144,16 @@ class TTSResult(SQLModel, table=True): ) +class TTSResultUpdate(BaseModel): + """Terminal outcome for one synthesis, applied only while the row is still PENDING.""" + + result_id: int + status: JobStatus + object_store_url: str | None = None + metadata: dict[str, Any] | None = None + error_message: str | None = None + + class TTSSampleCreate(BaseModel): """Request model for a single TTS sample.""" @@ -285,7 +298,7 @@ class TTSEvaluationRunCreate(BaseModel): run_name: str = Field(..., description="Name for this evaluation run", min_length=1) dataset_id: int = Field(..., description="ID of the TTS dataset to evaluate") models: list[str] = Field( - default_factory=lambda: ["gemini-2.5-pro-preview-tts"], + default_factory=lambda: [DEFAULT_TTS_MODEL], description="List of TTS models to use", min_length=1, ) diff --git a/backend/app/services/tts_evaluations/batch_job.py b/backend/app/services/tts_evaluations/batch_job.py index ad27d8d51..eebe87de4 100644 --- a/backend/app/services/tts_evaluations/batch_job.py +++ b/backend/app/services/tts_evaluations/batch_job.py @@ -1,35 +1,74 @@ -"""Celery task function for TTS evaluation batch submission.""" +"""Celery task function for TTS evaluation submission. + +Creates result rows, submits batch-mode models (Gemini) to the provider Batch API +and fans sync-mode models (Sarvam, ElevenLabs) out to chunked Celery tasks. +""" import logging +from dataclasses import dataclass +from typing import Any -from gevent import Timeout +from asgi_correlation_id import correlation_id +from celery import Task from celery.exceptions import SoftTimeLimitExceeded +from gevent import Timeout from sqlmodel import Session +from app.celery.utils import start_tts_sync_chunk +from app.core.cloud.storage import CloudStorage, get_cloud_storage from app.core.db import engine -from app.crud.tts_evaluations.batch import start_tts_evaluation_batch +from app.crud.tts_evaluations.batch import ( + TTSBatchSubmissionError, + start_tts_evaluation_batch, +) from app.crud.tts_evaluations.dataset import get_tts_dataset_by_id -from app.crud.tts_evaluations.result import create_tts_results -from app.crud.tts_evaluations.run import get_tts_run_by_id, update_tts_run -from app.services.tts_evaluations.dataset import get_sample_texts_from_dataset +from app.crud.tts_evaluations.result import ( + bulk_update_pending_tts_results, + create_tts_results, + get_pending_results_for_run, +) +from app.crud.tts_evaluations.run import ( + finalize_tts_run_status, + get_tts_run_by_id, + update_tts_run, +) +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult, TTSResultUpdate +from app.services.tts_evaluations.constants import ( + KAAPI_TAG, + TTS_SYNC_CHUNK_SIZE, + split_models_by_execution_mode, +) +from app.services.tts_evaluations.dataset import load_sample_texts +from app.services.tts_evaluations.language import resolve_dataset_tts_language_code logger = logging.getLogger(__name__) +DEFAULT_TRACE_ID = "N/A" + + +@dataclass(frozen=True) +class _SubmissionContext: + dataset_id: int + object_store_url: str | None + language_code: str + storage: CloudStorage | None + def execute_batch_submission( project_id: int, job_id: str, task_id: str, - task_instance, + task_instance: Task, organization_id: int, dataset_id: int, models: list[str], - **kwargs, -) -> dict: - """Execute TTS evaluation batch submission in a Celery worker. + **kwargs: Any, +) -> dict[str, Any]: + """Execute TTS evaluation submission in a Celery worker. - Handles result record creation, JSONL creation, Gemini file upload, - and batch job creation. + Handles result record creation, Gemini batch submission for batch-mode + models and chunked Celery fan-out for sync-mode models. Args: project_id: Project ID @@ -41,7 +80,7 @@ def execute_batch_submission( models: List of TTS model names to evaluate Returns: - dict: Result summary with batch job info + dict: Result summary with batch job and sync chunk info """ run_id = int(job_id) @@ -51,105 +90,287 @@ def execute_batch_submission( f"celery_task_id: {task_id}" ) - with Session(engine) as session: - try: + try: + batch_models, sync_models = split_models_by_execution_mode(models) + + context = _load_submission_context( + run_id=run_id, + organization_id=organization_id, + project_id=project_id, + dataset_id=dataset_id, + ) + if isinstance(context, dict): + return context + + # S3 read happens with no session open. + sample_texts: list[str] = [] + if context.object_store_url and context.storage is not None: + sample_texts = load_sample_texts( + storage=context.storage, + object_store_url=context.object_store_url, + dataset_id=context.dataset_id, + ) + else: + logger.warning( + f"[execute_batch_submission] No object_store_url | " + f"run_id: {run_id}, dataset_id: {dataset_id}" + ) + + if not sample_texts: + logger.warning( + f"[execute_batch_submission] No samples found | " + f"run_id: {run_id}, dataset_id: {dataset_id}" + ) + _mark_run_failed( + run_id=run_id, error_message="No samples found for dataset" + ) + return {"success": False, "error": "No samples found"} + + batch_result: dict[str, Any] = {} + sync_result_ids: dict[str, list[int]] = {} + result_count = 0 + + with Session(engine) as session: run = get_tts_run_by_id( session=session, run_id=run_id, org_id=organization_id, project_id=project_id, ) - if not run: logger.warning( f"[execute_batch_submission] Run not found | run_id: {run_id}" ) return {"success": False, "error": "Run not found"} - dataset = get_tts_dataset_by_id( + create_tts_results( session=session, - dataset_id=dataset_id, + sample_texts=sample_texts, + evaluation_run_id=run_id, org_id=organization_id, project_id=project_id, + models=models, ) + # One SELECT reloads every row create_tts_results' commit expired, + # instead of one lazy reload per row below. + results = get_pending_results_for_run(session=session, run_id=run_id) + result_count = len(results) - if not dataset: - logger.warning( - f"[execute_batch_submission] Dataset not found | " - f"run_id: {run_id}, dataset_id: {dataset_id}" - ) - update_tts_run( - session=session, - run_id=run_id, - status="failed", - error_message="Dataset not found", - ) - return {"success": False, "error": "Dataset not found"} + batch_results: list[TTSResult] = [] + for result in results: + if result.provider in batch_models: + batch_results.append(result) + elif result.provider in sync_models: + sync_result_ids.setdefault(result.provider, []).append(result.id) - sample_texts = get_sample_texts_from_dataset(session, dataset, project_id) + if batch_models: + try: + batch_result = start_tts_evaluation_batch( + session=session, + run=run, + results=batch_results, + models=batch_models, + org_id=organization_id, + project_id=project_id, + ) + except TTSBatchSubmissionError as e: + # Batch rows are already FAILED; sync models can still finish the run. + if not sync_models: + raise + logger.warning( + f"[execute_batch_submission] Batch models failed, " + f"continuing with sync models | run_id: {run_id}, " + f"error: {str(e)}" + ) - if not sample_texts: - logger.warning( - f"[execute_batch_submission] No samples found | " - f"run_id: {run_id}, dataset_id: {dataset_id}" - ) - update_tts_run( - session=session, - run_id=run_id, - status="failed", - error_message="No samples found for dataset", - ) - return {"success": False, "error": "No samples found"} + if sync_models: + # Must precede the enqueue: a fast chunk finalizing the run would + # otherwise be overwritten back to "processing" and strand it. + update_tts_run(session=session, run_id=run_id, status="processing") - # Create result records for each sample text and model - results = create_tts_results( - session=session, - sample_texts=sample_texts, - evaluation_run_id=run.id, - org_id=organization_id, + sync_chunks: dict[str, list[str]] = {} + if sync_result_ids: + sync_chunks = _dispatch_sync_chunks( + run_id=run_id, + organization_id=organization_id, project_id=project_id, - models=models, + result_ids_by_model=sync_result_ids, + language_code=context.language_code, ) - batch_result = start_tts_evaluation_batch( - session=session, - run=run, - results=results, - org_id=organization_id, - project_id=project_id, - ) + logger.info( + f"[execute_batch_submission] Submitted | run_id: {run_id}, " + f"batch_jobs: {list(batch_result.get('batch_jobs', {}).keys())}, " + f"sync_models: {list(sync_chunks.keys())}" + ) - logger.info( - f"[execute_batch_submission] Batch submitted | " - f"run_id: {run_id}, " - f"batch_jobs: {list(batch_result.get('batch_jobs', {}).keys())}" - ) + return { + "success": True, + "run_id": run_id, + "batch_jobs": batch_result.get("batch_jobs", {}), + "sync_chunks": sync_chunks, + "result_count": result_count, + } - return batch_result + except (Timeout, SoftTimeLimitExceeded): + timeout_err = TimeoutError("Task exceeded soft time limit") + logger.warning( + f"[execute_batch_submission] TTS batch submission timed out | run_id={run_id}" + ) + _mark_run_failed(run_id=run_id, error_message=str(timeout_err)) + raise - except (Timeout, SoftTimeLimitExceeded) as err: - timeout_err = TimeoutError("Task exceeded soft time limit") + except Exception as e: + logger.error( + f"[execute_batch_submission] Batch submission failed | " + f"run_id: {run_id}, error: {str(e)}", + exc_info=True, + ) + _mark_run_failed(run_id=run_id, error_message=str(e)) + return {"success": False, "error": str(e)} + + +def _load_submission_context( + *, + run_id: int, + organization_id: int, + project_id: int, + dataset_id: int, +) -> _SubmissionContext | dict[str, Any]: + """Validate run + dataset and capture what the session-free steps need. + + Returns a failure dict (already persisted on the run) when validation fails. + """ + with Session(engine) as session: + run = get_tts_run_by_id( + session=session, + run_id=run_id, + org_id=organization_id, + project_id=project_id, + ) + if not run: logger.warning( - f"[execute_batch_submission] TTS batch submission timed out | run_id={run_id}" + f"[_load_submission_context] Run not found | run_id: {run_id}" ) - update_tts_run( - session=session, - run_id=run_id, - status="failed", - error_message=str(timeout_err), - ) - raise + return {"success": False, "error": "Run not found"} - except Exception as e: - logger.error( - f"[execute_batch_submission] Batch submission failed | " - f"run_id: {run_id}, error: {str(e)}", - exc_info=True, + dataset = get_tts_dataset_by_id( + session=session, + dataset_id=dataset_id, + org_id=organization_id, + project_id=project_id, + ) + if not dataset: + logger.warning( + f"[_load_submission_context] Dataset not found | " + f"run_id: {run_id}, dataset_id: {dataset_id}" ) update_tts_run( session=session, run_id=run_id, status="failed", - error_message=str(e), + error_message="Dataset not found", ) - return {"success": False, "error": str(e)} + return {"success": False, "error": "Dataset not found"} + + object_store_url = dataset.object_store_url + language_code = resolve_dataset_tts_language_code( + session=session, language_id=dataset.language_id + ) + storage = ( + get_cloud_storage(session=session, project_id=project_id) + if object_store_url + else None + ) + + logger.info( + f"[_load_submission_context] Context loaded | run_id: {run_id}, " + f"dataset_id: {dataset_id}, language: {language_code}" + ) + + return _SubmissionContext( + dataset_id=dataset_id, + object_store_url=object_store_url, + language_code=language_code, + storage=storage, + ) + + +def _dispatch_sync_chunks( + *, + run_id: int, + organization_id: int, + project_id: int, + result_ids_by_model: dict[str, list[int]], + language_code: str, +) -> dict[str, list[str]]: + """Enqueue one `run_tts_sync_chunk` per (model, slice of result IDs). + + A chunk that can't be enqueued has its results marked FAILED, since no + other path would ever synthesize them. + + Returns: + dict[str, list[str]]: Celery task IDs per model + """ + trace_id = correlation_id.get() or DEFAULT_TRACE_ID + task_ids_by_model: dict[str, list[str]] = {} + failed_updates: list[TTSResultUpdate] = [] + + for model, result_ids in result_ids_by_model.items(): + task_ids: list[str] = [] + for start in range(0, len(result_ids), TTS_SYNC_CHUNK_SIZE): + chunk_ids = result_ids[start : start + TTS_SYNC_CHUNK_SIZE] + try: + celery_task_id = start_tts_sync_chunk( + project_id=project_id, + job_id=str(run_id), + trace_id=trace_id, + organization_id=organization_id, + model=model, + result_ids=chunk_ids, + language_code=language_code, + ) + except Exception as e: + logger.error( + f"[_dispatch_sync_chunks] Failed to enqueue chunk | " + f"run_id: {run_id}, model: {model}, " + f"chunk_size: {len(chunk_ids)}, error: {str(e)}", + exc_info=True, + ) + for result_id in chunk_ids: + failed_updates.append( + TTSResultUpdate( + result_id=result_id, + status=JobStatus.FAILED, + error_message=( + f"[{KAAPI_TAG}] Failed to queue synthesis for {model}: " + f"{str(e)}. Re-run the evaluation; contact Kaapi " + f"if it persists." + ), + ) + ) + continue + task_ids.append(celery_task_id) + + task_ids_by_model[model] = task_ids + logger.info( + f"[_dispatch_sync_chunks] Model dispatched | run_id: {run_id}, " + f"model: {model}, results: {len(result_ids)}, chunks: {len(task_ids)}" + ) + + if failed_updates: + with Session(engine) as session: + bulk_update_pending_tts_results(session=session, updates=failed_updates) + finalize_tts_run_status(session=session, run_id=run_id) + + return task_ids_by_model + + +def _mark_run_failed(*, run_id: int, error_message: str) -> None: + with Session(engine) as session: + update_tts_run( + session=session, + run_id=run_id, + status="failed", + error_message=error_message, + ) diff --git a/backend/app/services/tts_evaluations/batch_result_processing.py b/backend/app/services/tts_evaluations/batch_result_processing.py index a2dc3c208..d45c63e76 100644 --- a/backend/app/services/tts_evaluations/batch_result_processing.py +++ b/backend/app/services/tts_evaluations/batch_result_processing.py @@ -2,6 +2,9 @@ Processes completed Gemini TTS batch results: downloads JSONL, extracts audio, converts PCM to WAV, uploads to S3, updates DB. + +The DB session is only opened for short prefetch/write/finalize steps; the +download, audio conversion and S3 uploads run with no connection checked out. """ import base64 @@ -9,39 +12,51 @@ import uuid from typing import Any -from gevent import Timeout +from celery import Task from celery.exceptions import SoftTimeLimitExceeded -from sqlmodel import Session, select +from gevent import Timeout +from sqlmodel import Session +from app.core.audio_utils import calculate_duration, pcm_to_wav from app.core.batch import BATCH_KEY, GeminiBatchProvider, GeminiClient -from app.core.cloud.storage import get_cloud_storage +from app.core.cloud.storage import CloudStorage, get_cloud_storage from app.core.db import engine from app.core.storage_utils import upload_to_object_store -from app.core.util import now -from app.crud.tts_evaluations.result import count_results_by_status -from app.crud.tts_evaluations.run import update_tts_run +from app.crud.tts_evaluations.result import ( + bulk_update_pending_tts_results, + get_pending_results_for_run, +) +from app.crud.tts_evaluations.run import finalize_tts_run_status, update_tts_run from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResult -from app.core.audio_utils import calculate_duration, pcm_to_wav +from app.models.tts_evaluation import TTSResultUpdate +from app.services.tts_evaluations.constants import ( + TTS_AUDIO_CONTENT_TYPE, + TTS_AUDIO_FILE_EXTENSION, + TTS_AUDIO_SUBDIRECTORY, + TTS_RESULT_WRITE_CHUNK_SIZE, +) logger = logging.getLogger(__name__) +DURATION_DECIMALS = 3 + def execute_tts_result_processing( project_id: int, job_id: str, task_id: str, - task_instance: Any, + task_instance: Task, organization_id: int, evaluation_run_id: int, tts_provider: str, provider_batch_id: str, **kwargs: Any, -) -> dict: +) -> dict[str, Any]: """Process completed TTS batch results in a Celery worker. Downloads batch results from Gemini, extracts audio, converts to WAV, - uploads to S3, and updates TTSResult records. + uploads to S3, and updates TTSResult records. Only results still PENDING + are processed, so reprocessing a batch skips rows already finished. Args: project_id: Project ID @@ -62,211 +77,207 @@ def execute_tts_result_processing( f"provider={tts_provider}, celery_task_id={task_id}" ) - with Session(engine) as session: - try: - # Initialize Gemini client and batch provider + try: + with Session(engine) as session: gemini_client = GeminiClient.from_credentials( session=session, org_id=organization_id, project_id=project_id, ) - batch_provider = GeminiBatchProvider(client=gemini_client.client) - - # Get cloud storage for S3 uploads storage = get_cloud_storage(session=session, project_id=project_id) - - # Download batch results - results = batch_provider.download_batch_results(provider_batch_id) - + pending_rows = get_pending_results_for_run( + session=session, run_id=evaluation_run_id, provider=tts_provider + ) + pending_ids: set[int] = set() + for row in pending_rows: + pending_ids.add(row.id) + + logger.info( + f"[execute_tts_result_processing] Pre-fetched pending result ids | " + f"run_id={evaluation_run_id}, provider={tts_provider}, " + f"count={len(pending_ids)}" + ) + + batch_provider = GeminiBatchProvider(client=gemini_client.client) + results = batch_provider.download_batch_results(provider_batch_id) + + logger.info( + f"[execute_tts_result_processing] Got batch results | " + f"run_id={evaluation_run_id}, result_count={len(results)}" + ) + + if results: + first = results[0] + resp = first.get("response") or {} + resp_keys = ( + list(resp.keys()) if isinstance(resp, dict) else type(resp).__name__ + ) logger.info( - f"[execute_tts_result_processing] Got batch results | " - f"run_id={evaluation_run_id}, result_count={len(results)}" + f"[execute_tts_result_processing] First result structure | " + f"keys={list(first.keys())}, response_keys={resp_keys}" ) - if results: - first = results[0] - resp = first.get("response") or {} - resp_keys = ( - list(resp.keys()) if isinstance(resp, dict) else type(resp).__name__ + processed_count = 0 + failed_count = 0 + skipped_count = 0 + pending_updates: list[TTSResultUpdate] = [] + + for batch_result in results: + custom_id = batch_result[BATCH_KEY] + try: + result_id = int(custom_id) + except (ValueError, TypeError): + logger.warning( + f"[execute_tts_result_processing] Invalid {BATCH_KEY} | " + f"run_id={evaluation_run_id}, {BATCH_KEY}={custom_id}" ) + failed_count += 1 + continue + + if result_id not in pending_ids: logger.info( - f"[execute_tts_result_processing] First result structure | " - f"keys={list(first.keys())}, response_keys={resp_keys}" + f"[execute_tts_result_processing] Result not pending, skipping | " + f"result_id={result_id}" ) + skipped_count += 1 + continue - # Pre-fetch all TTSResult records for this run+provider - prefetch_stmt = select(TTSResult).where( - TTSResult.evaluation_run_id == evaluation_run_id, - TTSResult.provider == tts_provider, - ) - result_map: dict[int, TTSResult] = { - r.id: r for r in session.exec(prefetch_stmt).all() # type: ignore[misc] - } - - logger.info( - f"[execute_tts_result_processing] Pre-fetched result records | " - f"run_id={evaluation_run_id}, provider={tts_provider}, " - f"count={len(result_map)}" + update = _build_result_update( + result_id=result_id, batch_result=batch_result, storage=storage ) - - processed_count = 0 - failed_count = 0 - - for batch_result in results: - custom_id = batch_result[BATCH_KEY] - try: - result_id = int(custom_id) - except (ValueError, TypeError): - logger.warning( - f"[execute_tts_result_processing] Invalid {BATCH_KEY} | " - f"run_id={evaluation_run_id}, {BATCH_KEY}={custom_id}" - ) - failed_count += 1 - continue - - # Look up result record from pre-fetched map - result_record = result_map.get(result_id) - - if not result_record: - logger.warning( - f"[execute_tts_result_processing] Result record not found | " - f"result_id={result_id}" - ) - failed_count += 1 - continue - - if batch_result.get("response"): - try: - audio_b64 = _extract_audio_from_response( - batch_result["response"] - ) - - if not audio_b64: - result_record.status = JobStatus.FAILED.value - result_record.error_message = "No audio data in response" - result_record.updated_at = now() - failed_count += 1 - continue - - # Decode base64 -> raw PCM bytes - pcm_data = base64.b64decode(audio_b64) - - # Wrap in WAV container - wav_data = pcm_to_wav(pcm_data) - - # Calculate duration - duration = calculate_duration(len(pcm_data)) - - # Upload WAV to S3 - audio_filename = f"{uuid.uuid4()}.wav" - audio_url = upload_to_object_store( - storage=storage, - content=wav_data, - filename=audio_filename, - subdirectory="evaluations/tts/audio", - content_type="audio/wav", - ) - - if not audio_url: - result_record.status = JobStatus.FAILED.value - result_record.error_message = ( - "Audio upload to object store failed" - ) - result_record.updated_at = now() - failed_count += 1 - continue - - # Update result - result_record.object_store_url = audio_url - result_record.metadata_ = { - "duration_seconds": round(duration, 3), - "size_bytes": len(wav_data), - } - result_record.status = JobStatus.SUCCESS.value - result_record.updated_at = now() - processed_count += 1 - - except Exception as audio_err: - logger.warning( - f"[execute_tts_result_processing] Audio processing failed | " - f"result_id={result_id}, error={str(audio_err)}" - ) - result_record.status = JobStatus.FAILED.value - result_record.error_message = ( - f"Audio processing failed: {str(audio_err)}" - ) - result_record.updated_at = now() - failed_count += 1 - else: - result_record.status = JobStatus.FAILED.value - result_record.error_message = batch_result.get( - "error", "Unknown error" - ) - result_record.updated_at = now() - failed_count += 1 - - session.commit() - - # Finalize run status - status_counts = count_results_by_status( - session=session, run_id=evaluation_run_id + if update.status == JobStatus.SUCCESS: + processed_count += 1 + else: + failed_count += 1 + + pending_updates.append(update) + if len(pending_updates) >= TTS_RESULT_WRITE_CHUNK_SIZE: + _write_result_updates(pending_updates) + pending_updates = [] + + _write_result_updates(pending_updates) + + with Session(engine) as session: + run = finalize_tts_run_status(session=session, run_id=evaluation_run_id) + final_status = run.status if run else None + + logger.info( + f"[execute_tts_result_processing] Completed | " + f"run_id={evaluation_run_id}, provider={tts_provider}, " + f"processed={processed_count}, failed={failed_count}, " + f"skipped={skipped_count}, run_status={final_status}" + ) + + return { + "success": True, + "run_id": evaluation_run_id, + "processed": processed_count, + "failed": failed_count, + "run_status": final_status, + } + + except (Timeout, SoftTimeLimitExceeded): + timeout_err = TimeoutError("Task exceeded soft time limit") + logger.warning( + f"[execute_tts_result_processing] TTS result processing timed out | run_id={evaluation_run_id}" + ) + _mark_run_failed(run_id=evaluation_run_id, error_message=str(timeout_err)) + raise + + except Exception as e: + logger.error( + f"[execute_tts_result_processing] Failed | " + f"run_id={evaluation_run_id}, error={str(e)}", + exc_info=True, + ) + _mark_run_failed( + run_id=evaluation_run_id, + error_message=f"Result processing failed: {str(e)}", + ) + return {"success": False, "error": str(e)} + + +def _build_result_update( + *, + result_id: int, + batch_result: dict[str, Any], + storage: CloudStorage, +) -> TTSResultUpdate: + """Convert one Gemini batch result into a terminal update (uploads the WAV).""" + if not batch_result.get("response"): + return TTSResultUpdate( + result_id=result_id, + status=JobStatus.FAILED, + error_message=batch_result.get("error", "Unknown error"), + ) + + try: + audio_b64 = _extract_audio_from_response(batch_result["response"]) + + if not audio_b64: + return TTSResultUpdate( + result_id=result_id, + status=JobStatus.FAILED, + error_message="No audio data in response", ) - pending = status_counts.get(JobStatus.PENDING.value, 0) - total_failed = status_counts.get(JobStatus.FAILED.value, 0) - final_status = "completed" if pending == 0 else "processing" - error_message = ( - f"{total_failed} synthesis(es) failed" if total_failed > 0 else None + # Without validate=True, garbage input decodes to b"" and would be stored as a 0s "success". + pcm_data = base64.b64decode(audio_b64, validate=True) + wav_data = pcm_to_wav(pcm_data) + duration = calculate_duration(len(pcm_data)) + + audio_url = upload_to_object_store( + storage=storage, + content=wav_data, + filename=f"{uuid.uuid4()}.{TTS_AUDIO_FILE_EXTENSION}", + subdirectory=TTS_AUDIO_SUBDIRECTORY, + content_type=TTS_AUDIO_CONTENT_TYPE, + ) + + if not audio_url: + return TTSResultUpdate( + result_id=result_id, + status=JobStatus.FAILED, + error_message="Audio upload to object store failed", ) - update_tts_run( - session=session, - run_id=evaluation_run_id, - status=final_status, - error_message=error_message, - ) + return TTSResultUpdate( + result_id=result_id, + status=JobStatus.SUCCESS, + object_store_url=audio_url, + metadata={ + "duration_seconds": round(duration, DURATION_DECIMALS), + "size_bytes": len(wav_data), + }, + ) + + except Exception as audio_err: + logger.warning( + f"[_build_result_update] Audio processing failed | " + f"result_id={result_id}, error={str(audio_err)}" + ) + return TTSResultUpdate( + result_id=result_id, + status=JobStatus.FAILED, + error_message=f"Audio processing failed: {str(audio_err)}", + ) + + +def _write_result_updates(updates: list[TTSResultUpdate]) -> None: + if not updates: + return + with Session(engine) as session: + bulk_update_pending_tts_results(session=session, updates=updates) - logger.info( - f"[execute_tts_result_processing] Completed | " - f"run_id={evaluation_run_id}, provider={tts_provider}, " - f"processed={processed_count}, failed={failed_count}, " - f"run_status={final_status}" - ) - return { - "success": True, - "run_id": evaluation_run_id, - "processed": processed_count, - "failed": failed_count, - "run_status": final_status, - } - - except (Timeout, SoftTimeLimitExceeded) as err: - timeout_err = TimeoutError("Task exceeded soft time limit") - logger.warning( - f"[execute_tts_result_processing] TTS result processing timed out | run_id={evaluation_run_id}" - ) - update_tts_run( - session=session, - run_id=evaluation_run_id, - status="failed", - error_message=str(timeout_err), - ) - raise - - except Exception as e: - logger.error( - f"[execute_tts_result_processing] Failed | " - f"run_id={evaluation_run_id}, error={str(e)}", - exc_info=True, - ) - update_tts_run( - session=session, - run_id=evaluation_run_id, - status="failed", - error_message=f"Result processing failed: {str(e)}", - ) - return {"success": False, "error": str(e)} +def _mark_run_failed(*, run_id: int, error_message: str) -> None: + with Session(engine) as session: + update_tts_run( + session=session, + run_id=run_id, + status="failed", + error_message=error_message, + ) def _extract_audio_from_response(response: dict[str, Any]) -> str | None: diff --git a/backend/app/services/tts_evaluations/constants.py b/backend/app/services/tts_evaluations/constants.py index 169b9ff1e..92e720da9 100644 --- a/backend/app/services/tts_evaluations/constants.py +++ b/backend/app/services/tts_evaluations/constants.py @@ -1,13 +1,124 @@ -"""Shared constants for TTS evaluation services.""" +"""Shared constants for TTS evaluation services. -# Supported TTS models for evaluation -SUPPORTED_TTS_MODELS = ["gemini-2.5-pro-preview-tts"] +Kept free of `app.models` imports: `app.models.tts_evaluation` imports this module +for its request validator, so importing models here would create a cycle. +""" -# Default TTS model -DEFAULT_TTS_MODEL = "gemini-2.5-pro-preview-tts" +from dataclasses import dataclass +from enum import StrEnum + +from app.core.providers import Provider + + +class TTSExecutionModeEnum(StrEnum): + """How a TTS model's synthesis is executed. + + BATCH goes through a provider Batch API polled by the cron; SYNC is called + directly from chunked Celery tasks. + """ + + BATCH = "batch" + SYNC = "sync" -# Default voice configuration + +@dataclass(frozen=True) +class TTSModelSpec: + provider: Provider + execution_mode: TTSExecutionModeEnum + default_voice: str + + +GEMINI_PRO_TTS_MODEL = "gemini-2.5-pro-preview-tts" +SARVAM_BULBUL_V3_MODEL = "bulbul:v3" +ELEVENLABS_V3_MODEL = "eleven_v3" +ELEVENLABS_V4_MODEL = "eleven_v4" + +# Default voice configuration (Gemini prebuilt voice) DEFAULT_VOICE_NAME = "Kore" +SARVAM_DEFAULT_SPEAKER = "shubh" +# "Rahul S - Hindi Conversational": closest match to Sarvam's male speaker so the +# providers are roughly comparable in a side-by-side review. +ELEVENLABS_DEFAULT_VOICE_ID = "2cdvnKJ5TZi631y5PN1s" + +TTS_MODEL_REGISTRY: dict[str, TTSModelSpec] = { + GEMINI_PRO_TTS_MODEL: TTSModelSpec( + provider=Provider.GOOGLE_AISTUDIO, + execution_mode=TTSExecutionModeEnum.BATCH, + default_voice=DEFAULT_VOICE_NAME, + ), + SARVAM_BULBUL_V3_MODEL: TTSModelSpec( + provider=Provider.SARVAMAI, + execution_mode=TTSExecutionModeEnum.SYNC, + default_voice=SARVAM_DEFAULT_SPEAKER, + ), + ELEVENLABS_V3_MODEL: TTSModelSpec( + provider=Provider.ELEVENLABS, + execution_mode=TTSExecutionModeEnum.SYNC, + default_voice=ELEVENLABS_DEFAULT_VOICE_ID, + ), + ELEVENLABS_V4_MODEL: TTSModelSpec( + provider=Provider.ELEVENLABS, + execution_mode=TTSExecutionModeEnum.SYNC, + default_voice=ELEVENLABS_DEFAULT_VOICE_ID, + ), +} + +# Supported TTS models for evaluation +SUPPORTED_TTS_MODELS = list(TTS_MODEL_REGISTRY.keys()) + +# Default TTS model +DEFAULT_TTS_MODEL = GEMINI_PRO_TTS_MODEL # Default style prompt for TTS synthesis DEFAULT_STYLE_PROMPT = "Read in a calm, professional customer service tone" + +# BCP-47 tag used when the dataset has no language configured. +DEFAULT_TTS_LANGUAGE_CODE = "en-IN" + +# Every stored evaluation clip is normalized to this format so providers compare 1:1. +TTS_SAMPLE_RATE_HZ = 24000 +TTS_SAMPLE_WIDTH_BYTES = 2 +TTS_CHANNELS = 1 + +TTS_AUDIO_SUBDIRECTORY = "evaluations/tts/audio" +TTS_AUDIO_CONTENT_TYPE = "audio/wav" +TTS_AUDIO_FILE_EXTENSION = "wav" + +SARVAM_PACE = 1.0 +ELEVENLABS_OUTPUT_FORMAT = f"pcm_{TTS_SAMPLE_RATE_HZ}" + +# 25 items x 4 workers keeps a chunk well inside CELERY_TASK_SOFT_TIME_LIMIT +# even with retries on slow (~10s) ElevenLabs v3 calls. +TTS_SYNC_CHUNK_SIZE = 25 +TTS_SYNC_MAX_WORKERS = 4 +TTS_SYNC_REQUEST_TIMEOUT_SECONDS = 60 +TTS_SYNC_MAX_ATTEMPTS = 3 +TTS_SYNC_BACKOFF_BASE_SECONDS = 1.0 +TTS_SYNC_BACKOFF_MAX_SECONDS = 20.0 +TTS_RETRYABLE_STATUS_CODES = frozenset({408, 409, 429, 500, 502, 503, 504}) + +# Source tag for failures that are Kaapi-side (see error-handling convention). +KAAPI_TAG = "KAAPI" + +# Commit Gemini batch-result writes in slices so a timeout keeps earlier progress. +TTS_RESULT_WRITE_CHUNK_SIZE = 50 + + +def get_tts_model_spec(model: str) -> TTSModelSpec: + """Return the registry spec for a model; raises KeyError for unknown models.""" + return TTS_MODEL_REGISTRY[model] + + +def split_models_by_execution_mode( + models: list[str], +) -> tuple[list[str], list[str]]: + """Split requested models into (batch_models, sync_models), preserving order.""" + batch_models: list[str] = [] + sync_models: list[str] = [] + for model in models: + spec = get_tts_model_spec(model) + if spec.execution_mode == TTSExecutionModeEnum.BATCH: + batch_models.append(model) + else: + sync_models.append(model) + return batch_models, sync_models diff --git a/backend/app/services/tts_evaluations/dataset.py b/backend/app/services/tts_evaluations/dataset.py index 9db8aae5b..13c5c2cd0 100644 --- a/backend/app/services/tts_evaluations/dataset.py +++ b/backend/app/services/tts_evaluations/dataset.py @@ -8,6 +8,7 @@ from sqlmodel import Session from app.core.cloud import get_cloud_storage +from app.core.cloud.storage import CloudStorage from app.core.storage_utils import ( generate_timestamped_filename, upload_to_object_store, @@ -186,16 +187,48 @@ def get_sample_texts_from_dataset( try: storage = get_cloud_storage(session=session, project_id=project_id) - csv_bytes = storage.stream(dataset.object_store_url).read() + except Exception as e: + logger.error( + f"[get_sample_texts_from_dataset] Failed to init storage | " + f"dataset_id={dataset.id}, error={str(e)}", + exc_info=True, + ) + return [] + + return load_sample_texts( + storage=storage, + object_store_url=dataset.object_store_url, + dataset_id=dataset.id, + ) + + +def load_sample_texts( + *, + storage: CloudStorage, + object_store_url: str, + dataset_id: int, +) -> list[str]: + """Read a TTS dataset CSV from object storage; needs no DB session. + + Returns: + List of text strings (empty on read/parse failure) + """ + try: + csv_bytes = storage.stream(object_store_url).read() samples = parse_tts_samples_from_csv(csv_bytes) - return [s["text"] for s in samples] except Exception as e: logger.error( - f"[get_sample_texts_from_dataset] Failed to load CSV | " - f"dataset_id={dataset.id}, error={str(e)}" + f"[load_sample_texts] Failed to load CSV | " + f"dataset_id={dataset_id}, error={str(e)}", + exc_info=True, ) return [] + texts: list[str] = [] + for sample in samples: + texts.append(sample["text"]) + return texts + def parse_tts_samples_from_csv(csv_content: bytes) -> list[dict[str, Any]]: """Parse TTS samples from CSV content. diff --git a/backend/app/services/tts_evaluations/language.py b/backend/app/services/tts_evaluations/language.py new file mode 100644 index 000000000..336358629 --- /dev/null +++ b/backend/app/services/tts_evaluations/language.py @@ -0,0 +1,61 @@ +"""Language-code resolution for TTS evaluation providers. + +Datasets store an ISO 639-1 locale (`Language.locale`, e.g. "hi"); the canonical +code carried through the TTS pipeline is a BCP-47 tag (e.g. "hi-IN"), which is +what Sarvam takes directly and what ElevenLabs' ISO code is derived from. +""" + +import logging + +from sqlmodel import Session + +from app.crud.language import get_language_locale_by_id +from app.models.llm.constants import BCP47_TO_ELEVENLABS_LANG +from app.services.tts_evaluations.constants import DEFAULT_TTS_LANGUAGE_CODE + +logger = logging.getLogger(__name__) + +# Sarvam spells Odia "od-IN" while ISO 639-1 is "or"; it is absent from the +# ElevenLabs map, so it can't be derived from the reverse lookup. +_ODIA_ISO_CODE = "or" +_ODIA_BCP47_CODE = "od-IN" + +ISO_TO_BCP47_LANG: dict[str, str] = { + iso_code: bcp47_code for bcp47_code, iso_code in BCP47_TO_ELEVENLABS_LANG.items() +} +ISO_TO_BCP47_LANG[_ODIA_ISO_CODE] = _ODIA_BCP47_CODE + + +def resolve_tts_language_code(locale: str | None) -> str: + """Map an ISO 639-1 locale to the pipeline's BCP-47 tag, defaulting to en-IN.""" + if not locale: + return DEFAULT_TTS_LANGUAGE_CODE + + language_code = ISO_TO_BCP47_LANG.get(locale.lower()) + if language_code is None: + logger.warning( + f"[resolve_tts_language_code] Unmapped locale, using default | " + f"locale: {locale}, default: {DEFAULT_TTS_LANGUAGE_CODE}" + ) + return DEFAULT_TTS_LANGUAGE_CODE + + return language_code + + +def resolve_dataset_tts_language_code(session: Session, language_id: int | None) -> str: + """Resolve a dataset's `language_id` to a BCP-47 tag (en-IN when unset/unknown).""" + if language_id is None: + return DEFAULT_TTS_LANGUAGE_CODE + + locale = get_language_locale_by_id(session=session, language_id=language_id) + if locale is None: + logger.warning( + f"[resolve_dataset_tts_language_code] Language not found, using default | " + f"language_id: {language_id}, default: {DEFAULT_TTS_LANGUAGE_CODE}" + ) + return resolve_tts_language_code(locale) + + +def to_elevenlabs_language_code(language_code: str) -> str | None: + """ElevenLabs ISO 639-1 code, or None to let it auto-detect (e.g. Odia).""" + return BCP47_TO_ELEVENLABS_LANG.get(language_code) diff --git a/backend/app/services/tts_evaluations/sync_generation.py b/backend/app/services/tts_evaluations/sync_generation.py new file mode 100644 index 000000000..b4af6db77 --- /dev/null +++ b/backend/app/services/tts_evaluations/sync_generation.py @@ -0,0 +1,449 @@ +"""Celery task function for synchronous (non-batch) TTS evaluation synthesis. + +One task handles one model and a slice of that model's result IDs. The DB session +is only held for short reads/writes; provider calls and S3 uploads run with no +session so a slow provider can't pin pooled connections. +""" + +import logging +import uuid +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from typing import Any + +from celery import Task +from celery.exceptions import SoftTimeLimitExceeded +from gevent import Timeout +from sqlmodel import Session + +from app.core.cloud.storage import CloudStorage, get_cloud_storage +from app.core.db import engine +from app.core.providers import Provider +from app.core.storage_utils import upload_to_object_store +from app.crud.credentials import get_provider_credential +from app.crud.tts_evaluations.result import ( + bulk_update_pending_tts_results, + list_pending_tts_results_by_ids, +) +from app.crud.tts_evaluations.run import finalize_tts_run_status +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResultUpdate +from app.services.tts_evaluations.constants import ( + KAAPI_TAG, + TTS_AUDIO_CONTENT_TYPE, + TTS_AUDIO_FILE_EXTENSION, + TTS_AUDIO_SUBDIRECTORY, + TTS_SYNC_MAX_WORKERS, + get_tts_model_spec, +) +from app.services.tts_evaluations.synthesizers import ( + TTSClient, + TTSSynthesisError, + create_tts_client, + synthesize_tts, +) + +logger = logging.getLogger(__name__) + +DURATION_DECIMALS = 3 + + +@dataclass(frozen=True) +class _PendingItem: + result_id: int + text: str + + +@dataclass(frozen=True) +class _ChunkContext: + items: list[_PendingItem] + provider: Provider + voice: str + client: TTSClient | None + storage: CloudStorage | None + setup_error: str | None + + +def execute_tts_sync_chunk( + project_id: int, + job_id: str, + task_id: str, + task_instance: Task, + organization_id: int, + model: str, + result_ids: list[int], + language_code: str, + **kwargs: Any, +) -> dict[str, Any]: + """Synthesize one chunk of sync-model TTS results in a Celery worker. + + Args: + project_id: Project ID + job_id: Evaluation run ID (as string) + task_id: Celery task ID + task_instance: Celery task instance + organization_id: Organization ID + model: Sync-mode TTS model (e.g. bulbul:v3, eleven_v3) + result_ids: TTSResult IDs in this chunk (all for `model`) + language_code: BCP-47 tag resolved from the dataset (e.g. hi-IN) + + Returns: + dict: Summary with processed/failed counts and the run status + """ + run_id = int(job_id) + + logger.info( + f"[execute_tts_sync_chunk] Starting | run_id={run_id}, model={model}, " + f"chunk_size={len(result_ids)}, language={language_code}, " + f"celery_task_id={task_id}" + ) + + try: + context = _prepare_chunk( + run_id=run_id, + organization_id=organization_id, + project_id=project_id, + model=model, + result_ids=result_ids, + ) + + if not context.items: + logger.info( + f"[execute_tts_sync_chunk] No pending results, skipping | " + f"run_id={run_id}, model={model}" + ) + run_status = _finalize_run(run_id) + return { + "success": True, + "run_id": run_id, + "processed": 0, + "failed": 0, + "run_status": run_status, + } + + if context.setup_error is not None: + failed_count = _fail_pending_results( + run_id=run_id, + result_ids=result_ids, + error_message=context.setup_error, + ) + run_status = _finalize_run(run_id) + return { + "success": False, + "run_id": run_id, + "processed": 0, + "failed": failed_count, + "run_status": run_status, + "error": context.setup_error, + } + + processed_count, failed_count = _synthesize_chunk( + context=context, + run_id=run_id, + model=model, + language_code=language_code, + ) + + run_status = _finalize_run(run_id) + + logger.info( + f"[execute_tts_sync_chunk] Completed | run_id={run_id}, model={model}, " + f"processed={processed_count}, failed={failed_count}, " + f"run_status={run_status}" + ) + + return { + "success": True, + "run_id": run_id, + "processed": processed_count, + "failed": failed_count, + "run_status": run_status, + } + + except (Timeout, SoftTimeLimitExceeded): + logger.warning( + f"[execute_tts_sync_chunk] Chunk timed out | run_id={run_id}, model={model}" + ) + # Nothing re-drives a sync chunk, so leftover PENDING rows would strand the run. + _fail_pending_results( + run_id=run_id, + result_ids=result_ids, + error_message=( + f"[{KAAPI_TAG}] Synthesis timed out before this sample was processed. " + f"Re-run the evaluation; if it persists, contact Kaapi." + ), + ) + _finalize_run(run_id) + raise + + except Exception as e: + logger.error( + f"[execute_tts_sync_chunk] Failed | run_id={run_id}, model={model}, " + f"error={str(e)}", + exc_info=True, + ) + _fail_pending_results( + run_id=run_id, + result_ids=result_ids, + error_message=( + f"[{KAAPI_TAG}] Synthesis chunk failed unexpectedly: {str(e)}. " + f"Contact Kaapi if the issue persists." + ), + ) + run_status = _finalize_run(run_id) + return { + "success": False, + "run_id": run_id, + "error": str(e), + "run_status": run_status, + } + + +def _prepare_chunk( + *, + run_id: int, + organization_id: int, + project_id: int, + model: str, + result_ids: list[int], +) -> _ChunkContext: + """Load pending items, credentials, client and storage in one short session.""" + spec = get_tts_model_spec(model) + + with Session(engine) as session: + pending_rows = list_pending_tts_results_by_ids( + session=session, run_id=run_id, result_ids=result_ids + ) + items: list[_PendingItem] = [] + for row in pending_rows: + items.append(_PendingItem(result_id=row.id, text=row.sample_text)) + + if not items: + return _ChunkContext( + items=items, + provider=spec.provider, + voice=spec.default_voice, + client=None, + storage=None, + setup_error=None, + ) + + credentials = get_provider_credential( + session=session, + org_id=organization_id, + project_id=project_id, + provider=spec.provider.value, + ) + storage = get_cloud_storage(session=session, project_id=project_id) + + if not isinstance(credentials, dict): + setup_error = ( + f"[{KAAPI_TAG}] {spec.provider.value} credentials are not configured " + f"for this project. Add them under credentials and re-run the evaluation." + ) + logger.warning( + f"[_prepare_chunk] {setup_error} | run_id={run_id}, model={model}" + ) + return _ChunkContext( + items=items, + provider=spec.provider, + voice=spec.default_voice, + client=None, + storage=storage, + setup_error=setup_error, + ) + + try: + client = create_tts_client(spec.provider, credentials) + except ValueError as e: + setup_error = ( + f"[{KAAPI_TAG}] Invalid {spec.provider.value} credentials: {str(e)}. " + f"Verify the stored API key and re-run the evaluation." + ) + logger.warning( + f"[_prepare_chunk] {setup_error} | run_id={run_id}, model={model}", + exc_info=True, + ) + return _ChunkContext( + items=items, + provider=spec.provider, + voice=spec.default_voice, + client=None, + storage=storage, + setup_error=setup_error, + ) + + return _ChunkContext( + items=items, + provider=spec.provider, + voice=spec.default_voice, + client=client, + storage=storage, + setup_error=None, + ) + + +def _synthesize_chunk( + *, + context: _ChunkContext, + run_id: int, + model: str, + language_code: str, +) -> tuple[int, int]: + """Fan synthesis out over a bounded pool; persist each outcome as it lands. + + Returns: + tuple[int, int]: (processed_count, failed_count) actually written + """ + if context.client is None or context.storage is None: + raise ValueError("Chunk context is missing its client or storage") + + client = context.client + storage = context.storage + processed_count = 0 + failed_count = 0 + workers = max(1, min(TTS_SYNC_MAX_WORKERS, len(context.items))) + + # Managed by hand rather than `with`: on a gevent timeout, `with` would block + # in shutdown(wait=True) until every in-flight provider call returned. + executor = ThreadPoolExecutor(max_workers=workers) + try: + futures: list[Future[TTSResultUpdate]] = [] + for item in context.items: + future = executor.submit( + _synthesize_and_upload, + client=client, + storage=storage, + provider=context.provider, + model=model, + voice=context.voice, + language_code=language_code, + item=item, + ) + futures.append(future) + + for future in as_completed(futures): + update = future.result() + written = _write_result_updates([update]) + if not written: + continue + if update.status == JobStatus.SUCCESS: + processed_count += 1 + else: + failed_count += 1 + finally: + executor.shutdown(wait=False, cancel_futures=True) + + logger.info( + f"[_synthesize_chunk] Chunk synthesized | run_id={run_id}, model={model}, " + f"processed={processed_count}, failed={failed_count}" + ) + return processed_count, failed_count + + +def _synthesize_and_upload( + *, + client: TTSClient, + storage: CloudStorage, + provider: Provider, + model: str, + voice: str, + language_code: str, + item: _PendingItem, +) -> TTSResultUpdate: + """Synthesize + upload one item. Never raises: failures become a FAILED update.""" + try: + audio = synthesize_tts( + client=client, + provider=provider, + text=item.text, + model=model, + language_code=language_code, + voice=voice, + ) + except TTSSynthesisError as e: + return TTSResultUpdate( + result_id=item.result_id, + status=JobStatus.FAILED, + error_message=e.message, + ) + except Exception as e: + # A worker exception would abort the whole chunk via future.result(). + error_message = ( + f"[{KAAPI_TAG}] Unexpected error during {provider.value} synthesis: " + f"{str(e)}. Contact Kaapi if the issue persists." + ) + logger.error( + f"[_synthesize_and_upload] {error_message} | " + f"result_id={item.result_id}, model={model}", + exc_info=True, + ) + return TTSResultUpdate( + result_id=item.result_id, + status=JobStatus.FAILED, + error_message=error_message, + ) + + audio_url = upload_to_object_store( + storage=storage, + content=audio.wav_bytes, + filename=f"{uuid.uuid4()}.{TTS_AUDIO_FILE_EXTENSION}", + subdirectory=TTS_AUDIO_SUBDIRECTORY, + content_type=TTS_AUDIO_CONTENT_TYPE, + ) + + if not audio_url: + error_message = ( + f"[{KAAPI_TAG}] Audio upload to object store failed. Check the " + f"project's storage configuration; contact Kaapi if it persists." + ) + logger.error( + f"[_synthesize_and_upload] {error_message} | " + f"result_id={item.result_id}, model={model}" + ) + return TTSResultUpdate( + result_id=item.result_id, + status=JobStatus.FAILED, + error_message=error_message, + ) + + return TTSResultUpdate( + result_id=item.result_id, + status=JobStatus.SUCCESS, + object_store_url=audio_url, + metadata={ + "duration_seconds": round(audio.duration_seconds, DURATION_DECIMALS), + "size_bytes": len(audio.wav_bytes), + }, + ) + + +def _write_result_updates(updates: list[TTSResultUpdate]) -> int: + with Session(engine) as session: + return bulk_update_pending_tts_results(session=session, updates=updates) + + +def _fail_pending_results( + *, run_id: int, result_ids: list[int], error_message: str +) -> int: + """Mark every still-PENDING result of this chunk FAILED in a fresh session.""" + with Session(engine) as session: + pending_rows = list_pending_tts_results_by_ids( + session=session, run_id=run_id, result_ids=result_ids + ) + updates: list[TTSResultUpdate] = [] + for row in pending_rows: + updates.append( + TTSResultUpdate( + result_id=row.id, + status=JobStatus.FAILED, + error_message=error_message, + ) + ) + return bulk_update_pending_tts_results(session=session, updates=updates) + + +def _finalize_run(run_id: int) -> str | None: + with Session(engine) as session: + run = finalize_tts_run_status(session=session, run_id=run_id) + return run.status if run else None diff --git a/backend/app/services/tts_evaluations/synthesizers.py b/backend/app/services/tts_evaluations/synthesizers.py new file mode 100644 index 000000000..2bcfee7b2 --- /dev/null +++ b/backend/app/services/tts_evaluations/synthesizers.py @@ -0,0 +1,403 @@ +"""Synchronous TTS synthesizers for evaluation runs (Sarvam, ElevenLabs). + +Every synthesizer returns 24 kHz / 16-bit / mono WAV so clips from different +providers are stored and reviewed in one format. SDK and network failures are +mapped to `TTSSynthesisError`, carrying a source-tagged message and whether the +failure is transient (429 / 5xx / network), which drives the retry policy. +""" + +import base64 +import io +import logging +import wave +from dataclasses import dataclass +from typing import Any, cast + +import httpx +from elevenlabs import ElevenLabs +from elevenlabs.core.api_error import ApiError as ElevenLabsApiError +from elevenlabs.core.request_options import ( + RequestOptions as ElevenLabsRequestOptions, +) +from pydub import AudioSegment +from sarvamai import SarvamAI +from sarvamai.core.api_error import ApiError as SarvamApiError +from sarvamai.core.request_options import RequestOptions as SarvamRequestOptions +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception, + stop_after_attempt, + wait_random_exponential, +) + +from app.core.audio_utils import calculate_duration, pcm_to_wav +from app.core.providers import Provider +from app.services.llm.providers.eleven_ai import ElevenlabsAIProvider +from app.services.llm.providers.sarvam_ai import SarvamAIProvider +from app.services.tts_evaluations.constants import ( + ELEVENLABS_OUTPUT_FORMAT, + KAAPI_TAG, + SARVAM_PACE, + TTS_CHANNELS, + TTS_RETRYABLE_STATUS_CODES, + TTS_SAMPLE_RATE_HZ, + TTS_SAMPLE_WIDTH_BYTES, + TTS_SYNC_BACKOFF_BASE_SECONDS, + TTS_SYNC_BACKOFF_MAX_SECONDS, + TTS_SYNC_MAX_ATTEMPTS, + TTS_SYNC_REQUEST_TIMEOUT_SECONDS, +) +from app.services.tts_evaluations.language import to_elevenlabs_language_code + +logger = logging.getLogger(__name__) + +TTSClient = SarvamAI | ElevenLabs + +SARVAM_TAG = "SARVAM" +ELEVENLABS_TAG = "ELEVENLABS" + +_BITS_PER_BYTE = 8 + +# Our tenacity policy owns retries; SDK-level retries would multiply attempts. +_SARVAM_REQUEST_OPTIONS = SarvamRequestOptions( + max_retries=0, timeout_in_seconds=TTS_SYNC_REQUEST_TIMEOUT_SECONDS +) +_ELEVENLABS_REQUEST_OPTIONS = ElevenLabsRequestOptions( + max_retries=0, timeout_in_seconds=TTS_SYNC_REQUEST_TIMEOUT_SECONDS +) + +_STATUS_HINTS: dict[int, str] = { + 400: "Review the input text, model, language and voice — the request shape or content may be invalid.", + 401: "Verify the API key is valid, not expired, and configured for this project.", + 403: "The API key lacks access to the requested model/voice — check plan and key scopes.", + 404: "Verify the model name and voice ID are correct and available on your plan.", + 408: "The provider timed out handling the request — retry in a few seconds.", + 409: "Request conflicts with current resource state — review concurrent requests before retrying.", + 413: "Text exceeds the provider's size limit — shorten the sample.", + 422: "Provider rejected the payload — check input format and parameter values against the API spec.", + 429: "Hit the provider's rate/quota — wait at least 1 min and retry. Request a quota increase or contact Kaapi if persistent.", + 500: "Typically transient — retry in a few seconds. If it persists, contact Kaapi.", + 502: "Provider gateway error — typically transient, retry in a few seconds.", + 503: "Provider temporarily down or overloaded — retry in a few seconds.", + 504: "Provider took too long — retry with a shorter sample.", +} +_DEFAULT_STATUS_HINT = "If this persists, contact Kaapi." + + +class TTSSynthesisError(Exception): + """A single synthesis failed; `message` is safe to store on the result row.""" + + def __init__(self, message: str, *, retryable: bool = False) -> None: + super().__init__(message) + self.message = message + self.retryable = retryable + + +@dataclass(frozen=True) +class SynthesizedAudio: + wav_bytes: bytes + duration_seconds: float + + +def create_tts_client(provider: Provider, credentials: dict[str, Any]) -> TTSClient: + """Build the SDK client for a sync TTS provider from stored credentials. + + Raises: + ValueError: If the credentials lack an API key or the provider isn't sync-capable + """ + if provider == Provider.SARVAMAI: + return SarvamAIProvider.create_client(credentials) + if provider == Provider.ELEVENLABS: + return ElevenlabsAIProvider.create_client(credentials) + raise ValueError(f"Provider '{provider.value}' has no synchronous TTS client") + + +def _api_error( + *, + tag: str, + status_code: int | None, + body: Any, + model: str, + function_name: str, +) -> TTSSynthesisError: + hint = _STATUS_HINTS.get(status_code or 0, _DEFAULT_STATUS_HINT) + message = f"[{tag}] TTS request failed (code: {status_code}): {body}. {hint}" + # 5xx is provider-side (alert-worthy); 4xx is caller's fault (noise if alerted) + log = logger.error if status_code and status_code >= 500 else logger.warning + log( + f"[{function_name}] {message} | provider={tag.lower()}, model={model}", + exc_info=True, + ) + return TTSSynthesisError( + message, retryable=status_code in TTS_RETRYABLE_STATUS_CODES + ) + + +def _network_error( + *, + err: httpx.TransportError, + tag: str, + model: str, + function_name: str, +) -> TTSSynthesisError: + code = type(err).__name__ + if isinstance(err, httpx.TimeoutException): + message = ( + f"[{KAAPI_TAG}] TTS request to {tag.title()} timed out (code: {code}). " + f"Retry with a shorter sample; if persistent, contact Kaapi." + ) + else: + message = ( + f"[{KAAPI_TAG}] Could not reach {tag.title()} (code: {code}): {err}. " + f"Network/DNS issue reaching the provider — check connectivity; if " + f"persistent, contact Kaapi." + ) + logger.error( + f"[{function_name}] {message} | provider={tag.lower()}, model={model}", + exc_info=True, + ) + return TTSSynthesisError(message, retryable=True) + + +def _normalize_pcm( + pcm_data: bytes, *, channels: int, sample_width: int, frame_rate: int +) -> bytes: + """Convert PCM to the evaluation format (24 kHz, 16-bit, mono) if it differs.""" + if ( + channels == TTS_CHANNELS + and sample_width == TTS_SAMPLE_WIDTH_BYTES + and frame_rate == TTS_SAMPLE_RATE_HZ + ): + return pcm_data + + segment = AudioSegment( + data=pcm_data, + sample_width=sample_width, + frame_rate=frame_rate, + channels=channels, + ) + segment = segment.set_channels(TTS_CHANNELS) + segment = segment.set_sample_width(TTS_SAMPLE_WIDTH_BYTES) + segment = segment.set_frame_rate(TTS_SAMPLE_RATE_HZ) + return bytes(segment.raw_data or b"") + + +def _wav_chunk_to_pcm(wav_bytes: bytes) -> bytes: + with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file: + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + frame_rate = wav_file.getframerate() + frames = wav_file.readframes(wav_file.getnframes()) + return _normalize_pcm( + frames, channels=channels, sample_width=sample_width, frame_rate=frame_rate + ) + + +def _to_synthesized_audio(pcm_data: bytes) -> SynthesizedAudio: + wav_bytes = pcm_to_wav( + pcm_data, + sample_rate=TTS_SAMPLE_RATE_HZ, + bits_per_sample=TTS_SAMPLE_WIDTH_BYTES * _BITS_PER_BYTE, + channels=TTS_CHANNELS, + ) + duration = calculate_duration( + len(pcm_data), + sample_rate=TTS_SAMPLE_RATE_HZ, + bits_per_sample=TTS_SAMPLE_WIDTH_BYTES * _BITS_PER_BYTE, + channels=TTS_CHANNELS, + ) + return SynthesizedAudio(wav_bytes=wav_bytes, duration_seconds=duration) + + +def synthesize_sarvam( + *, + client: SarvamAI, + text: str, + model: str, + language_code: str, + speaker: str, +) -> SynthesizedAudio: + """Synthesize one text with Sarvam and return it as normalized WAV. + + Raises: + TTSSynthesisError: On any provider, network, or decoding failure + """ + try: + response = client.text_to_speech.convert( + text=text, + model=model, + target_language_code=language_code, + speaker=speaker, + pace=SARVAM_PACE, + speech_sample_rate=TTS_SAMPLE_RATE_HZ, + request_options=_SARVAM_REQUEST_OPTIONS, + ) + except SarvamApiError as e: + raise _api_error( + tag=SARVAM_TAG, + status_code=e.status_code, + body=e.body, + model=model, + function_name="synthesize_sarvam", + ) from e + except httpx.TransportError as e: + raise _network_error( + err=e, tag=SARVAM_TAG, model=model, function_name="synthesize_sarvam" + ) from e + + if not response.audios: + message = ( + f"[{SARVAM_TAG}] TTS response contains no audio data. Sarvam accepted " + f"the request but returned an empty audio list — retry later; if it " + f"persists, contact Kaapi." + ) + logger.warning( + f"[synthesize_sarvam] {message} | provider=sarvam, model={model}, " + f"request_id={response.request_id}" + ) + raise TTSSynthesisError(message) + + # Long inputs come back as several base64 chunks, each a complete WAV file. + # Joining the base64 strings (or the decoded bytes) would leave the later + # chunks' RIFF headers inside the PCM stream as audible clicks, so each chunk + # is decoded and stripped separately and the PCM re-wrapped once. + pcm_parts: list[bytes] = [] + try: + for audio_b64 in response.audios: + pcm_parts.append(_wav_chunk_to_pcm(base64.b64decode(audio_b64))) + except (ValueError, wave.Error, EOFError) as e: + message = ( + f"[{KAAPI_TAG}] Failed to decode Sarvam audio (code: {type(e).__name__}): " + f"{e}. The response was not valid base64 WAV — contact Kaapi if this persists." + ) + logger.error( + f"[synthesize_sarvam] {message} | provider=sarvam, model={model}, " + f"request_id={response.request_id}", + exc_info=True, + ) + raise TTSSynthesisError(message) from e + + logger.info( + f"[synthesize_sarvam] Synthesized | model={model}, " + f"chunks={len(response.audios)}, request_id={response.request_id}" + ) + return _to_synthesized_audio(b"".join(pcm_parts)) + + +def synthesize_elevenlabs( + *, + client: ElevenLabs, + text: str, + model: str, + language_code: str, + voice_id: str, +) -> SynthesizedAudio: + """Synthesize one text with ElevenLabs (raw 24 kHz PCM) and return it as WAV. + + Raises: + TTSSynthesisError: On any provider or network failure + """ + convert_kwargs: dict[str, Any] = {} + elevenlabs_language = to_elevenlabs_language_code(language_code) + if elevenlabs_language: + convert_kwargs["language_code"] = elevenlabs_language + + # The SDK streams lazily, so HTTP errors can surface while joining the chunks. + try: + chunks = client.text_to_speech.convert( + voice_id=voice_id, + text=text, + model_id=model, + output_format=ELEVENLABS_OUTPUT_FORMAT, + request_options=_ELEVENLABS_REQUEST_OPTIONS, + **convert_kwargs, + ) + pcm_data = b"".join(chunks) + except ElevenLabsApiError as e: + raise _api_error( + tag=ELEVENLABS_TAG, + status_code=e.status_code, + body=e.body, + model=model, + function_name="synthesize_elevenlabs", + ) from e + except httpx.TransportError as e: + raise _network_error( + err=e, + tag=ELEVENLABS_TAG, + model=model, + function_name="synthesize_elevenlabs", + ) from e + + if not pcm_data: + message = ( + f"[{ELEVENLABS_TAG}] TTS response contains no audio data. ElevenLabs " + f"accepted the request but returned an empty audio stream — retry " + f"later; if it persists, contact Kaapi." + ) + logger.warning( + f"[synthesize_elevenlabs] {message} | provider=elevenlabs, " + f"model={model}, voice_id={voice_id}" + ) + raise TTSSynthesisError(message) + + logger.info( + f"[synthesize_elevenlabs] Synthesized | model={model}, " + f"voice_id={voice_id}, pcm_bytes={len(pcm_data)}" + ) + return _to_synthesized_audio(pcm_data) + + +def _is_retryable_synthesis_error(err: BaseException) -> bool: + return isinstance(err, TTSSynthesisError) and err.retryable + + +@retry( + retry=retry_if_exception(_is_retryable_synthesis_error), + wait=wait_random_exponential( + multiplier=TTS_SYNC_BACKOFF_BASE_SECONDS, max=TTS_SYNC_BACKOFF_MAX_SECONDS + ), + stop=stop_after_attempt(TTS_SYNC_MAX_ATTEMPTS), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, +) +def synthesize_tts( + *, + client: TTSClient, + provider: Provider, + text: str, + model: str, + language_code: str, + voice: str, +) -> SynthesizedAudio: + """Dispatch to the provider's synthesizer, retrying transient failures. + + Raises: + TTSSynthesisError: On a non-retryable failure or after the final attempt + """ + # Dispatch on the registry provider rather than isinstance so the client + # can be substituted (e.g. a mock) without matching the SDK class. + if provider == Provider.SARVAMAI: + return synthesize_sarvam( + client=cast(SarvamAI, client), + text=text, + model=model, + language_code=language_code, + speaker=voice, + ) + if provider == Provider.ELEVENLABS: + return synthesize_elevenlabs( + client=cast(ElevenLabs, client), + text=text, + model=model, + language_code=language_code, + voice_id=voice, + ) + + message = ( + f"[{KAAPI_TAG}] No synchronous synthesizer for provider " + f"'{provider.value}'. Contact Kaapi." + ) + logger.error(f"[synthesize_tts] {message} | model={model}") + raise TTSSynthesisError(message) diff --git a/backend/app/tests/api/routes/test_tts_evaluation.py b/backend/app/tests/api/routes/test_tts_evaluation.py index ae9ade1c8..1add26532 100644 --- a/backend/app/tests/api/routes/test_tts_evaluation.py +++ b/backend/app/tests/api/routes/test_tts_evaluation.py @@ -603,6 +603,40 @@ def test_start_evaluation_multiple_models_total_items( # 5 samples × 1 model assert data["total_items"] == 5 + @patch("app.api.routes.tts_evaluations.evaluation.start_tts_batch_submission") + def test_start_evaluation_mixed_models_reports_voice_per_model( + self, + mock_start_job: MagicMock, + client: TestClient, + user_api_key_header: dict[str, str], + test_dataset_with_samples: EvaluationDataset, + ) -> None: + mock_start_job.return_value = "mock-celery-task-id" + models = ["gemini-2.5-pro-preview-tts", "bulbul:v3", "eleven_v3"] + + response = client.post( + "/api/v1/evaluations/tts/runs", + json={ + "run_name": "mixed_models_run", + "dataset_id": test_dataset_with_samples.id, + "models": models, + }, + headers=user_api_key_header, + ) + + assert response.status_code == 200, response.text + data = response.json()["data"] + assert data["models"] == models + assert data["total_items"] == 9 + assert data["run_metadata"]["voices"] == { + "gemini-2.5-pro-preview-tts": "Kore", + "bulbul:v3": "shubh", + "eleven_v3": "2cdvnKJ5TZi631y5PN1s", + } + # legacy single-voice key kept for existing clients + assert data["run_metadata"]["voice_name"] == "Kore" + assert mock_start_job.call_args.kwargs["models"] == models + @patch("app.api.routes.tts_evaluations.evaluation.start_tts_batch_submission") def test_start_evaluation_celery_failure( self, diff --git a/backend/app/tests/celery/test_tts_tasks.py b/backend/app/tests/celery/test_tts_tasks.py new file mode 100644 index 000000000..d53affbf5 --- /dev/null +++ b/backend/app/tests/celery/test_tts_tasks.py @@ -0,0 +1,84 @@ +from typing import Any +from unittest.mock import MagicMock, patch + +from asgi_correlation_id import correlation_id + +from app.celery.tasks import job_execution +from app.celery.utils import start_tts_sync_chunk + +CHUNK_KWARGS: dict[str, Any] = { + "organization_id": 3, + "model": "bulbul:v3", + "result_ids": [11, 12], + "language_code": "hi-IN", +} + + +class TestStartTTSSyncChunk: + def test_enqueues_task_with_kwargs_and_trace_headers(self) -> None: + # apply_async is the broker boundary; nothing is published in tests. + with patch.object( + job_execution.run_tts_sync_chunk, + "apply_async", + return_value=MagicMock(id="celery-123"), + ) as apply_async: + task_id = start_tts_sync_chunk( + project_id=7, job_id="42", trace_id="trace-abc", **CHUNK_KWARGS + ) + + assert task_id == "celery-123" + call = apply_async.call_args.kwargs + assert call["kwargs"] == { + "project_id": 7, + "job_id": "42", + "trace_id": "trace-abc", + **CHUNK_KWARGS, + } + assert "otel" in call["headers"] + + def test_trace_id_defaults_to_na(self) -> None: + with patch.object( + job_execution.run_tts_sync_chunk, + "apply_async", + return_value=MagicMock(id="celery-123"), + ) as apply_async: + start_tts_sync_chunk(project_id=7, job_id="42", **CHUNK_KWARGS) + + assert apply_async.call_args.kwargs["kwargs"]["trace_id"] == "N/A" + + +class TestRunTTSSyncChunk: + def test_runs_chunk_with_task_context_and_trace(self) -> None: + seen: dict[str, Any] = {} + + def fake_execute(**kwargs: Any) -> dict[str, Any]: + seen.update(kwargs) + seen["correlation_id"] = correlation_id.get() + return {"success": True, "processed": 2} + + with patch( + "app.services.tts_evaluations.sync_generation.execute_tts_sync_chunk", + side_effect=fake_execute, + ): + result = job_execution.run_tts_sync_chunk.apply( + kwargs={ + "project_id": 7, + "job_id": "42", + "trace_id": "trace-abc", + **CHUNK_KWARGS, + }, + task_id="task-xyz", + ).get() + + assert result == {"success": True, "processed": 2} + assert seen["project_id"] == 7 + assert seen["job_id"] == "42" + assert seen["task_id"] == "task-xyz" + assert seen["task_instance"] is not None + assert seen["correlation_id"] == "trace-abc" + for key, value in CHUNK_KWARGS.items(): + assert seen[key] == value + + def test_task_is_on_default_queue_at_low_priority(self) -> None: + assert job_execution.run_tts_sync_chunk.queue == "default" + assert job_execution.run_tts_sync_chunk.priority == 2 diff --git a/backend/app/tests/core/batch/test_operations.py b/backend/app/tests/core/batch/test_operations.py new file mode 100644 index 000000000..013069fcf --- /dev/null +++ b/backend/app/tests/core/batch/test_operations.py @@ -0,0 +1,102 @@ +from collections.abc import Iterator +from typing import Any + +import pytest +from sqlmodel import Session, delete, select + +from app.core.batch.operations import start_batch_job +from app.core.db import engine +from app.models.batch_job import BatchJob, BatchJobType +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.utils import random_lower_string + + +class _RecordingProvider: + """Fake batch provider that snapshots DB state while the 'network' call runs.""" + + def __init__( + self, session: Session, marker: str, error: Exception | None = None + ) -> None: + self._session = session + self._marker = marker + self._error = error + self.in_transaction_during_call: bool | None = None + self.job_visible_to_other_connections: bool | None = None + + def create_batch( + self, jsonl_data: list[dict[str, Any]], config: dict[str, Any] + ) -> dict[str, Any]: + self.in_transaction_during_call = self._session.in_transaction() + with Session(engine) as other: + jobs = other.exec( + select(BatchJob).where(BatchJob.provider == self._marker) + ).all() + self.job_visible_to_other_connections = len(jobs) == 1 + if self._error is not None: + raise self._error + return { + "provider_batch_id": "batches/abc", + "provider_file_id": "files/abc", + "provider_status": "JOB_STATE_PENDING", + } + + +@pytest.fixture +def committed_session(user_api_key: TestAuthContext) -> Iterator[tuple[Session, str]]: + # The conftest `db` session always sits inside a savepoint, so it can never + # report "no open transaction"; this needs a real session, cleaned up by hand. + marker = f"test-provider-{random_lower_string()}" + session = Session(engine) + try: + yield session, marker + finally: + session.rollback() + session.exec(delete(BatchJob).where(BatchJob.provider == marker)) + session.commit() + session.close() + + +def _start( + session: Session, provider: _RecordingProvider, marker: str, auth: TestAuthContext +) -> BatchJob: + return start_batch_job( + session=session, + provider=provider, + provider_name=marker, + job_type=BatchJobType.TTS_EVALUATION.value, + organization_id=auth.organization_id, + project_id=auth.project_id, + jsonl_data=[{"key": "1"}, {"key": "2"}], + config={"model": "m"}, + ) + + +class TestStartBatchJob: + def test_no_transaction_is_held_during_provider_call( + self, committed_session: tuple[Session, str], user_api_key: TestAuthContext + ) -> None: + session, marker = committed_session + provider = _RecordingProvider(session, marker) + + job = _start(session, provider, marker, user_api_key) + + assert provider.in_transaction_during_call is False + assert provider.job_visible_to_other_connections is True + assert job.provider_batch_id == "batches/abc" + assert job.provider_status == "JOB_STATE_PENDING" + assert job.total_items == 2 + + def test_provider_failure_marks_job_failed_and_reraises( + self, committed_session: tuple[Session, str], user_api_key: TestAuthContext + ) -> None: + session, marker = committed_session + provider = _RecordingProvider( + session, marker, error=RuntimeError("upload refused") + ) + + with pytest.raises(RuntimeError, match="upload refused"): + _start(session, provider, marker, user_api_key) + + [job] = session.exec(select(BatchJob).where(BatchJob.provider == marker)).all() + assert job.provider_status == "failed" + assert job.error_message == "Batch creation failed: upload refused" diff --git a/backend/app/tests/crud/test_language.py b/backend/app/tests/crud/test_language.py new file mode 100644 index 000000000..0398563fe --- /dev/null +++ b/backend/app/tests/crud/test_language.py @@ -0,0 +1,17 @@ +from sqlmodel import Session + +from app.crud.language import get_language_by_locale, get_language_locale_by_id +from app.models.language import Language +from app.tests.utils.utils import get_non_existent_id + + +def test_get_language_locale_by_id_returns_seeded_locale(db: Session) -> None: + hindi = get_language_by_locale(session=db, locale="hi") + + assert get_language_locale_by_id(session=db, language_id=hindi.id) == "hi" + + +def test_get_language_locale_by_id_returns_none_when_missing(db: Session) -> None: + missing_id = get_non_existent_id(db, Language) + + assert get_language_locale_by_id(session=db, language_id=missing_id) is None diff --git a/backend/app/tests/crud/tts_evaluations/__init__.py b/backend/app/tests/crud/tts_evaluations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/app/tests/crud/tts_evaluations/test_batch.py b/backend/app/tests/crud/tts_evaluations/test_batch.py new file mode 100644 index 000000000..bc7c452f0 --- /dev/null +++ b/backend/app/tests/crud/tts_evaluations/test_batch.py @@ -0,0 +1,180 @@ +import pytest +from sqlmodel import Session, select + +from app.crud.tts_evaluations.batch import ( + TTSBatchSubmissionError, + start_tts_evaluation_batch, +) +from app.models import EvaluationRun +from app.models.batch_job import BatchJob, BatchJobType +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.constants import ( + TTS_MODEL_REGISTRY, + TTSExecutionModeEnum, + TTSModelSpec, +) +from app.core.providers import Provider +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, + fake_gemini_batch, +) + +MODULE = "app.crud.tts_evaluations.batch" +GEMINI = "gemini-2.5-pro-preview-tts" +# A second batch-mode model so partial (per-model) failure is observable. +SECOND_BATCH_MODEL = "gemini-test-batch-tts" + + +@pytest.fixture +def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + models=[GEMINI, "bulbul:v3"], + ) + + +@pytest.fixture +def second_batch_model(monkeypatch: pytest.MonkeyPatch) -> str: + monkeypatch.setitem( + TTS_MODEL_REGISTRY, + SECOND_BATCH_MODEL, + TTSModelSpec( + provider=Provider.GOOGLE_AISTUDIO, + execution_mode=TTSExecutionModeEnum.BATCH, + default_voice="Puck", + ), + ) + return SECOND_BATCH_MODEL + + +def _submit( + db: Session, run: EvaluationRun, results: list[TTSResult], models: list[str] +) -> dict: + return start_tts_evaluation_batch( + session=db, + run=run, + results=results, + models=models, + org_id=run.organization_id, + project_id=run.project_id, + ) + + +def _batch_jobs_for(db: Session, run_id: int) -> list[BatchJob]: + jobs = db.exec( + select(BatchJob).where(BatchJob.job_type == BatchJobType.TTS_EVALUATION.value) + ).all() + return [j for j in jobs if j.config.get("evaluation_run_id") == run_id] + + +class TestStartTTSEvaluationBatch: + def test_rejects_sync_model(self, db: Session, run: EvaluationRun) -> None: + with fake_gemini_batch(MODULE) as fake: + with pytest.raises(ValueError, match="'bulbul:v3' is not a batch-mode"): + _submit(db, run, [], [GEMINI, "bulbul:v3"]) + + assert fake.submitted == {} + + def test_submits_batch_rows_and_links_run( + self, db: Session, run: EvaluationRun + ) -> None: + gemini_rows = [create_test_tts_result_row(db, run=run) for _ in range(2)] + sync_row = create_test_tts_result_row(db, run=run, provider="bulbul:v3") + + with fake_gemini_batch(MODULE) as fake: + outcome = _submit(db, run, [*gemini_rows, sync_row], [GEMINI]) + + [job] = _batch_jobs_for(db, run.id) + assert outcome == { + "success": True, + "run_id": run.id, + "batch_jobs": { + GEMINI: { + "batch_job_id": job.id, + "provider_batch_id": f"batches/{GEMINI}", + } + }, + "result_count": 2, + } + assert [r["key"] for r in fake.submitted[f"models/{GEMINI}"]] == [ + str(r.id) for r in gemini_rows + ] + assert job.config["tts_provider"] == GEMINI + assert job.config["voice_name"] == "Kore" + assert job.provider == "google-aistudio" + db.refresh(run) + assert run.status == "processing" + assert run.batch_job_id == job.id + + def test_all_models_failing_raises_and_fails_rows( + self, db: Session, run: EvaluationRun + ) -> None: + row = create_test_tts_result_row(db, run=run) + sync_row = create_test_tts_result_row(db, run=run, provider="bulbul:v3") + + with fake_gemini_batch(MODULE, {f"models/{GEMINI}": RuntimeError("quota")}): + with pytest.raises(TTSBatchSubmissionError): + _submit(db, run, [row, sync_row], [GEMINI]) + + db.refresh(row) + db.refresh(sync_row) + db.refresh(run) + assert row.status == JobStatus.FAILED.value + assert row.error_message == f"Batch submission failed for {GEMINI}: quota" + assert sync_row.status == JobStatus.PENDING.value + assert run.batch_job_id is None + [job] = _batch_jobs_for(db, run.id) + assert job.provider_status == "failed" + + def test_model_without_rows_is_skipped( + self, db: Session, run: EvaluationRun + ) -> None: + with fake_gemini_batch(MODULE) as fake: + with pytest.raises(TTSBatchSubmissionError): + _submit(db, run, [], [GEMINI]) + + assert fake.submitted == {} + assert _batch_jobs_for(db, run.id) == [] + + def test_one_model_failing_only_fails_its_rows( + self, db: Session, run: EvaluationRun, second_batch_model: str + ) -> None: + failing_row = create_test_tts_result_row(db, run=run) + ok_row = create_test_tts_result_row(db, run=run, provider=second_batch_model) + + with fake_gemini_batch(MODULE, {f"models/{GEMINI}": RuntimeError("bad key")}): + outcome = _submit( + db, run, [failing_row, ok_row], [GEMINI, second_batch_model] + ) + + assert list(outcome["batch_jobs"]) == [second_batch_model] + assert outcome["result_count"] == 1 + db.refresh(failing_row) + db.refresh(ok_row) + db.refresh(run) + assert failing_row.status == JobStatus.FAILED.value + assert ok_row.status == JobStatus.PENDING.value + assert ( + run.batch_job_id + == outcome["batch_jobs"][second_batch_model]["batch_job_id"] + ) + + def test_run_links_first_submitted_job( + self, db: Session, run: EvaluationRun, second_batch_model: str + ) -> None: + rows = [ + create_test_tts_result_row(db, run=run), + create_test_tts_result_row(db, run=run, provider=second_batch_model), + ] + + with fake_gemini_batch(MODULE): + outcome = _submit(db, run, rows, [GEMINI, second_batch_model]) + + assert len(_batch_jobs_for(db, run.id)) == 2 + db.refresh(run) + assert run.batch_job_id == outcome["batch_jobs"][GEMINI]["batch_job_id"] diff --git a/backend/app/tests/crud/tts_evaluations/test_cron.py b/backend/app/tests/crud/tts_evaluations/test_cron.py new file mode 100644 index 000000000..48ac80498 --- /dev/null +++ b/backend/app/tests/crud/tts_evaluations/test_cron.py @@ -0,0 +1,191 @@ +from unittest.mock import MagicMock, patch + +import pytest +from sqlmodel import Session + +from app.core.batch import BatchJobState +from app.crud.tts_evaluations.cron import poll_tts_run +from app.models import EvaluationRun +from app.models.batch_job import BatchJob, BatchJobType +from app.models.job import JobStatus +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, +) + +GEMINI = "gemini-2.5-pro-preview-tts" + + +@pytest.fixture +def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: + run = create_test_tts_run_with_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + models=[GEMINI, "bulbul:v3"], + ) + run.status = "processing" + db.add(run) + db.commit() + return run + + +def _terminal_batch_job( + db: Session, run: EvaluationRun, state: BatchJobState, error: str | None = None +) -> BatchJob: + job = BatchJob( + provider="google-aistudio", + job_type=BatchJobType.TTS_EVALUATION.value, + config={"evaluation_run_id": run.id, "tts_provider": GEMINI}, + provider_batch_id="batches/1", + provider_status=state.value, + error_message=error, + total_items=1, + organization_id=run.organization_id, + project_id=run.project_id, + ) + db.add(job) + db.commit() + db.refresh(job) + return job + + +async def _poll(db: Session, run: EvaluationRun, expected_dispatches: int = 0) -> dict: + # Gemini is never reached: batch jobs are either terminal or polled via the + # patched poll_batch_status in the tests that need it. + with patch( + "app.crud.tts_evaluations.cron.start_tts_result_processing", + return_value="celery-task", + ) as dispatch: + outcome = await poll_tts_run( + session=db, run=run, batch_provider=MagicMock(), org_id=run.organization_id + ) + assert dispatch.call_count == expected_dispatches + if expected_dispatches: + assert dispatch.call_args.kwargs["evaluation_run_id"] == run.id + assert dispatch.call_args.kwargs["tts_provider"] == GEMINI + return outcome + + +def _provider_reports(db: Session, state: BatchJobState): + def fake_poll(*, session: Session, provider: object, batch_job: BatchJob) -> None: + batch_job.provider_status = state.value + session.add(batch_job) + session.commit() + + return patch( + "app.crud.evaluations.cron_utils.poll_batch_status", side_effect=fake_poll + ) + + +@pytest.mark.asyncio +class TestPollTTSRunFinalize: + async def test_succeeded_batch_with_sync_rows_pending_stays_processing( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + create_test_tts_result_row(db, run=run, provider="bulbul:v3") + + outcome = await _poll(db, run) + + assert outcome["action"] == "completed" + assert outcome["current_status"] == "processing" + db.refresh(run) + assert run.status == "processing" + + async def test_all_done_completes_with_failure_count( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + create_test_tts_result_row( + db, run=run, provider="bulbul:v3", status=JobStatus.FAILED + ) + + outcome = await _poll(db, run) + + assert outcome["current_status"] == "completed" + assert outcome["error"] == "1 synthesis(es) failed" + db.refresh(run) + assert run.status == "completed" + assert run.error_message == "1 synthesis(es) failed" + + async def test_failed_batch_error_takes_precedence( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.FAILED, error="quota exceeded") + create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) + + outcome = await _poll(db, run) + + assert outcome["action"] == "failed" + assert outcome["current_status"] == "completed" + assert outcome["error"] == f"{GEMINI}: quota exceeded" + db.refresh(run) + assert run.error_message == f"{GEMINI}: quota exceeded" + + async def test_run_vanishing_before_finalize_falls_back_to_poll_state( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.FAILED, error="expired") + + with patch( + "app.crud.tts_evaluations.cron.finalize_tts_run_status", return_value=None + ): + outcome = await _poll(db, run) + + assert outcome["current_status"] == "processing" + assert outcome["error"] == f"{GEMINI}: expired" + + +@pytest.mark.asyncio +class TestPollTTSRunDispatch: + async def test_no_batch_jobs_fails_run( + self, db: Session, run: EvaluationRun + ) -> None: + outcome = await _poll(db, run) + + assert (outcome["action"], outcome["error"]) == ( + "failed", + "No batch jobs found", + ) + db.refresh(run) + assert run.status == "failed" + + async def test_still_running_batch_is_no_change( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.RUNNING) + + with _provider_reports(db, BatchJobState.RUNNING): + outcome = await _poll(db, run) + + assert outcome["action"] == "no_change" + db.refresh(run) + assert run.status == "processing" + + async def test_newly_succeeded_batch_dispatches_processing( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.RUNNING) + create_test_tts_result_row(db, run=run) + + with _provider_reports(db, BatchJobState.SUCCEEDED): + outcome = await _poll(db, run, expected_dispatches=1) + + assert (outcome["action"], outcome["current_status"]) == ( + "dispatched", + "processing", + ) + + async def test_already_succeeded_with_pending_rows_redispatches( + self, db: Session, run: EvaluationRun + ) -> None: + _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) + create_test_tts_result_row(db, run=run) + + outcome = await _poll(db, run, expected_dispatches=1) + + assert outcome["action"] == "dispatched" diff --git a/backend/app/tests/crud/tts_evaluations/test_result.py b/backend/app/tests/crud/tts_evaluations/test_result.py new file mode 100644 index 000000000..f7f7cdb69 --- /dev/null +++ b/backend/app/tests/crud/tts_evaluations/test_result.py @@ -0,0 +1,120 @@ +from sqlmodel import Session + +from app.crud.tts_evaluations.result import ( + bulk_update_pending_tts_results, + list_pending_tts_results_by_ids, +) +from app.models import EvaluationRun +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResultUpdate +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, +) + + +def _make_run(db: Session, auth: TestAuthContext) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, organization_id=auth.organization_id, project_id=auth.project_id + ) + + +class TestListPendingTTSResultsByIds: + def test_returns_only_pending_rows_of_run_in_id_order( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + other_run = _make_run(db, user_api_key) + pending_a = create_test_tts_result_row(db, run=run) + done = create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + pending_b = create_test_tts_result_row(db, run=run) + not_requested = create_test_tts_result_row(db, run=run) + foreign = create_test_tts_result_row(db, run=other_run) + + rows = list_pending_tts_results_by_ids( + session=db, + run_id=run.id, + result_ids=[pending_b.id, done.id, pending_a.id, foreign.id], + ) + + assert [r.id for r in rows] == [pending_a.id, pending_b.id] + assert not_requested.id not in {r.id for r in rows} + + def test_empty_ids_returns_empty_list( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + create_test_tts_result_row(db, run=run) + + assert ( + list_pending_tts_results_by_ids(session=db, run_id=run.id, result_ids=[]) + == [] + ) + + +class TestBulkUpdatePendingTTSResults: + def test_applies_success_and_failure_fields( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + ok = create_test_tts_result_row(db, run=run) + bad = create_test_tts_result_row(db, run=run) + + updated = bulk_update_pending_tts_results( + session=db, + updates=[ + TTSResultUpdate( + result_id=ok.id, + status=JobStatus.SUCCESS, + object_store_url="s3://bucket/a.wav", + metadata={"duration_seconds": 1.5, "size_bytes": 72044}, + ), + TTSResultUpdate( + result_id=bad.id, + status=JobStatus.FAILED, + error_message="[SARVAM] boom", + ), + ], + ) + + assert updated == 2 + db.refresh(ok) + db.refresh(bad) + assert ok.status == JobStatus.SUCCESS.value + assert ok.object_store_url == "s3://bucket/a.wav" + assert ok.metadata_ == {"duration_seconds": 1.5, "size_bytes": 72044} + assert ok.error_message is None + assert bad.status == JobStatus.FAILED.value + assert bad.error_message == "[SARVAM] boom" + assert bad.object_store_url is None + + def test_rows_no_longer_pending_are_not_clobbered( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + finished = create_test_tts_result_row( + db, run=run, status=JobStatus.SUCCESS, object_store_url="s3://b/keep.wav" + ) + pending = create_test_tts_result_row(db, run=run) + + updated = bulk_update_pending_tts_results( + session=db, + updates=[ + TTSResultUpdate( + result_id=finished.id, status=JobStatus.FAILED, error_message="late" + ), + TTSResultUpdate( + result_id=pending.id, status=JobStatus.FAILED, error_message="late" + ), + ], + ) + + assert updated == 1 + db.refresh(finished) + assert finished.status == JobStatus.SUCCESS.value + assert finished.object_store_url == "s3://b/keep.wav" + assert finished.error_message is None + + def test_empty_updates_returns_zero(self, db: Session) -> None: + assert bulk_update_pending_tts_results(session=db, updates=[]) == 0 diff --git a/backend/app/tests/crud/tts_evaluations/test_run.py b/backend/app/tests/crud/tts_evaluations/test_run.py new file mode 100644 index 000000000..2b690aa76 --- /dev/null +++ b/backend/app/tests/crud/tts_evaluations/test_run.py @@ -0,0 +1,85 @@ +from sqlmodel import Session + +from app.crud.tts_evaluations.run import finalize_tts_run_status +from app.models import EvaluationRun +from app.models.job import JobStatus +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, +) +from app.tests.utils.utils import get_non_existent_id + + +def _make_run(db: Session, auth: TestAuthContext) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, organization_id=auth.organization_id, project_id=auth.project_id + ) + + +class TestFinalizeTTSRunStatus: + def test_all_success_completes_without_error( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + + finalized = finalize_tts_run_status(session=db, run_id=run.id) + + assert finalized is not None + assert finalized.status == "completed" + assert finalized.error_message is None + + def test_pending_rows_keep_run_processing( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + create_test_tts_result_row(db, run=run, status=JobStatus.PENDING) + + finalized = finalize_tts_run_status(session=db, run_id=run.id) + + assert finalized.status == "processing" + db.refresh(run) + assert run.status == "processing" + + def test_failed_rows_produce_count_message( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) + create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) + + finalized = finalize_tts_run_status(session=db, run_id=run.id) + + assert finalized.status == "completed" + assert finalized.error_message == "2 synthesis(es) failed" + + def test_explicit_error_message_wins_over_failed_count( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) + + finalized = finalize_tts_run_status( + session=db, run_id=run.id, error_message="gemini: JOB_STATE_FAILED" + ) + + assert finalized.error_message == "gemini: JOB_STATE_FAILED" + + def test_run_with_no_results_completes( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key) + + finalized = finalize_tts_run_status(session=db, run_id=run.id) + + assert finalized.status == "completed" + assert finalized.error_message is None + + def test_missing_run_returns_none(self, db: Session) -> None: + missing_id = get_non_existent_id(db, EvaluationRun) + + assert finalize_tts_run_status(session=db, run_id=missing_id) is None diff --git a/backend/app/tests/models/test_tts_evaluation.py b/backend/app/tests/models/test_tts_evaluation.py new file mode 100644 index 000000000..63cb9e679 --- /dev/null +++ b/backend/app/tests/models/test_tts_evaluation.py @@ -0,0 +1,52 @@ +import pytest +from pydantic import ValidationError + +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSEvaluationRunCreate, TTSResultUpdate + + +class TestTTSEvaluationRunCreate: + def test_accepts_sync_and_batch_models_together(self) -> None: + run = TTSEvaluationRunCreate( + run_name="mixed", + dataset_id=1, + models=[ + "gemini-2.5-pro-preview-tts", + "bulbul:v3", + "eleven_v3", + "eleven_v4", + ], + ) + + assert run.models == [ + "gemini-2.5-pro-preview-tts", + "bulbul:v3", + "eleven_v3", + "eleven_v4", + ] + + def test_defaults_to_gemini(self) -> None: + run = TTSEvaluationRunCreate(run_name="default", dataset_id=1) + + assert run.models == ["gemini-2.5-pro-preview-tts"] + + def test_rejects_unknown_model(self) -> None: + with pytest.raises( + ValidationError, match="Unsupported model\\(s\\): eleven_v9" + ): + TTSEvaluationRunCreate( + run_name="bad", dataset_id=1, models=["bulbul:v3", "eleven_v9"] + ) + + +class TestTTSResultUpdate: + def test_optional_fields_default_to_none(self) -> None: + update = TTSResultUpdate(result_id=7, status=JobStatus.FAILED) + + assert update.object_store_url is None + assert update.metadata is None + assert update.error_message is None + + def test_rejects_unknown_status(self) -> None: + with pytest.raises(ValidationError): + TTSResultUpdate(result_id=7, status="exploded") diff --git a/backend/app/tests/services/tts_evaluations/__init__.py b/backend/app/tests/services/tts_evaluations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/app/tests/services/tts_evaluations/test_batch_job.py b/backend/app/tests/services/tts_evaluations/test_batch_job.py new file mode 100644 index 000000000..c9c756fda --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_batch_job.py @@ -0,0 +1,357 @@ +import io +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from celery.exceptions import SoftTimeLimitExceeded +from gevent import Timeout +from sqlmodel import Session, select + +from app.crud.language import get_language_by_locale +from app.models import EvaluationRun +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.batch_job import execute_batch_submission +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + TEST_DATASET_URL, + create_test_tts_result_row, + create_test_tts_run_with_dataset, + fake_gemini_batch, + use_test_session, +) +from app.tests.utils.utils import get_non_existent_id + +MODULE = "app.services.tts_evaluations.batch_job" +GEMINI = "gemini-2.5-pro-preview-tts" +SARVAM = "bulbul:v3" +ELEVEN = "eleven_v3" + + +def _csv(texts: list[str]) -> bytes: + return ("text\n" + "\n".join(texts) + "\n").encode() + + +@dataclass +class Harness: + csv: bytes = field(default_factory=lambda: _csv(["one", "two"])) + enqueued: list[dict[str, Any]] = field(default_factory=list) + enqueue_failures: set[str] = field(default_factory=set) + on_stream: Any = None + + def stream(self, url: str) -> io.BytesIO: + assert url == TEST_DATASET_URL + if self.on_stream is not None: + self.on_stream() + return io.BytesIO(self.csv) + + def enqueue(self, **kwargs: Any) -> str: + if kwargs["model"] in self.enqueue_failures: + raise ConnectionError("broker unreachable") + self.enqueued.append(kwargs) + return f"task-{len(self.enqueued)}" + + +@pytest.fixture +def harness(db: Session) -> Iterator[Harness]: + h = Harness() + storage = MagicMock() + storage.stream.side_effect = h.stream + with ( + use_test_session(MODULE, db), + patch(f"{MODULE}.get_cloud_storage", return_value=storage), + patch(f"{MODULE}.start_tts_sync_chunk", side_effect=h.enqueue), + ): + yield h + + +def _run( + db: Session, auth: TestAuthContext, models: list[str], **kwargs: Any +) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, + organization_id=auth.organization_id, + project_id=auth.project_id, + models=models, + **kwargs, + ) + + +def _execute( + run: EvaluationRun, models: list[str], dataset_id: int | None = None +) -> dict[str, Any]: + return execute_batch_submission( + project_id=run.project_id, + job_id=str(run.id), + task_id="celery-task", + task_instance=MagicMock(), + organization_id=run.organization_id, + dataset_id=dataset_id if dataset_id is not None else run.dataset_id, + models=models, + ) + + +def _results( + db: Session, run: EvaluationRun, model: str | None = None +) -> list[TTSResult]: + stmt = select(TTSResult).where(TTSResult.evaluation_run_id == run.id) + if model: + stmt = stmt.where(TTSResult.provider == model) + return list(db.exec(stmt.order_by(TTSResult.id)).all()) + + +@pytest.mark.usefixtures("harness") +class TestExecuteBatchSubmission: + def test_batch_only_submits_gemini_and_links_run( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [GEMINI]) + + with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: + outcome = _execute(run, [GEMINI]) + + assert outcome["success"] is True + assert outcome["result_count"] == 2 + assert outcome["sync_chunks"] == {} + assert list(outcome["batch_jobs"]) == [GEMINI] + assert len(gemini.submitted[f"models/{GEMINI}"]) == 2 + assert harness.enqueued == [] + db.refresh(run) + assert run.status == "processing" + assert run.batch_job_id == outcome["batch_jobs"][GEMINI]["batch_job_id"] + + def test_sync_only_chunks_by_25_with_default_language( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + harness.csv = _csv([f"sample {i}" for i in range(30)]) + run = _run(db, user_api_key, [SARVAM]) + + with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: + outcome = _execute(run, [SARVAM]) + + assert outcome["success"] is True + assert outcome["batch_jobs"] == {} + assert outcome["sync_chunks"] == {SARVAM: ["task-1", "task-2"]} + assert gemini.submitted == {} + result_ids = [r.id for r in _results(db, run)] + assert [c["result_ids"] for c in harness.enqueued] == [ + result_ids[:25], + result_ids[25:], + ] + for chunk in harness.enqueued: + assert chunk["model"] == SARVAM + assert chunk["language_code"] == "en-IN" + assert chunk["job_id"] == str(run.id) + assert chunk["organization_id"] == run.organization_id + assert chunk["trace_id"] == "N/A" + db.refresh(run) + assert run.status == "processing" + assert run.batch_job_id is None + + def test_dataset_language_is_passed_to_chunks( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + hindi = get_language_by_locale(session=db, locale="hi") + run = _run(db, user_api_key, [ELEVEN], language_id=hindi.id) + + _execute(run, [ELEVEN]) + + assert [c["language_code"] for c in harness.enqueued] == ["hi-IN"] + + def test_mixed_models_route_rows_by_execution_mode( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + models = [GEMINI, SARVAM, ELEVEN] + run = _run(db, user_api_key, models) + + with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: + outcome = _execute(run, models) + + assert outcome["result_count"] == 6 + assert list(outcome["batch_jobs"]) == [GEMINI] + assert set(outcome["sync_chunks"]) == {SARVAM, ELEVEN} + gemini_keys = {r["key"] for r in gemini.submitted[f"models/{GEMINI}"]} + assert gemini_keys == {str(r.id) for r in _results(db, run, GEMINI)} + enqueued_by_model = {c["model"]: c["result_ids"] for c in harness.enqueued} + assert enqueued_by_model == { + SARVAM: [r.id for r in _results(db, run, SARVAM)], + ELEVEN: [r.id for r in _results(db, run, ELEVEN)], + } + db.refresh(run) + assert run.status == "processing" + assert run.batch_job_id is not None + + def test_batch_failure_with_sync_models_continues( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [GEMINI, SARVAM]) + + with fake_gemini_batch( + "app.crud.tts_evaluations.batch", + {f"models/{GEMINI}": RuntimeError("quota")}, + ): + outcome = _execute(run, [GEMINI, SARVAM]) + + assert outcome["success"] is True + assert outcome["batch_jobs"] == {} + assert outcome["sync_chunks"] == {SARVAM: ["task-1"]} + assert {r.status for r in _results(db, run, GEMINI)} == {JobStatus.FAILED.value} + assert {r.status for r in _results(db, run, SARVAM)} == { + JobStatus.PENDING.value + } + db.refresh(run) + assert run.status == "processing" + + def test_batch_failure_without_sync_models_fails_run( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [GEMINI]) + + with fake_gemini_batch( + "app.crud.tts_evaluations.batch", + {f"models/{GEMINI}": RuntimeError("quota")}, + ): + outcome = _execute(run, [GEMINI]) + + assert outcome == { + "success": False, + "error": "Batch submission failed for all models", + } + db.refresh(run) + assert run.status == "failed" + assert run.error_message == "Batch submission failed for all models" + + def test_enqueue_failure_fails_that_models_results( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + harness.enqueue_failures = {ELEVEN} + run = _run(db, user_api_key, [SARVAM, ELEVEN]) + + outcome = _execute(run, [SARVAM, ELEVEN]) + + assert outcome["sync_chunks"] == {SARVAM: ["task-1"], ELEVEN: []} + eleven_rows = _results(db, run, ELEVEN) + assert {r.status for r in eleven_rows} == {JobStatus.FAILED.value} + assert eleven_rows[0].error_message.startswith( + f"[KAAPI] Failed to queue synthesis for {ELEVEN}: broker unreachable." + ) + assert {r.status for r in _results(db, run, SARVAM)} == { + JobStatus.PENDING.value + } + db.refresh(run) + assert run.status == "processing" + assert run.error_message == "2 synthesis(es) failed" + + def test_pending_rows_of_unrequested_models_are_not_enqueued( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [SARVAM]) + stray = create_test_tts_result_row(db, run=run, provider="eleven_v4") + + outcome = _execute(run, [SARVAM]) + + assert list(outcome["sync_chunks"]) == [SARVAM] + assert all(stray.id not in c["result_ids"] for c in harness.enqueued) + + def test_run_not_found(self, db: Session, user_api_key: TestAuthContext) -> None: + run = _run(db, user_api_key, [SARVAM]) + missing = EvaluationRun( + id=get_non_existent_id(db, EvaluationRun), + project_id=run.project_id, + organization_id=run.organization_id, + dataset_id=run.dataset_id, + ) + + assert _execute(missing, [SARVAM]) == { + "success": False, + "error": "Run not found", + } + + def test_run_deleted_mid_submission( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [SARVAM]) + run_id = run.id + + def delete_run() -> None: + db.delete(db.get(EvaluationRun, run_id)) + db.commit() + + harness.on_stream = delete_run + + assert _execute(run, [SARVAM]) == {"success": False, "error": "Run not found"} + assert harness.enqueued == [] + + def test_dataset_not_found_fails_run( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _run(db, user_api_key, [SARVAM]) + + outcome = _execute(run, [SARVAM], dataset_id=run.dataset_id + 10_000_000) + + assert outcome == {"success": False, "error": "Dataset not found"} + db.refresh(run) + assert (run.status, run.error_message) == ("failed", "Dataset not found") + + def test_dataset_without_csv_fails_run( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + run = _run(db, user_api_key, [SARVAM], object_store_url=None) + + outcome = _execute(run, [SARVAM]) + + assert outcome == {"success": False, "error": "No samples found"} + db.refresh(run) + assert (run.status, run.error_message) == ( + "failed", + "No samples found for dataset", + ) + assert _results(db, run) == [] + + def test_empty_csv_fails_run( + self, db: Session, user_api_key: TestAuthContext, harness: Harness + ) -> None: + harness.csv = b"text\n" + run = _run(db, user_api_key, [SARVAM]) + + assert _execute(run, [SARVAM]) == { + "success": False, + "error": "No samples found", + } + + @pytest.mark.parametrize( + "timeout", [Timeout(), SoftTimeLimitExceeded()], ids=["gevent", "soft"] + ) + def test_timeout_fails_run_and_reraises( + self, + db: Session, + user_api_key: TestAuthContext, + harness: Harness, + timeout: BaseException, + ) -> None: + run = _run(db, user_api_key, [SARVAM]) + with patch(f"{MODULE}.load_sample_texts", side_effect=timeout): + with pytest.raises(type(timeout)): + _execute(run, [SARVAM]) + + db.refresh(run) + assert (run.status, run.error_message) == ( + "failed", + "Task exceeded soft time limit", + ) + + def test_unexpected_error_fails_run( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _run(db, user_api_key, [SARVAM]) + + with patch( + f"{MODULE}.get_cloud_storage", side_effect=RuntimeError("no bucket") + ): + outcome = _execute(run, [SARVAM]) + + assert outcome == {"success": False, "error": "no bucket"} + db.refresh(run) + assert (run.status, run.error_message) == ("failed", "no bucket") diff --git a/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py b/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py new file mode 100644 index 000000000..fb0ecd350 --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py @@ -0,0 +1,289 @@ +import base64 +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from celery.exceptions import SoftTimeLimitExceeded +from gevent import Timeout +from sqlmodel import Session + +from app.models import EvaluationRun +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.batch_result_processing import ( + execute_tts_result_processing, +) +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, + use_test_session, +) + +MODULE = "app.services.tts_evaluations.batch_result_processing" +GEMINI = "gemini-2.5-pro-preview-tts" +# 24 kHz / 16-bit / mono: 48000 bytes is one second; the WAV adds a 44-byte header. +ONE_SECOND_PCM_B64 = base64.b64encode(b"\x00\x01" * 24000).decode() + + +def _audio_response( + data: str = ONE_SECOND_PCM_B64, key: str = "inlineData" +) -> dict[str, Any]: + return { + "candidates": [ + {"content": {"parts": [{key: {"mimeType": "audio/L16", "data": data}}]}} + ] + } + + +def _ok(result_id: int | str, **kwargs: Any) -> dict[str, Any]: + return {"custom_id": str(result_id), "response": _audio_response(**kwargs)} + + +@dataclass +class Boundaries: + gemini_client: MagicMock + download: MagicMock + upload: MagicMock + + +def _fake_upload(*, subdirectory: str, filename: str, **_kwargs: Any) -> str: + return f"s3://bucket/{subdirectory}/{filename}" + + +@pytest.fixture +def boundaries(db: Session) -> Iterator[Boundaries]: + with ( + use_test_session(MODULE, db), + patch(f"{MODULE}.GeminiClient") as gemini_client, + patch(f"{MODULE}.GeminiBatchProvider") as batch_provider, + patch(f"{MODULE}.get_cloud_storage"), + patch(f"{MODULE}.upload_to_object_store", side_effect=_fake_upload) as upload, + ): + download = batch_provider.return_value.download_batch_results + download.return_value = [] + yield Boundaries(gemini_client, download, upload) + + +@pytest.fixture +def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + models=[GEMINI, "bulbul:v3"], + ) + + +def _execute(run: EvaluationRun) -> dict[str, Any]: + return execute_tts_result_processing( + project_id=run.project_id, + job_id="1", + task_id="celery-task", + task_instance=MagicMock(), + organization_id=run.organization_id, + evaluation_run_id=run.id, + tts_provider=GEMINI, + provider_batch_id="batches/abc", + ) + + +def _refresh(db: Session, *rows: TTSResult | EvaluationRun) -> None: + for row in rows: + db.refresh(row) + + +@pytest.mark.usefixtures("boundaries") +class TestExecuteTTSResultProcessing: + @pytest.mark.parametrize("key", ["inlineData", "inline_data"]) + def test_success_uploads_wav_and_completes_run( + self, db: Session, run: EvaluationRun, boundaries: Boundaries, key: str + ) -> None: + rows = [create_test_tts_result_row(db, run=run) for _ in range(2)] + boundaries.download.return_value = [_ok(r.id, key=key) for r in rows] + + outcome = _execute(run) + + assert outcome == { + "success": True, + "run_id": run.id, + "processed": 2, + "failed": 0, + "run_status": "completed", + } + _refresh(db, run, *rows) + for row in rows: + assert row.status == JobStatus.SUCCESS.value + assert row.object_store_url.startswith("s3://bucket/evaluations/tts/audio/") + assert row.metadata_ == {"duration_seconds": 1.0, "size_bytes": 48044} + assert run.status == "completed" + boundaries.download.assert_called_once_with("batches/abc") + + def test_non_pending_and_unknown_results_are_skipped( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + done = create_test_tts_result_row( + db, + run=run, + status=JobStatus.SUCCESS, + object_store_url="s3://bucket/keep.wav", + ) + other_model = create_test_tts_result_row(db, run=run, provider="bulbul:v3") + pending = create_test_tts_result_row(db, run=run) + boundaries.download.return_value = [ + _ok(done.id), + _ok(other_model.id), + _ok(pending.id), + ] + + outcome = _execute(run) + + assert (outcome["processed"], outcome["failed"]) == (1, 0) + assert outcome["run_status"] == "processing" + assert boundaries.upload.call_count == 1 + _refresh(db, done, other_model, pending) + assert done.object_store_url == "s3://bucket/keep.wav" + assert other_model.status == JobStatus.PENDING.value + assert pending.status == JobStatus.SUCCESS.value + + def test_invalid_custom_id_is_counted_failed( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.download.return_value = [ + _ok("not-a-number"), + {"custom_id": None, "response": {}}, + ] + + outcome = _execute(run) + + assert (outcome["processed"], outcome["failed"]) == (0, 2) + + def test_failure_shapes_mark_results_failed( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + errored, no_error_text, silent, bad_b64, odd_shape = ( + create_test_tts_result_row(db, run=run) for _ in range(5) + ) + boundaries.download.return_value = [ + {"custom_id": str(errored.id), "response": None, "error": "SAFETY block"}, + {"custom_id": str(no_error_text.id)}, + { + "custom_id": str(silent.id), + "response": {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}, + }, + _ok(bad_b64.id, data="abc"), # incorrect padding -> binascii.Error + {"custom_id": str(odd_shape.id), "response": ["not", "a", "dict"]}, + ] + + outcome = _execute(run) + + assert (outcome["processed"], outcome["failed"]) == (0, 5) + assert outcome["run_status"] == "completed" + _refresh(db, run, errored, no_error_text, silent, bad_b64, odd_shape) + assert errored.error_message == "SAFETY block" + assert no_error_text.error_message == "Unknown error" + assert silent.error_message == "No audio data in response" + assert bad_b64.error_message.startswith("Audio processing failed:") + assert odd_shape.error_message.startswith("Audio processing failed:") + assert { + r.status for r in (errored, no_error_text, silent, bad_b64, odd_shape) + } == {JobStatus.FAILED.value} + assert run.error_message == "5 synthesis(es) failed" + + def test_non_base64_audio_is_not_stored_as_empty_success( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + row = create_test_tts_result_row(db, run=run) + boundaries.download.return_value = [_ok(row.id, data="###")] + + outcome = _execute(run) + + assert (outcome["processed"], outcome["failed"]) == (0, 1) + boundaries.upload.assert_not_called() + _refresh(db, row) + assert row.status == JobStatus.FAILED.value + assert row.error_message.startswith("Audio processing failed:") + + def test_upload_failure_marks_result_failed( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.upload.side_effect = None + boundaries.upload.return_value = None + row = create_test_tts_result_row(db, run=run) + boundaries.download.return_value = [_ok(row.id)] + + outcome = _execute(run) + + assert outcome["failed"] == 1 + _refresh(db, row) + assert (row.status, row.error_message) == ( + JobStatus.FAILED.value, + "Audio upload to object store failed", + ) + + def test_empty_batch_finalizes_run( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + + outcome = _execute(run) + + assert (outcome["processed"], outcome["run_status"]) == (0, "completed") + + def test_timeout_keeps_already_written_chunks( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + rows = [create_test_tts_result_row(db, run=run) for _ in range(60)] + boundaries.download.return_value = [_ok(r.id) for r in rows] + uploads = 0 + + def upload_then_time_out(**kwargs: Any) -> str: + nonlocal uploads + uploads += 1 + if uploads == 55: + raise Timeout() + return _fake_upload(**kwargs) + + boundaries.upload.side_effect = upload_then_time_out + + with pytest.raises(Timeout): + _execute(run) + + _refresh(db, run, *rows) + statuses = [r.status for r in rows] + # writes land in slices of 50, so results 51-54 were still buffered + assert statuses.count(JobStatus.SUCCESS.value) == 50 + assert statuses.count(JobStatus.PENDING.value) == 10 + assert (run.status, run.error_message) == ( + "failed", + "Task exceeded soft time limit", + ) + + def test_soft_time_limit_fails_run_and_reraises( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.download.side_effect = SoftTimeLimitExceeded() + + with pytest.raises(SoftTimeLimitExceeded): + _execute(run) + + _refresh(db, run) + assert run.status == "failed" + + def test_unexpected_error_fails_run( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.gemini_client.from_credentials.side_effect = ValueError( + "no gemini key" + ) + + outcome = _execute(run) + + assert outcome == {"success": False, "error": "no gemini key"} + _refresh(db, run) + assert (run.status, run.error_message) == ( + "failed", + "Result processing failed: no gemini key", + ) diff --git a/backend/app/tests/services/tts_evaluations/test_constants.py b/backend/app/tests/services/tts_evaluations/test_constants.py new file mode 100644 index 000000000..f298174fd --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_constants.py @@ -0,0 +1,62 @@ +import pytest + +from app.core.providers import Provider +from app.services.tts_evaluations.constants import ( + SUPPORTED_TTS_MODELS, + TTSExecutionModeEnum, + get_tts_model_spec, + split_models_by_execution_mode, +) + + +class TestGetTTSModelSpec: + @pytest.mark.parametrize( + ("model", "provider", "mode"), + [ + ( + "gemini-2.5-pro-preview-tts", + Provider.GOOGLE_AISTUDIO, + TTSExecutionModeEnum.BATCH, + ), + ("bulbul:v3", Provider.SARVAMAI, TTSExecutionModeEnum.SYNC), + ("eleven_v3", Provider.ELEVENLABS, TTSExecutionModeEnum.SYNC), + ("eleven_v4", Provider.ELEVENLABS, TTSExecutionModeEnum.SYNC), + ], + ) + def test_registry_routes_models( + self, model: str, provider: Provider, mode: TTSExecutionModeEnum + ) -> None: + spec = get_tts_model_spec(model) + + assert spec.provider == provider + assert spec.execution_mode == mode + assert spec.default_voice + + def test_unknown_model_raises_key_error(self) -> None: + with pytest.raises(KeyError): + get_tts_model_spec("unknown-tts") + + def test_supported_models_match_registry(self) -> None: + assert set(SUPPORTED_TTS_MODELS) == { + "gemini-2.5-pro-preview-tts", + "bulbul:v3", + "eleven_v3", + "eleven_v4", + } + + +class TestSplitModelsByExecutionMode: + def test_mixed_models_split_preserving_order(self) -> None: + batch, sync = split_models_by_execution_mode( + ["eleven_v3", "gemini-2.5-pro-preview-tts", "bulbul:v3"] + ) + + assert batch == ["gemini-2.5-pro-preview-tts"] + assert sync == ["eleven_v3", "bulbul:v3"] + + def test_empty_list(self) -> None: + assert split_models_by_execution_mode([]) == ([], []) + + def test_unknown_model_raises(self) -> None: + with pytest.raises(KeyError): + split_models_by_execution_mode(["nope"]) diff --git a/backend/app/tests/services/tts_evaluations/test_dataset.py b/backend/app/tests/services/tts_evaluations/test_dataset.py new file mode 100644 index 000000000..44f5240c0 --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_dataset.py @@ -0,0 +1,113 @@ +import io +from unittest.mock import MagicMock, patch + +from sqlmodel import Session + +from app.services.tts_evaluations.dataset import ( + get_sample_texts_from_dataset, + load_sample_texts, +) +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import TEST_DATASET_URL, create_test_tts_dataset + +CSV_BYTES = b"text\nHello world\n \n Good morning \n" + + +def _storage(content: bytes = CSV_BYTES) -> MagicMock: + def stream(url: str) -> io.BytesIO: + if url != TEST_DATASET_URL: + raise FileNotFoundError(url) + return io.BytesIO(content) + + storage = MagicMock() + storage.stream.side_effect = stream + return storage + + +class TestLoadSampleTexts: + def test_reads_non_blank_stripped_texts(self) -> None: + storage = _storage() + + texts = load_sample_texts( + storage=storage, object_store_url=TEST_DATASET_URL, dataset_id=1 + ) + + assert texts == ["Hello world", "Good morning"] + storage.stream.assert_called_once_with(TEST_DATASET_URL) + + def test_stream_failure_returns_empty(self) -> None: + storage = MagicMock() + storage.stream.side_effect = OSError("s3 down") + + assert ( + load_sample_texts( + storage=storage, object_store_url=TEST_DATASET_URL, dataset_id=1 + ) + == [] + ) + + def test_undecodable_csv_returns_empty(self) -> None: + texts = load_sample_texts( + storage=_storage(b"\xff\xfe\xfa"), + object_store_url=TEST_DATASET_URL, + dataset_id=1, + ) + + assert texts == [] + + +class TestGetSampleTextsFromDataset: + def test_reads_dataset_csv( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + dataset = create_test_tts_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + with patch( + "app.services.tts_evaluations.dataset.get_cloud_storage", + return_value=_storage(), + ): + texts = get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) + + assert texts == ["Hello world", "Good morning"] + + def test_dataset_without_url_returns_empty( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + dataset = create_test_tts_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + object_store_url=None, + ) + + with patch( + "app.services.tts_evaluations.dataset.get_cloud_storage" + ) as get_storage: + assert ( + get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) + == [] + ) + + get_storage.assert_not_called() + + def test_storage_init_failure_returns_empty( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + dataset = create_test_tts_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + with patch( + "app.services.tts_evaluations.dataset.get_cloud_storage", + side_effect=ValueError("no storage credentials"), + ): + assert ( + get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) + == [] + ) diff --git a/backend/app/tests/services/tts_evaluations/test_language.py b/backend/app/tests/services/tts_evaluations/test_language.py new file mode 100644 index 000000000..51c156123 --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_language.py @@ -0,0 +1,75 @@ +import logging + +import pytest +from sqlmodel import Session + +from app.crud.language import get_language_by_locale +from app.models.language import Language +from app.services.tts_evaluations.language import ( + resolve_dataset_tts_language_code, + resolve_tts_language_code, + to_elevenlabs_language_code, +) +from app.tests.utils.utils import get_non_existent_id + + +class TestResolveTTSLanguageCode: + @pytest.mark.parametrize( + ("locale", "expected"), + [ + ("hi", "hi-IN"), + ("HI", "hi-IN"), + ("ta", "ta-IN"), + ("en", "en-IN"), + # Sarvam spells Odia "od-IN", not the ISO-derived "or-IN" + ("or", "od-IN"), + ], + ) + def test_maps_iso_locale_to_bcp47(self, locale: str, expected: str) -> None: + assert resolve_tts_language_code(locale) == expected + + @pytest.mark.parametrize("locale", [None, ""]) + def test_missing_locale_defaults_to_en_in(self, locale: str | None) -> None: + assert resolve_tts_language_code(locale) == "en-IN" + + def test_unmapped_locale_defaults_with_warning( + self, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + assert resolve_tts_language_code("xx") == "en-IN" + + assert "Unmapped locale" in caplog.text + + +class TestResolveDatasetTTSLanguageCode: + def test_none_language_id_defaults_to_en_in_silently( + self, db: Session, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + assert resolve_dataset_tts_language_code(db, None) == "en-IN" + + # An unset language is a normal dataset shape, not a misconfiguration. + assert "Language not found" not in caplog.text + + def test_seeded_language_resolves(self, db: Session) -> None: + odia = get_language_by_locale(session=db, locale="or") + + assert resolve_dataset_tts_language_code(db, odia.id) == "od-IN" + + def test_unknown_language_id_defaults_with_warning( + self, db: Session, caplog: pytest.LogCaptureFixture + ) -> None: + missing_id = get_non_existent_id(db, Language) + + with caplog.at_level(logging.WARNING): + assert resolve_dataset_tts_language_code(db, missing_id) == "en-IN" + + assert "Language not found" in caplog.text + + +class TestToElevenlabsLanguageCode: + def test_known_bcp47_maps_to_iso(self) -> None: + assert to_elevenlabs_language_code("hi-IN") == "hi" + + def test_odia_returns_none_for_auto_detect(self) -> None: + assert to_elevenlabs_language_code("od-IN") is None diff --git a/backend/app/tests/services/tts_evaluations/test_sync_generation.py b/backend/app/tests/services/tts_evaluations/test_sync_generation.py new file mode 100644 index 000000000..71f82a3c2 --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_sync_generation.py @@ -0,0 +1,347 @@ +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from celery.exceptions import SoftTimeLimitExceeded +from gevent import Timeout +from sqlmodel import Session + +from app.models import EvaluationRun +from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.sync_generation import execute_tts_sync_chunk +from app.services.tts_evaluations.synthesizers import ( + SynthesizedAudio, + TTSSynthesisError, +) +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.tts_evaluation import ( + create_test_tts_result_row, + create_test_tts_run_with_dataset, + use_test_session, +) + +MODULE = "app.services.tts_evaluations.sync_generation" +SARVAM_MODEL = "bulbul:v3" +FAIL_TEXT = "please fail" +CRASH_TEXT = "please crash" + + +@dataclass +class Boundaries: + credentials: MagicMock + storage: MagicMock + upload: MagicMock + create_client: MagicMock + synthesize: MagicMock + + +def _fake_synthesize(*, text: str, **_kwargs: Any) -> SynthesizedAudio: + if text == FAIL_TEXT: + raise TTSSynthesisError( + "[SARVAM] TTS request failed (code: 400): bad", retryable=False + ) + if text == CRASH_TEXT: + raise KeyError("audios") + return SynthesizedAudio(wav_bytes=b"w" * 100, duration_seconds=1.23456) + + +def _fake_upload(*, subdirectory: str, filename: str, **_kwargs: Any) -> str: + return f"s3://bucket/{subdirectory}/{filename}" + + +@pytest.fixture +def boundaries(db: Session) -> Iterator[Boundaries]: + with ( + use_test_session(MODULE, db), + patch( + f"{MODULE}.get_provider_credential", return_value={"api_key": "k"} + ) as creds, + patch(f"{MODULE}.get_cloud_storage") as storage, + patch(f"{MODULE}.upload_to_object_store", side_effect=_fake_upload) as upload, + patch(f"{MODULE}.create_tts_client", return_value=MagicMock()) as create_client, + patch(f"{MODULE}.synthesize_tts", side_effect=_fake_synthesize) as synthesize, + ): + yield Boundaries(creds, storage, upload, create_client, synthesize) + + +@pytest.fixture +def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: + return create_test_tts_run_with_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + models=[SARVAM_MODEL], + ) + + +def _pending(db: Session, run: EvaluationRun, text: str | None = None) -> TTSResult: + return create_test_tts_result_row( + db, run=run, provider=SARVAM_MODEL, sample_text=text + ) + + +def _execute(run: EvaluationRun, result_ids: list[int]) -> dict[str, Any]: + return execute_tts_sync_chunk( + project_id=run.project_id, + job_id=str(run.id), + task_id="celery-task", + task_instance=MagicMock(), + organization_id=run.organization_id, + model=SARVAM_MODEL, + result_ids=result_ids, + language_code="hi-IN", + ) + + +def _reload(db: Session, *rows: TTSResult | EvaluationRun) -> None: + for row in rows: + db.refresh(row) + + +@pytest.mark.usefixtures("boundaries") +class TestExecuteTTSSyncChunk: + def test_success_writes_audio_and_completes_run( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + first = _pending(db, run) + second = _pending(db, run) + + outcome = _execute(run, [first.id, second.id]) + + assert outcome == { + "success": True, + "run_id": run.id, + "processed": 2, + "failed": 0, + "run_status": "completed", + } + _reload(db, first, second, run) + for row in (first, second): + assert row.status == JobStatus.SUCCESS.value + assert row.object_store_url.startswith("s3://bucket/evaluations/tts/audio/") + assert row.object_store_url.endswith(".wav") + assert row.metadata_ == {"duration_seconds": 1.235, "size_bytes": 100} + assert row.error_message is None + assert run.status == "completed" + assert run.error_message is None + assert boundaries.synthesize.call_args.kwargs["language_code"] == "hi-IN" + assert boundaries.synthesize.call_args.kwargs["voice"] == "shubh" + + def test_other_pending_rows_keep_run_processing( + self, db: Session, run: EvaluationRun + ) -> None: + mine = _pending(db, run) + _pending(db, run) # belongs to a different chunk + + outcome = _execute(run, [mine.id]) + + assert outcome["run_status"] == "processing" + _reload(db, run) + assert run.status == "processing" + + def test_redelivered_chunk_skips_finished_rows( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + done = create_test_tts_result_row( + db, + run=run, + provider=SARVAM_MODEL, + status=JobStatus.SUCCESS, + object_store_url="s3://bucket/original.wav", + ) + pending = _pending(db, run) + + outcome = _execute(run, [done.id, pending.id]) + + assert outcome["processed"] == 1 + assert boundaries.synthesize.call_count == 1 + _reload(db, done) + assert done.object_store_url == "s3://bucket/original.wav" + + def test_row_finished_concurrently_is_not_counted_or_overwritten( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + row = _pending(db, run) + + def concurrent_writer_wins(**kwargs: Any) -> SynthesizedAudio: + # A redelivered copy of this chunk lands its write first. The main + # thread is parked in as_completed, so the session isn't shared concurrently. + db.refresh(row) + row.status = JobStatus.SUCCESS.value + row.object_store_url = "s3://bucket/other-worker.wav" + db.add(row) + db.commit() + return _fake_synthesize(**kwargs) + + boundaries.synthesize.side_effect = concurrent_writer_wins + + outcome = _execute(run, [row.id]) + + assert (outcome["processed"], outcome["failed"]) == (0, 0) + _reload(db, row) + assert row.object_store_url == "s3://bucket/other-worker.wav" + + def test_no_pending_rows_is_a_noop_that_still_finalizes( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + done = create_test_tts_result_row( + db, run=run, provider=SARVAM_MODEL, status=JobStatus.SUCCESS + ) + + outcome = _execute(run, [done.id]) + + assert outcome == { + "success": True, + "run_id": run.id, + "processed": 0, + "failed": 0, + "run_status": "completed", + } + boundaries.synthesize.assert_not_called() + boundaries.credentials.assert_not_called() + + def test_missing_credentials_fail_the_chunk( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.credentials.return_value = None + rows = [_pending(db, run), _pending(db, run)] + + outcome = _execute(run, [r.id for r in rows]) + + assert outcome["success"] is False + assert outcome["failed"] == 2 + assert outcome["run_status"] == "completed" + assert "sarvamai credentials are not configured" in outcome["error"] + _reload(db, run, *rows) + for row in rows: + assert row.status == JobStatus.FAILED.value + assert row.error_message == outcome["error"] + assert run.error_message == "2 synthesis(es) failed" + boundaries.synthesize.assert_not_called() + + def test_invalid_credentials_fail_the_chunk( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.create_client.side_effect = ValueError( + "API Key for SarvamAI Not Set" + ) + row = _pending(db, run) + + outcome = _execute(run, [row.id]) + + assert outcome["success"] is False + assert outcome["error"].startswith( + "[KAAPI] Invalid sarvamai credentials: API Key for SarvamAI Not Set." + ) + _reload(db, row) + assert row.status == JobStatus.FAILED.value + + def test_synthesis_errors_fail_individual_results( + self, db: Session, run: EvaluationRun + ) -> None: + ok = _pending(db, run) + rejected = _pending(db, run, text=FAIL_TEXT) + crashed = _pending(db, run, text=CRASH_TEXT) + + outcome = _execute(run, [ok.id, rejected.id, crashed.id]) + + assert (outcome["success"], outcome["processed"], outcome["failed"]) == ( + True, + 1, + 2, + ) + _reload(db, ok, rejected, crashed, run) + assert ok.status == JobStatus.SUCCESS.value + assert rejected.status == JobStatus.FAILED.value + assert rejected.error_message == "[SARVAM] TTS request failed (code: 400): bad" + assert crashed.status == JobStatus.FAILED.value + assert crashed.error_message.startswith( + "[KAAPI] Unexpected error during sarvamai synthesis: 'audios'." + ) + assert run.status == "completed" + assert run.error_message == "2 synthesis(es) failed" + + def test_upload_failure_fails_result( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.upload.side_effect = None + boundaries.upload.return_value = None + row = _pending(db, run) + + outcome = _execute(run, [row.id]) + + assert (outcome["processed"], outcome["failed"]) == (0, 1) + _reload(db, row) + assert row.status == JobStatus.FAILED.value + assert row.object_store_url is None + assert row.error_message.startswith( + "[KAAPI] Audio upload to object store failed." + ) + + def test_gevent_timeout_fails_leftover_rows_and_reraises( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.synthesize.side_effect = Timeout() + done = create_test_tts_result_row( + db, run=run, provider=SARVAM_MODEL, status=JobStatus.SUCCESS + ) + row = _pending(db, run) + + with pytest.raises(Timeout): + _execute(run, [done.id, row.id]) + + _reload(db, done, row, run) + assert row.status == JobStatus.FAILED.value + assert row.error_message.startswith("[KAAPI] Synthesis timed out") + assert done.status == JobStatus.SUCCESS.value + assert run.status == "completed" + + def test_soft_time_limit_fails_leftover_rows_and_reraises( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.upload.side_effect = SoftTimeLimitExceeded() + row = _pending(db, run) + + with pytest.raises(SoftTimeLimitExceeded): + _execute(run, [row.id]) + + _reload(db, row, run) + assert row.status == JobStatus.FAILED.value + assert row.error_message.startswith("[KAAPI] Synthesis timed out") + assert run.status == "completed" + + def test_unexpected_exception_fails_leftover_rows( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.upload.side_effect = RuntimeError("s3 exploded") + row = _pending(db, run) + + outcome = _execute(run, [row.id]) + + assert outcome == { + "success": False, + "run_id": run.id, + "error": "s3 exploded", + "run_status": "completed", + } + _reload(db, row) + assert row.status == JobStatus.FAILED.value + assert row.error_message.startswith( + "[KAAPI] Synthesis chunk failed unexpectedly: s3 exploded." + ) + + def test_client_factory_returning_nothing_fails_chunk( + self, db: Session, run: EvaluationRun, boundaries: Boundaries + ) -> None: + boundaries.create_client.return_value = None + row = _pending(db, run) + + outcome = _execute(run, [row.id]) + + assert outcome["success"] is False + assert "missing its client or storage" in outcome["error"] + _reload(db, row) + assert row.status == JobStatus.FAILED.value diff --git a/backend/app/tests/services/tts_evaluations/test_synthesizers.py b/backend/app/tests/services/tts_evaluations/test_synthesizers.py new file mode 100644 index 000000000..2b0890cdc --- /dev/null +++ b/backend/app/tests/services/tts_evaluations/test_synthesizers.py @@ -0,0 +1,368 @@ +import base64 +import io +import logging +import wave +from collections.abc import Iterator +from types import SimpleNamespace +from unittest.mock import MagicMock + +import httpx +import pytest +from elevenlabs import ElevenLabs +from elevenlabs.core.api_error import ApiError as ElevenLabsApiError +from sarvamai import SarvamAI +from sarvamai.core.api_error import ApiError as SarvamApiError +from tenacity import wait_none + +from app.core.providers import Provider +from app.services.tts_evaluations.synthesizers import ( + SynthesizedAudio, + TTSSynthesisError, + create_tts_client, + synthesize_elevenlabs, + synthesize_sarvam, + synthesize_tts, +) + + +def _wav(pcm: bytes, *, rate: int = 24000, width: int = 2, channels: int = 1) -> bytes: + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setnchannels(channels) + w.setsampwidth(width) + w.setframerate(rate) + w.writeframes(pcm) + return buf.getvalue() + + +def _read_wav(wav_bytes: bytes) -> tuple[int, int, int, bytes]: + with wave.open(io.BytesIO(wav_bytes), "rb") as w: + return ( + w.getnchannels(), + w.getsampwidth(), + w.getframerate(), + w.readframes(w.getnframes()), + ) + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def _sarvam_client( + *, audios: list[str] | None = None, error: Exception | None = None +) -> MagicMock: + client = MagicMock() + if error is not None: + client.text_to_speech.convert.side_effect = error + else: + client.text_to_speech.convert.return_value = SimpleNamespace( + audios=audios, request_id="req-1" + ) + return client + + +def _elevenlabs_client(chunks: Iterator[bytes] | list[bytes]) -> MagicMock: + client = MagicMock() + client.text_to_speech.convert.return_value = chunks + return client + + +def _call_sarvam(client: MagicMock) -> SynthesizedAudio: + return synthesize_sarvam( + client=client, + text="namaste", + model="bulbul:v3", + language_code="hi-IN", + speaker="shubh", + ) + + +def _call_elevenlabs( + client: MagicMock, language_code: str = "hi-IN" +) -> SynthesizedAudio: + return synthesize_elevenlabs( + client=client, + text="namaste", + model="eleven_v3", + language_code=language_code, + voice_id="voice-1", + ) + + +# 24 kHz / 16-bit / mono: 48000 bytes is exactly one second. +ONE_SECOND_PCM = b"\x01\x00" * 24000 + + +class TestSynthesizeSarvam: + def test_multi_chunk_wavs_are_joined_under_one_header(self) -> None: + first = b"\x01\x00" * 100 + second = b"\x02\x00" * 50 + client = _sarvam_client(audios=[_b64(_wav(first)), _b64(_wav(second))]) + + audio = _call_sarvam(client) + + channels, width, rate, frames = _read_wav(audio.wav_bytes) + assert (channels, width, rate) == (1, 2, 24000) + assert frames == first + second + # a second RIFF header left inside the PCM would play as a click + assert audio.wav_bytes.count(b"RIFF") == 1 + assert audio.duration_seconds == pytest.approx(150 / 24000) + + def test_chunk_at_other_sample_rate_is_resampled_to_24k(self) -> None: + ten_ms_at_16k = b"\x00\x10" * 160 + client = _sarvam_client(audios=[_b64(_wav(ten_ms_at_16k, rate=16000))]) + + audio = _call_sarvam(client) + + _, _, rate, frames = _read_wav(audio.wav_bytes) + assert rate == 24000 + # 10 ms at 24 kHz is 240 frames; ratecv may drop one to rounding. + assert len(frames) // 2 in (239, 240) + assert audio.duration_seconds == pytest.approx(0.01, abs=2 / 24000) + + def test_one_second_duration(self) -> None: + client = _sarvam_client(audios=[_b64(_wav(ONE_SECOND_PCM))]) + + assert _call_sarvam(client).duration_seconds == pytest.approx(1.0) + + @pytest.mark.parametrize("audios", [[], None]) + def test_empty_audio_list_is_non_retryable(self, audios: list[str] | None) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_sarvam(_sarvam_client(audios=audios)) + + assert exc.value.retryable is False + assert exc.value.message.startswith("[SARVAM] TTS response contains no audio") + + @pytest.mark.parametrize( + "bad_chunk", + [_b64(b"definitely not a wav file"), "abc"], + ids=["not-wav", "bad-base64-padding"], + ) + def test_undecodable_audio_is_kaapi_error(self, bad_chunk: str) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_sarvam(_sarvam_client(audios=[bad_chunk])) + + assert exc.value.retryable is False + assert exc.value.message.startswith("[KAAPI] Failed to decode Sarvam audio") + + @pytest.mark.parametrize( + ("status_code", "retryable", "hint"), + [ + (400, False, "Review the input text"), + (401, False, "Verify the API key"), + (429, True, "Hit the provider's rate/quota"), + (500, True, "Typically transient"), + (503, True, "temporarily down"), + (None, False, "If this persists, contact Kaapi."), + ], + ) + def test_api_error_maps_status_to_retryability( + self, status_code: int | None, retryable: bool, hint: str + ) -> None: + error = SarvamApiError(status_code=status_code, body={"detail": "nope"}) + + with pytest.raises(TTSSynthesisError) as exc: + _call_sarvam(_sarvam_client(error=error)) + + assert exc.value.retryable is retryable + assert exc.value.message.startswith( + f"[SARVAM] TTS request failed (code: {status_code})" + ) + assert hint in exc.value.message + + def test_server_errors_log_at_error_and_client_errors_at_warning( + self, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + for status_code in (500, 404): + with pytest.raises(TTSSynthesisError): + _call_sarvam( + _sarvam_client(error=SarvamApiError(status_code=status_code)) + ) + + levels = { + r.message.split("(code: ")[1][:3]: r.levelno + for r in caplog.records + if "code: " in r.message + } + assert levels == {"500": logging.ERROR, "404": logging.WARNING} + + def test_network_error_is_retryable(self) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_sarvam(_sarvam_client(error=httpx.ConnectError("dns failure"))) + + assert exc.value.retryable is True + assert ( + "[KAAPI] Could not reach Sarvam (code: ConnectError)" in exc.value.message + ) + + def test_timeout_is_retryable_with_timeout_message(self) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_sarvam(_sarvam_client(error=httpx.ReadTimeout("slow"))) + + assert exc.value.retryable is True + assert ( + "[KAAPI] TTS request to Sarvam timed out (code: ReadTimeout)" + in exc.value.message + ) + + +class TestSynthesizeElevenlabs: + def test_streamed_pcm_is_wrapped_as_wav(self) -> None: + client = _elevenlabs_client([ONE_SECOND_PCM[:1000], ONE_SECOND_PCM[1000:]]) + + audio = _call_elevenlabs(client) + + channels, width, rate, frames = _read_wav(audio.wav_bytes) + assert (channels, width, rate) == (1, 2, 24000) + assert frames == ONE_SECOND_PCM + assert audio.duration_seconds == pytest.approx(1.0) + + def test_language_hint_sent_for_supported_language(self) -> None: + client = _elevenlabs_client([ONE_SECOND_PCM]) + + _call_elevenlabs(client, language_code="hi-IN") + + kwargs = client.text_to_speech.convert.call_args.kwargs + assert kwargs["language_code"] == "hi" + assert kwargs["output_format"] == "pcm_24000" + + def test_language_hint_omitted_for_odia_auto_detect(self) -> None: + client = _elevenlabs_client([ONE_SECOND_PCM]) + + _call_elevenlabs(client, language_code="od-IN") + + assert "language_code" not in client.text_to_speech.convert.call_args.kwargs + + def test_empty_stream_is_non_retryable(self) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_elevenlabs(_elevenlabs_client([])) + + assert exc.value.retryable is False + assert exc.value.message.startswith( + "[ELEVENLABS] TTS response contains no audio" + ) + + @pytest.mark.parametrize( + ("status_code", "retryable"), [(422, False), (429, True), (502, True)] + ) + def test_api_error_raised_while_streaming( + self, status_code: int, retryable: bool + ) -> None: + def lazy_stream() -> Iterator[bytes]: + yield b"\x00\x00" + raise ElevenLabsApiError(status_code=status_code, body="bad") + + with pytest.raises(TTSSynthesisError) as exc: + _call_elevenlabs(_elevenlabs_client(lazy_stream())) + + assert exc.value.retryable is retryable + assert exc.value.message.startswith( + f"[ELEVENLABS] TTS request failed (code: {status_code})" + ) + + def test_network_error_is_retryable(self) -> None: + client = MagicMock() + client.text_to_speech.convert.side_effect = httpx.ConnectTimeout("t") + + with pytest.raises(TTSSynthesisError) as exc: + _call_elevenlabs(client) + + assert exc.value.retryable is True + assert "TTS request to Elevenlabs timed out" in exc.value.message + + +@pytest.fixture +def no_retry_wait(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(synthesize_tts.retry, "wait", wait_none()) + + +def _call_tts(client: MagicMock, provider: Provider) -> SynthesizedAudio: + return synthesize_tts( + client=client, + provider=provider, + text="hello", + model="bulbul:v3", + language_code="en-IN", + voice="shubh", + ) + + +@pytest.mark.usefixtures("no_retry_wait") +class TestSynthesizeTTS: + def test_dispatches_to_sarvam(self) -> None: + client = _sarvam_client(audios=[_b64(_wav(ONE_SECOND_PCM))]) + + audio = _call_tts(client, Provider.SARVAMAI) + + assert audio.duration_seconds == pytest.approx(1.0) + assert client.text_to_speech.convert.call_args.kwargs["speaker"] == "shubh" + + def test_dispatches_to_elevenlabs(self) -> None: + client = _elevenlabs_client([ONE_SECOND_PCM]) + + audio = _call_tts(client, Provider.ELEVENLABS) + + assert audio.duration_seconds == pytest.approx(1.0) + assert client.text_to_speech.convert.call_args.kwargs["voice_id"] == "shubh" + + def test_transient_failures_are_retried_until_success(self) -> None: + client = MagicMock() + client.text_to_speech.convert.side_effect = [ + SarvamApiError(status_code=503), + httpx.ConnectError("blip"), + SimpleNamespace(audios=[_b64(_wav(ONE_SECOND_PCM))], request_id="r"), + ] + + audio = _call_tts(client, Provider.SARVAMAI) + + assert audio.duration_seconds == pytest.approx(1.0) + assert client.text_to_speech.convert.call_count == 3 + + def test_gives_up_after_three_attempts(self) -> None: + client = _sarvam_client(error=SarvamApiError(status_code=429)) + + with pytest.raises(TTSSynthesisError) as exc: + _call_tts(client, Provider.SARVAMAI) + + assert exc.value.retryable is True + assert client.text_to_speech.convert.call_count == 3 + + def test_non_retryable_failure_is_not_retried(self) -> None: + client = _sarvam_client(error=SarvamApiError(status_code=400)) + + with pytest.raises(TTSSynthesisError): + _call_tts(client, Provider.SARVAMAI) + + assert client.text_to_speech.convert.call_count == 1 + + def test_unknown_provider_is_rejected(self) -> None: + with pytest.raises(TTSSynthesisError) as exc: + _call_tts(MagicMock(), Provider.GOOGLE_AISTUDIO) + + assert exc.value.retryable is False + assert ( + "No synchronous synthesizer for provider 'google-aistudio'" + in exc.value.message + ) + + +class TestCreateTTSClient: + def test_sarvam_client(self) -> None: + assert isinstance( + create_tts_client(Provider.SARVAMAI, {"api_key": "k"}), SarvamAI + ) + + def test_elevenlabs_client(self) -> None: + assert isinstance( + create_tts_client(Provider.ELEVENLABS, {"api_key": "k"}), ElevenLabs + ) + + def test_missing_api_key_raises(self) -> None: + with pytest.raises(ValueError, match="API Key"): + create_tts_client(Provider.SARVAMAI, {}) + + def test_batch_only_provider_raises(self) -> None: + with pytest.raises(ValueError, match="has no synchronous TTS client"): + create_tts_client(Provider.GOOGLE_AISTUDIO, {"api_key": "k"}) diff --git a/backend/app/tests/utils/tts_evaluation.py b/backend/app/tests/utils/tts_evaluation.py new file mode 100644 index 000000000..dadeb4c3f --- /dev/null +++ b/backend/app/tests/utils/tts_evaluation.py @@ -0,0 +1,167 @@ +"""Factories and session helpers for TTS evaluation tests.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +from sqlmodel import Session + +from app.core.util import now +from app.crud.tts_evaluations import create_tts_run +from app.models import EvaluationDataset, EvaluationRun +from app.models.job import JobStatus +from app.models.stt_evaluation import EvaluationType +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.constants import GEMINI_PRO_TTS_MODEL +from app.tests.utils.utils import random_lower_string + +TEST_DATASET_URL = "s3://test-bucket/tts_datasets/test.csv" + + +class NonClosingSession: + """Lets a service's `with Session(engine)` reuse the transactional test session. + + Services open their own sessions; pointing them at the conftest `db` keeps + every write inside the per-test rollback and visible to the test's asserts. + """ + + def __init__(self, session: Session) -> None: + self._session = session + + def __enter__(self) -> Session: + return self._session + + def __exit__(self, *_exc: object) -> bool: + return False + + +@contextmanager +def use_test_session(module_path: str, db: Session) -> Iterator[Session]: + with patch( + f"{module_path}.Session", side_effect=lambda *_a, **_k: NonClosingSession(db) + ): + yield db + + +class FakeGeminiBatchProvider: + """Stands in for the Gemini Batch API (the network boundary of a TTS batch). + + `failures` maps a model path ("models/") to the exception its + create_batch raises; every call's JSONL is recorded per model path. + """ + + def __init__(self, failures: dict[str, Exception] | None = None) -> None: + self.failures = failures or {} + self.submitted: dict[str, list[dict[str, Any]]] = {} + + def __call__( + self, *, client: Any, model: str | None = None + ) -> "FakeGeminiBatchProvider": + self._model = model or "" + return self + + def create_batch( + self, jsonl_data: list[dict[str, Any]], config: dict[str, Any] + ) -> dict[str, Any]: + if self._model in self.failures: + raise self.failures[self._model] + self.submitted[self._model] = jsonl_data + return { + "provider_batch_id": f"batches/{self._model.split('/')[-1]}", + "provider_file_id": "files/input", + "provider_status": "JOB_STATE_PENDING", + "total_items": len(jsonl_data), + } + + +@contextmanager +def fake_gemini_batch( + module_path: str, failures: dict[str, Exception] | None = None +) -> Iterator[FakeGeminiBatchProvider]: + fake = FakeGeminiBatchProvider(failures) + with ( + patch(f"{module_path}.GeminiClient") as gemini_client, + patch(f"{module_path}.GeminiBatchProvider", side_effect=fake), + ): + gemini_client.from_credentials.return_value = MagicMock() + yield fake + + +def create_test_tts_dataset( + db: Session, + *, + organization_id: int, + project_id: int, + language_id: int | None = None, + object_store_url: str | None = TEST_DATASET_URL, + sample_count: int = 1, +) -> EvaluationDataset: + dataset = EvaluationDataset( + name=f"tts_dataset_{random_lower_string()}", + type=EvaluationType.TTS.value, + language_id=language_id, + object_store_url=object_store_url, + dataset_metadata={"sample_count": sample_count}, + organization_id=organization_id, + project_id=project_id, + inserted_at=now(), + updated_at=now(), + ) + db.add(dataset) + db.commit() + db.refresh(dataset) + return dataset + + +def create_test_tts_run_with_dataset( + db: Session, + *, + organization_id: int, + project_id: int, + models: list[str] | None = None, + language_id: int | None = None, + object_store_url: str | None = TEST_DATASET_URL, +) -> EvaluationRun: + dataset = create_test_tts_dataset( + db, + organization_id=organization_id, + project_id=project_id, + language_id=language_id, + object_store_url=object_store_url, + ) + return create_tts_run( + session=db, + run_name=f"tts_run_{random_lower_string()}", + dataset_id=dataset.id, + dataset_name=dataset.name, + org_id=organization_id, + project_id=project_id, + models=models or [GEMINI_PRO_TTS_MODEL], + ) + + +def create_test_tts_result_row( + db: Session, + *, + run: EvaluationRun, + provider: str = GEMINI_PRO_TTS_MODEL, + status: JobStatus = JobStatus.PENDING, + sample_text: str | None = None, + object_store_url: str | None = None, +) -> TTSResult: + result = TTSResult( + sample_text=sample_text or f"text {random_lower_string()}", + object_store_url=object_store_url, + provider=provider, + status=status.value, + evaluation_run_id=run.id, + organization_id=run.organization_id, + project_id=run.project_id, + inserted_at=now(), + updated_at=now(), + ) + db.add(result) + db.commit() + db.refresh(result) + return result diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 00669c4b8..a95f0384d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -1,6 +1,6 @@ # Module: Evaluations -Measures LLM config quality on golden datasets. Three families sharing one dataset/run schema: `text` (cosine + LLM-judge, OpenAI), `stt` (WER/CER, Gemini), `tts` (human annotation, Gemini). Batch-first (provider Batch APIs, not `/llm/call`), plus a synchronous `fast` text mode capped at `EVAL_FAST_MAX_UNIQUE_ROWS`. +Measures LLM config quality on golden datasets. Three families sharing one dataset/run schema: `text` (cosine + LLM-judge, OpenAI), `stt` (WER/CER, Gemini), `tts` (human annotation; Gemini batch + Sarvam/ElevenLabs sync). Batch-first (provider Batch APIs, not `/llm/call`), plus a synchronous `fast` text mode capped at `EVAL_FAST_MAX_UNIQUE_ROWS`. Deep dive: `docs/architecture/kaapi-evaluations-ARCHITECTURE.md` (§3 data model, §5 text pipeline, §6 STT, §7 TTS, §8 batch infra, §9 cron, §12 design decisions). All paths relative to `backend/app/`. @@ -32,11 +32,14 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes + graph wiring only), `iteration_state.py` (`EvaluationIterationState`, `advance_round_state` — the pure per-round bookkeeping/stop arithmetic, `build_iteration_report`), `iteration_checkpointer.py` (`get_evaluation_iteration_checkpointer`, psycopg pool) - `services/stt_evaluations/`, `services/tts_evaluations/` +- TTS (`services/tts_evaluations/`): `constants.py` holds `TTS_MODEL_REGISTRY` (model → `TTSModelSpec`: provider, `TTSExecutionModeEnum` BATCH/SYNC, default voice); `SUPPORTED_TTS_MODELS` (the request validator's allow-list) is derived from it. Models: `gemini-2.5-pro-preview-tts` (BATCH), `bulbul:v3` (Sarvam, SYNC), `eleven_v3` / `eleven_v4` (ElevenLabs, SYNC). `language.py` resolves the dataset's `language_id` → ISO locale → BCP-47 (`hi` → `hi-IN`, `or` → `od-IN`), defaulting to `en-IN` when unset/unmapped; ElevenLabs gets the ISO code via `BCP47_TO_ELEVENLABS_LANG` (omitted → auto-detect). `synthesizers.py` (`synthesize_sarvam`, `synthesize_elevenlabs`, retrying `synthesize_tts`) normalizes every clip to 24 kHz/16-bit/mono WAV; Sarvam's multi-chunk responses are decoded and header-stripped per chunk before re-wrapping once. Failures raise `TTSSynthesisError` (source-tagged message + `retryable` for 429/5xx/network; SDK retries disabled so tenacity owns the policy). `crud/tts_evaluations/run.py::finalize_tts_run_status` is the single run-finalizer (locks the run row `FOR UPDATE`, counts results: `completed` when no PENDING else `processing`, `"N synthesis(es) failed"`), shared by sync chunks, Gemini result processing, and the TTS cron. - `crud/evaluations/` — `core.py`, `batch.py`, `fast.py` (orchestration + S3/OpenAI IO only; the pieces below carry the rest), `fast_results.py` (pure per-item S3 unit shapes, usage sums, failure threshold), `fast_chunks.py` (`batch_job` bookkeeping + chunk queries/cleanup), `fast_cosine.py` (pure v1 cosine scoring), `fast_traces.py` (pure `TraceData` building), `judge_stage.py` (run-level v2 judge: `resolve_config_prompt`, `judge_rows` pool, per-metric summary rollup), `judge_prompts.py` (rubric text, moved out of `score.py`), `retry.py` (shared OpenAI retry policy), `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. +- TTS submission (`run_tts_batch_submission` → `services/tts_evaluations/batch_job.py::execute_batch_submission`) creates one `tts_result` per text×model, then splits models by execution mode: BATCH models go to `start_tts_evaluation_batch` (Gemini Batch; sets `batch_job_id` so the cron polls it), SYNC models fan out `ceil(n / TTS_SYNC_CHUNK_SIZE)` `run_tts_sync_chunk` tasks per model (`celery/utils.py::start_tts_sync_chunk`, priority 2) → `services/tts_evaluations/sync_generation.py::execute_tts_sync_chunk`. Sync-only runs leave `batch_job_id` NULL, so the Gemini cron never picks them up; the last chunk to finish finalizes the run. A chunk only synthesizes rows still PENDING (idempotent on redelivery), runs provider calls + S3 uploads in a bounded thread pool with no DB session held, and writes each outcome in its own short session; on timeout/crash its leftover PENDING rows are marked FAILED so the run can't strand. If every batch model fails in a mixed run, the run continues with its sync models. +- TTS DB-connection discipline: submission, sync chunks and Gemini result processing (`batch_result_processing.py`) never hold a session across network I/O — short sessions for prefetch, session-free download/convert/upload, result writes committed in `TTS_RESULT_WRITE_CHUNK_SIZE` slices, fresh session for failure marking. `core/batch/operations.py::start_batch_job` commits (ending the read transaction opened by `create_batch_job`'s refresh) before `provider.create_batch` for every caller. The cron's `poll_batch_status` loop still runs inside the cron session's open transaction. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. - Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. From 8c0479c36950c58ef523ce3569eea59f7d9e8489 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Wed, 7 Oct 2026 19:25:01 +0530 Subject: [PATCH 2/3] feat: sync models of sarvamai, 11labs hook up with tts eval path --- .../docs/tts_evaluation/start_evaluation.md | 2 +- .../api/routes/tts_evaluations/evaluation.py | 14 +- backend/app/celery/tasks/job_execution.py | 32 +- backend/app/celery/utils.py | 9 +- backend/app/crud/language.py | 6 - backend/app/crud/tts_evaluations/__init__.py | 21 +- backend/app/crud/tts_evaluations/batch.py | 83 ++- backend/app/crud/tts_evaluations/cron.py | 328 +++++++---- backend/app/crud/tts_evaluations/result.py | 81 +-- backend/app/crud/tts_evaluations/run.py | 25 +- backend/app/models/tts_evaluation.py | 17 +- .../app/services/tts_evaluations/batch_job.py | 367 ++++-------- .../batch_result_processing.py | 241 ++++---- .../app/services/tts_evaluations/constants.py | 145 ++--- .../app/services/tts_evaluations/dataset.py | 53 +- .../app/services/tts_evaluations/language.py | 61 -- .../tts_evaluations/sync_generation.py | 550 ++++++------------ .../services/tts_evaluations/synthesizers.py | 403 ------------- backend/app/tests/celery/test_tts_tasks.py | 110 ++-- backend/app/tests/crud/test_language.py | 17 - .../tests/crud/tts_evaluations/test_batch.py | 180 ------ .../tests/crud/tts_evaluations/test_cron.py | 330 +++++++---- .../tests/crud/tts_evaluations/test_result.py | 120 ---- .../tests/crud/tts_evaluations/test_run.py | 79 +-- .../app/tests/models/test_tts_evaluation.py | 52 -- .../tts_evaluations/test_batch_job.py | 398 +++++-------- .../test_batch_result_processing.py | 160 ++--- .../tts_evaluations/test_constants.py | 62 -- .../services/tts_evaluations/test_dataset.py | 146 ++--- .../services/tts_evaluations/test_language.py | 75 --- .../tts_evaluations/test_sync_generation.py | 449 +++++++------- .../tts_evaluations/test_synthesizers.py | 368 ------------ backend/app/tests/utils/tts_evaluation.py | 50 +- docs/wiki/modules/evaluations.md | 7 +- 34 files changed, 1570 insertions(+), 3471 deletions(-) delete mode 100644 backend/app/services/tts_evaluations/language.py delete mode 100644 backend/app/services/tts_evaluations/synthesizers.py delete mode 100644 backend/app/tests/crud/test_language.py delete mode 100644 backend/app/tests/crud/tts_evaluations/test_batch.py delete mode 100644 backend/app/tests/crud/tts_evaluations/test_result.py delete mode 100644 backend/app/tests/models/test_tts_evaluation.py delete mode 100644 backend/app/tests/services/tts_evaluations/test_constants.py delete mode 100644 backend/app/tests/services/tts_evaluations/test_language.py delete mode 100644 backend/app/tests/services/tts_evaluations/test_synthesizers.py diff --git a/backend/app/api/docs/tts_evaluation/start_evaluation.md b/backend/app/api/docs/tts_evaluation/start_evaluation.md index 8a2107ca5..fda71407f 100644 --- a/backend/app/api/docs/tts_evaluation/start_evaluation.md +++ b/backend/app/api/docs/tts_evaluation/start_evaluation.md @@ -15,4 +15,4 @@ The evaluation will: Synthesis uses the dataset's language; datasets without a language are synthesized as `en-IN`. The response's `run_metadata.voices` lists the voice used per model. -**Supported models:** `gemini-2.5-pro-preview-tts`, `bulbul:v3` (Sarvam), `eleven_v3`, `eleven_v4` (ElevenLabs) +**Supported models:** `gemini-2.5-pro-preview-tts`, `gemini-3.1-flash-tts-preview`, `bulbul:v3` (Sarvam), `eleven_v3`, `eleven_v4` (ElevenLabs) diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index 7ef45790c..b51f59793 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -25,7 +25,10 @@ from app.services.tts_evaluations.constants import ( DEFAULT_STYLE_PROMPT, DEFAULT_VOICE_NAME, - get_tts_model_spec, + ELEVENLABS_TTS_MODELS, + ELEVENLABS_VOICE_ID, + SARVAM_SPEAKER, + SARVAM_TTS_MODELS, ) from app.utils import APIResponse, load_description @@ -84,7 +87,7 @@ def start_tts_evaluation( total_items=sample_count * len(run_create.models), ) - # Offload submission (result creation, Gemini batch, sync-model fan-out) to Celery worker + # Offload batch submission (result creation, JSONL, Gemini upload) to Celery worker trace_id = correlation_id.get() or "N/A" try: celery_task_id = start_tts_batch_submission( @@ -117,7 +120,12 @@ def start_tts_evaluation( voices: dict[str, str] = {} for model in run_create.models: - voices[model] = get_tts_model_spec(model).default_voice + if model in SARVAM_TTS_MODELS: + voices[model] = SARVAM_SPEAKER + elif model in ELEVENLABS_TTS_MODELS: + voices[model] = ELEVENLABS_VOICE_ID + else: + voices[model] = DEFAULT_VOICE_NAME return APIResponse.success_response( data=TTSEvaluationRunPublic.from_model( diff --git a/backend/app/celery/tasks/job_execution.py b/backend/app/celery/tasks/job_execution.py index d33ea4503..dc087c7c6 100644 --- a/backend/app/celery/tasks/job_execution.py +++ b/backend/app/celery/tasks/job_execution.py @@ -6,8 +6,7 @@ 9 LLM call + LLM chain (run_llm_job, run_llm_chain_job, run_response_job) 6 Fast evaluation (run_evaluation_fast_chunk, run_evaluation_fast_aggregate, run_prompt_improvement, run_evaluation_iteration_graph_step) - 2 Everything else (doctransform, collections, STT/TTS evaluation incl. - run_tts_sync_chunk, assessment) + 2 Everything else (doctransform, collections, STT/TTS evaluation, assessment) 1 Notifications (send_eval_completion_notification) Higher priority drains first; within the same priority, delivery is FIFO. @@ -27,6 +26,7 @@ from app.celery.celery_app import celery_app from app.celery.utils import gevent_timeout from app.core.config import settings +from app.services.tts_evaluations.constants import TTS_SYNC_TASK_TIME_LIMIT_SECONDS if TYPE_CHECKING: from app.services.notifications.eval_completion import ( @@ -430,20 +430,28 @@ def run_tts_result_processing( ) -@celery_app.task(bind=True, queue="default", priority=2) -@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_tts_sync_chunk") -def run_tts_sync_chunk(self, project_id: int, job_id: str, trace_id: str, **kwargs): - """Synthesize one chunk of a TTS run's sync-model results. - - Idempotent: only results still PENDING are synthesized and written, so a - redelivered chunk skips rows an earlier attempt already finished. - """ - from app.services.tts_evaluations.sync_generation import execute_tts_sync_chunk +# Same hard-minus-soft grace as the global limits, on top of the longer per-model soft limit. +@celery_app.task( + bind=True, + queue="default", + priority=2, + soft_time_limit=TTS_SYNC_TASK_TIME_LIMIT_SECONDS, + time_limit=TTS_SYNC_TASK_TIME_LIMIT_SECONDS + + settings.CELERY_TASK_TIME_LIMIT + - settings.CELERY_TASK_SOFT_TIME_LIMIT, +) +@gevent_timeout(TTS_SYNC_TASK_TIME_LIMIT_SECONDS, "run_tts_sync_generation") +def run_tts_sync_generation( + self, project_id: int, job_id: str, trace_id: str, **kwargs +): + from app.services.tts_evaluations.sync_generation import ( + execute_tts_sync_generation, + ) _set_trace(trace_id) return _run_with_otel_parent( self, - lambda: execute_tts_sync_chunk( + lambda: execute_tts_sync_generation( project_id=project_id, job_id=job_id, task_id=current_task.request.id, diff --git a/backend/app/celery/utils.py b/backend/app/celery/utils.py index 98c416da3..df9e9643c 100644 --- a/backend/app/celery/utils.py +++ b/backend/app/celery/utils.py @@ -270,21 +270,20 @@ def start_tts_result_processing( return task_id -def start_tts_sync_chunk( +def start_tts_sync_generation( project_id: int, job_id: str, trace_id: str = "N/A", **kwargs: Any ) -> str: - """Enqueue one synchronous TTS synthesis chunk (one model, a slice of result IDs).""" - from app.celery.tasks.job_execution import run_tts_sync_chunk + from app.celery.tasks.job_execution import run_tts_sync_generation task_id = _enqueue_with_trace_context( - run_tts_sync_chunk, + run_tts_sync_generation, project_id=project_id, job_id=job_id, trace_id=trace_id, **kwargs, ) logger.info( - f"[start_tts_sync_chunk] Started job {job_id} with Celery task {task_id}" + f"[start_tts_sync_generation] Started job {job_id} with Celery task {task_id}" ) return task_id diff --git a/backend/app/crud/language.py b/backend/app/crud/language.py index 4d6fc3004..765e50542 100644 --- a/backend/app/crud/language.py +++ b/backend/app/crud/language.py @@ -36,9 +36,3 @@ def get_language_by_locale(session: Session, locale: str) -> Optional[Language]: """Retrieve a language by its locale code.""" statement = select(Language).where(Language.locale == locale) return session.exec(statement).first() - - -def get_language_locale_by_id(session: Session, language_id: int) -> str | None: - """Return a language's ISO 639-1 locale, or None if the ID doesn't exist.""" - statement = select(Language.locale).where(Language.id == language_id) - return session.exec(statement).first() diff --git a/backend/app/crud/tts_evaluations/__init__.py b/backend/app/crud/tts_evaluations/__init__.py index bb476b7d9..fcced112f 100644 --- a/backend/app/crud/tts_evaluations/__init__.py +++ b/backend/app/crud/tts_evaluations/__init__.py @@ -1,31 +1,27 @@ """TTS Evaluation CRUD operations.""" -from .batch import TTSBatchSubmissionError, start_tts_evaluation_batch +from .batch import start_tts_evaluation_batch from .cron import poll_all_pending_tts_evaluations from .dataset import ( create_tts_dataset, get_tts_dataset_by_id, list_tts_datasets, ) -from .result import ( - bulk_update_pending_tts_results, - create_tts_results, - get_results_by_run_id, - get_tts_result_by_id, - list_pending_tts_results_by_ids, - update_tts_human_feedback, -) from .run import ( create_tts_run, - finalize_tts_run_status, get_tts_run_by_id, list_tts_runs, update_tts_run, ) +from .result import ( + create_tts_results, + get_tts_result_by_id, + get_results_by_run_id, + update_tts_human_feedback, +) __all__ = [ # Batch - "TTSBatchSubmissionError", "start_tts_evaluation_batch", # Cron "poll_all_pending_tts_evaluations", @@ -35,15 +31,12 @@ "list_tts_datasets", # Run "create_tts_run", - "finalize_tts_run_status", "get_tts_run_by_id", "list_tts_runs", "update_tts_run", # Result - "bulk_update_pending_tts_results", "create_tts_results", "get_tts_result_by_id", "get_results_by_run_id", - "list_pending_tts_results_by_ids", "update_tts_human_feedback", ] diff --git a/backend/app/crud/tts_evaluations/batch.py b/backend/app/crud/tts_evaluations/batch.py index 639cc8cb5..3a9abb543 100644 --- a/backend/app/crud/tts_evaluations/batch.py +++ b/backend/app/crud/tts_evaluations/batch.py @@ -11,38 +11,33 @@ create_tts_batch_requests, start_batch_job, ) +from app.models.batch_job import BatchJobType from app.crud.tts_evaluations.result import ( get_pending_results_for_run, update_tts_result, ) from app.crud.tts_evaluations.run import update_tts_run from app.models import EvaluationRun -from app.models.batch_job import BatchJobType from app.models.job import JobStatus from app.models.tts_evaluation import TTSResult from app.services.tts_evaluations.constants import ( DEFAULT_STYLE_PROMPT, - TTSExecutionModeEnum, - get_tts_model_spec, + DEFAULT_TTS_MODEL, + DEFAULT_VOICE_NAME, ) logger = logging.getLogger(__name__) -class TTSBatchSubmissionError(Exception): - """Raised when no batch-mode model could be submitted to its provider.""" - - def start_tts_evaluation_batch( *, session: Session, run: EvaluationRun, results: list[TTSResult], - models: list[str], org_id: int, project_id: int, ) -> dict[str, Any]: - """Submit Gemini batch jobs for the batch-mode models of a TTS evaluation. + """Submit Gemini batch jobs for TTS evaluation. Submits one batch job per model. Each batch job is tracked via its config containing evaluation_run_id and tts_provider. @@ -50,8 +45,7 @@ def start_tts_evaluation_batch( Args: session: Database session run: The evaluation run record - results: TTSResult records (loaded); rows for non-batch models are ignored - models: Batch-mode models to submit (sync models must not be passed) + results: List of TTSResult records (contains sample_text) org_id: Organization ID project_id: Project ID @@ -59,50 +53,47 @@ def start_tts_evaluation_batch( dict: Result with batch job information per model Raises: - ValueError: If a non-batch model is passed - TTSBatchSubmissionError: If batch submission fails for every model + Exception: If batch submission fails for all models """ - for model in models: - if get_tts_model_spec(model).execution_mode != TTSExecutionModeEnum.BATCH: - raise ValueError(f"Model '{model}' is not a batch-mode TTS model") - - # Capture plain values up front: start_batch_job commits, which expires every - # ORM instance in the session, and touching them later would issue a reload - # per row (and reopen a transaction right before the next network call). - run_id = run.id - texts_by_model: dict[str, list[str]] = {} - keys_by_model: dict[str, list[str]] = {} - for result in results: - texts_by_model.setdefault(result.provider, []).append(result.sample_text) - keys_by_model.setdefault(result.provider, []).append(str(result.id)) + models = run.providers or [DEFAULT_TTS_MODEL] logger.info( f"[start_tts_evaluation_batch] Starting batch submission | " - f"run_id: {run_id}, result_count: {len(results)}, " + f"run_id: {run.id}, result_count: {len(results)}, " f"models: {models}" ) + # Initialize Gemini client gemini_client = GeminiClient.from_credentials( session=session, org_id=org_id, project_id=project_id, ) + # Collect unique sample texts and their result IDs per model + # Group results by model to build per-model batch requests + results_by_model: dict[str, list[TTSResult]] = {} + for result in results: + results_by_model.setdefault(result.provider, []).append(result) + + # Submit one batch job per model batch_jobs: dict[str, Any] = {} first_batch_job_id: int | None = None - submitted_count = 0 for model in models: - texts = texts_by_model.get(model, []) - if not texts: + model_results = results_by_model.get(model, []) + if not model_results: continue - voice_name = get_tts_model_spec(model).default_voice + texts = [r.sample_text for r in model_results] + keys = [str(r.id) for r in model_results] + + # Create JSONL batch requests for TTS jsonl_data = create_tts_batch_requests( texts=texts, - voice_name=voice_name, + voice_name=DEFAULT_VOICE_NAME, style_prompt=DEFAULT_STYLE_PROMPT, - keys=keys_by_model[model], + keys=keys, ) model_path = f"models/{model}" @@ -114,7 +105,7 @@ def start_tts_evaluation_batch( batch_job = start_batch_job( session=session, provider=batch_provider, - provider_name=get_tts_model_spec(model).provider.value, + provider_name="google-aistudio", job_type=BatchJobType.TTS_EVALUATION, organization_id=org_id, project_id=project_id, @@ -122,8 +113,8 @@ def start_tts_evaluation_batch( config={ "model": model, "tts_provider": model, - "evaluation_run_id": run_id, - "voice_name": voice_name, + "evaluation_run_id": run.id, + "voice_name": DEFAULT_VOICE_NAME, "style_prompt": DEFAULT_STYLE_PROMPT, }, ) @@ -132,25 +123,23 @@ def start_tts_evaluation_batch( "batch_job_id": batch_job.id, "provider_batch_id": batch_job.provider_batch_id, } - submitted_count += len(texts) if first_batch_job_id is None: first_batch_job_id = batch_job.id logger.info( f"[start_tts_evaluation_batch] Batch job created | " - f"run_id: {run_id}, model: {model}, " + f"run_id: {run.id}, model: {model}, " f"batch_job_id: {batch_job.id}" ) except Exception as e: logger.error( f"[start_tts_evaluation_batch] Failed to submit batch | " - f"run_id: {run_id}, model: {model}, error: {str(e)}", - exc_info=True, + f"model: {model}, error: {str(e)}" ) pending = get_pending_results_for_run( - session=session, run_id=run_id, provider=model + session=session, run_id=run.id, provider=model ) for result in pending: update_tts_result( @@ -162,25 +151,25 @@ def start_tts_evaluation_batch( session.commit() if not batch_jobs: - raise TTSBatchSubmissionError("Batch submission failed for all models") + raise Exception("Batch submission failed for all models") # Link first batch job to the evaluation run (for pending run detection) update_tts_run( session=session, - run_id=run_id, + run_id=run.id, status="processing", batch_job_id=first_batch_job_id, ) logger.info( f"[start_tts_evaluation_batch] Batch submission complete | " - f"run_id: {run_id}, models_submitted: {list(batch_jobs.keys())}, " - f"result_count: {submitted_count}" + f"run_id: {run.id}, models_submitted: {list(batch_jobs.keys())}, " + f"result_count: {len(results)}" ) return { "success": True, - "run_id": run_id, + "run_id": run.id, "batch_jobs": batch_jobs, - "result_count": submitted_count, + "result_count": len(results), } diff --git a/backend/app/crud/tts_evaluations/cron.py b/backend/app/crud/tts_evaluations/cron.py index 5bd988064..a451341b8 100644 --- a/backend/app/crud/tts_evaluations/cron.py +++ b/backend/app/crud/tts_evaluations/cron.py @@ -1,39 +1,56 @@ """Cron processing functions for TTS evaluations. -This module provides functions that are called periodically to process -pending TTS evaluations - polling batch status and dispatching result -processing to Celery workers. - -Follows the same pattern as STT evaluations: single query to fetch all -processing runs, grouped by project_id for credential management. +The cron is the only code that sets a TTS run's final status. Each tick it polls +Gemini batches (dispatching result processing on success), fails the rows of +batches that ended badly or of sync models whose worker is gone, then recomputes +the run status from its result rows via `finalize_tts_run_status`. """ import logging +from datetime import timedelta from typing import Any -from sqlmodel import Session +from sqlmodel import Session, select from app.celery.utils import start_tts_result_processing -from app.core.batch import GeminiBatchProvider +from app.core.batch import BatchJobState, GeminiBatchProvider, GeminiClient +from app.core.util import now from app.crud.evaluations.cron_utils import ( + TERMINAL_STATES, get_batch_jobs_for_run, + make_empty_summary, make_poll_result, - poll_all_pending_evaluations_by_type, poll_batch_jobs, ) -from app.crud.tts_evaluations.result import get_pending_results_for_run -from app.crud.tts_evaluations.run import finalize_tts_run_status, update_tts_run +from app.crud.tts_evaluations.result import ( + get_pending_results_for_run, + update_tts_result, +) +from app.crud.tts_evaluations.run import finalize_tts_run_status from app.models import EvaluationRun from app.models.batch_job import BatchJob, BatchJobType +from app.models.job import JobStatus from app.models.stt_evaluation import EvaluationType +from app.services.tts_evaluations.constants import ( + GEMINI_TTS_MODELS, + SYNC_TTS_MODELS, + TTS_SYNC_STALE_AFTER_SECONDS, +) logger = logging.getLogger(__name__) +EVAL_TYPE_LABEL = "tts" +SUCCESS_ACTIONS = ("completed", "dispatched") + async def poll_all_pending_tts_evaluations( session: Session, ) -> dict[str, Any]: - """Poll all pending TTS evaluations across all organizations. + """Poll every processing TTS run, including runs with only Sarvam/ElevenLabs models. + + Not built on `poll_all_pending_evaluations_by_type`: that helper skips runs + without a `batch_job_id` and fails a whole project when its Gemini client + can't be built, which would break runs that don't use Gemini at all. Args: session: Database session @@ -41,15 +58,68 @@ async def poll_all_pending_tts_evaluations( Returns: Summary dict with total, processed, failed, still_processing counts """ - return await poll_all_pending_evaluations_by_type( - session, - eval_type="tts", - eval_type_enum_value=EvaluationType.TTS.value, - update_run_fn=update_tts_run, - poll_run_fn=poll_tts_run, - success_actions=("completed", "processed", "dispatched"), + logger.info("[poll_all_pending_tts_evaluations] Starting TTS evaluation polling") + + statement = select(EvaluationRun).where( + EvaluationRun.type == EvaluationType.TTS.value, + EvaluationRun.status == "processing", + ) + pending_runs = list(session.exec(statement).all()) + + if not pending_runs: + logger.info("[poll_all_pending_tts_evaluations] No pending TTS runs found") + return make_empty_summary() + + all_results: list[dict[str, Any]] = [] + total_processed = 0 + total_failed = 0 + total_still_processing = 0 + + for run in pending_runs: + try: + result = await poll_tts_run( + session=session, run=run, org_id=run.organization_id + ) + except Exception as e: + # Left in `processing` so the next tick retries; a transient poll error + # must not fail models that may still succeed. + logger.error( + f"[poll_all_pending_tts_evaluations] Failed to poll TTS run | " + f"run_id={run.id} | {e}", + exc_info=True, + ) + session.rollback() + result = make_poll_result( + run=run, + eval_type=EVAL_TYPE_LABEL, + previous_status=run.status, + current_status=run.status, + action="no_change", + error=f"Polling failed: {str(e)}", + ) + all_results.append(result) + + if result["action"] in SUCCESS_ACTIONS: + total_processed += 1 + elif result["action"] == "failed": + total_failed += 1 + else: + total_still_processing += 1 + + logger.info( + f"[poll_all_pending_tts_evaluations] Polling summary | " + f"processed={total_processed} | failed={total_failed} | " + f"still_processing={total_still_processing}" ) + return { + "total": len(pending_runs), + "processed": total_processed, + "failed": total_failed, + "still_processing": total_still_processing, + "details": all_results, + } + def _dispatch_tts_result_processing( run: EvaluationRun, @@ -89,18 +159,13 @@ def _dispatch_tts_result_processing( async def poll_tts_run( session: Session, run: EvaluationRun, - batch_provider: GeminiBatchProvider, org_id: int, ) -> dict[str, Any]: - """Poll a single TTS evaluation run's batch status. - - Finds all batch jobs for this run (one per provider) and polls each. - When a batch reaches SUCCEEDED, dispatches result processing to Celery. + """Advance a single TTS run and recompute its status. Args: session: Database session run: The evaluation run to poll - batch_provider: Initialized GeminiBatchProvider org_id: Organization ID Returns: @@ -112,91 +177,154 @@ async def poll_tts_run( logger.info(f"{log_prefix} Polling run") previous_status = run.status - - batch_jobs = get_batch_jobs_for_run( - session=session, run=run, job_type=BatchJobType.TTS_EVALUATION - ) - - if not batch_jobs: - logger.warning(f"{log_prefix} No batch jobs found") - update_tts_run( - session=session, - run_id=run.id, - status="failed", - error_message="No batch jobs found", - ) - return make_poll_result( - run=run, - eval_type="tts", - previous_status=previous_status, - current_status="failed", - action="failed", - error="No batch jobs found", - ) - - async def _on_batch_succeeded(batch_job: BatchJob, provider_name: str) -> bool: - _dispatch_tts_result_processing(run, batch_job, org_id, provider_name) - return True - - async def _on_already_succeeded(batch_job: BatchJob, provider_name: str) -> bool: - pending = get_pending_results_for_run( - session=session, run_id=run.id, provider=provider_name - ) - if pending: - logger.info( - f"{log_prefix} Dispatching reprocessing for " - f"{len(pending)} pending results | " - f"batch_job_id={batch_job.id}" + models = run.providers or [] + batch_models = [model for model in models if model in GEMINI_TTS_MODELS] + sync_models = [model for model in models if model in SYNC_TTS_MODELS] + # Both timestamps are naive UTC (`now()` strips tzinfo before storing). + is_stale = now() - run.inserted_at > timedelta(seconds=TTS_SYNC_STALE_AFTER_SECONDS) + any_dispatched = False + + batch_jobs: list[BatchJob] = [] + if batch_models: + # A job whose creation failed has no provider_batch_id and nothing to poll; + # submission already marked its rows FAILED. + batch_jobs = [ + batch_job + for batch_job in get_batch_jobs_for_run( + session=session, run=run, job_type=BatchJobType.TTS_EVALUATION + ) + if batch_job.provider_batch_id + ] + + # Submission marks rows FAILED when a batch can't be created, so leftover PENDING + # rows without a batch job mean the submission task died part-way. + if batch_models and not batch_jobs and is_stale: + logger.warning(f"{log_prefix} No batch jobs found for Gemini models") + for model in batch_models: + _fail_pending_results( + session=session, + run_id=run.id, + model=model, + error_message="No batch jobs found", ) - _dispatch_tts_result_processing(run, batch_job, org_id, provider_name) - return True - return False - - result = await poll_batch_jobs( - session=session, - batch_jobs=batch_jobs, - batch_provider=batch_provider, - provider_config_key="tts_provider", - log_prefix=log_prefix, - on_succeeded=_on_batch_succeeded, - on_already_succeeded=_on_already_succeeded, - ) - - if not result.all_terminal: - return make_poll_result( - run=run, - eval_type="tts", - previous_status=previous_status, - current_status=run.status, - action="no_change", - ) - # If we dispatched processing to Celery, keep the run as "processing". - # The Celery task will finalize the run status when done. - if result.any_dispatched: - return make_poll_result( - run=run, - eval_type="tts", - previous_status=previous_status, - current_status="processing", - action="dispatched", - ) + if batch_jobs: + batch_provider: GeminiBatchProvider | None = None + try: + gemini_client = GeminiClient.from_credentials( + session=session, org_id=org_id, project_id=run.project_id + ) + batch_provider = GeminiBatchProvider(client=gemini_client.client) + except Exception as client_err: + logger.error( + f"{log_prefix} Failed to get Gemini client | error={client_err}" + ) + for model in batch_models: + _fail_pending_results( + session=session, + run_id=run.id, + model=model, + error_message=f"Gemini client initialization failed: {str(client_err)}", + ) + + if batch_provider is not None: + + async def _on_batch_succeeded( + batch_job: BatchJob, provider_name: str + ) -> bool: + _dispatch_tts_result_processing(run, batch_job, org_id, provider_name) + return True + + async def _on_already_succeeded( + batch_job: BatchJob, provider_name: str + ) -> bool: + pending = get_pending_results_for_run( + session=session, run_id=run.id, provider=provider_name + ) + if not pending: + return False + logger.info( + f"{log_prefix} Dispatching reprocessing for " + f"{len(pending)} pending results | batch_job_id={batch_job.id}" + ) + _dispatch_tts_result_processing(run, batch_job, org_id, provider_name) + return True + + poll_result = await poll_batch_jobs( + session=session, + batch_jobs=batch_jobs, + batch_provider=batch_provider, + provider_config_key="tts_provider", + log_prefix=log_prefix, + on_succeeded=_on_batch_succeeded, + on_already_succeeded=_on_already_succeeded, + ) + any_dispatched = poll_result.any_dispatched + + for batch_job in batch_jobs: + if batch_job.provider_status not in TERMINAL_STATES: + continue + if batch_job.provider_status == BatchJobState.SUCCEEDED.value: + continue + _fail_pending_results( + session=session, + run_id=run.id, + model=batch_job.config.get("tts_provider", ""), + error_message=( + f"Gemini batch ended: " + f"{batch_job.error_message or batch_job.provider_status}" + ), + ) + + # Sync workers write rows as they go; past the stale window the worker is gone. + if is_stale: + for model in sync_models: + _fail_pending_results( + session=session, + run_id=run.id, + model=model, + error_message="Synthesis did not complete", + ) - # All batch jobs are done and no dispatching needed - finalize the run - batch_error = "; ".join(result.errors) if result.any_failed else None - finalized_run = finalize_tts_run_status( - session=session, run_id=run.id, error_message=batch_error - ) + finalized_run = finalize_tts_run_status(session=session, run_id=run.id) final_status = finalized_run.status if finalized_run else run.status - error_message = finalized_run.error_message if finalized_run else batch_error + error_message = finalized_run.error_message if finalized_run else None - action = "completed" if not result.any_failed else "failed" + if final_status in ("completed", "failed"): + action = final_status + elif any_dispatched: + action = "dispatched" + else: + action = "no_change" return make_poll_result( run=run, - eval_type="tts", + eval_type=EVAL_TYPE_LABEL, previous_status=previous_status, current_status=final_status, action=action, error=error_message, ) + + +def _fail_pending_results( + *, session: Session, run_id: int, model: str, error_message: str +) -> None: + """Mark one model's PENDING rows of a run FAILED.""" + pending = get_pending_results_for_run( + session=session, run_id=run_id, provider=model + ) + if not pending: + return + logger.warning( + f"[_fail_pending_results] Failing pending results | run_id={run_id}, " + f"model={model}, count={len(pending)}, error={error_message}" + ) + for result in pending: + update_tts_result( + session=session, + result_id=result.id, + status=JobStatus.FAILED.value, + error_message=error_message, + ) + session.commit() diff --git a/backend/app/crud/tts_evaluations/result.py b/backend/app/crud/tts_evaluations/result.py index 67a4b6d61..6ebf872ab 100644 --- a/backend/app/crud/tts_evaluations/result.py +++ b/backend/app/crud/tts_evaluations/result.py @@ -3,13 +3,13 @@ import logging from typing import Any -from sqlmodel import Session, col, func, select +from sqlmodel import Session, func, select from app.core.cloud.storage import CloudStorage from app.core.exception_handlers import HTTPException from app.core.util import now from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResult, TTSResultPublic, TTSResultUpdate +from app.models.tts_evaluation import TTSResult, TTSResultPublic logger = logging.getLogger(__name__) @@ -285,83 +285,6 @@ def get_pending_results_for_run( return list(session.exec(statement).all()) -def list_pending_tts_results_by_ids( - *, - session: Session, - run_id: int, - result_ids: list[int], -) -> list[TTSResult]: - """List the still-PENDING results of a run among the given IDs. - - Filtering on PENDING is what makes a redelivered sync chunk a no-op for rows - an earlier attempt already finished. - """ - if not result_ids: - return [] - - statement = ( - select(TTSResult) - .where( - TTSResult.evaluation_run_id == run_id, - TTSResult.status == JobStatus.PENDING.value, - col(TTSResult.id).in_(result_ids), - ) - .order_by(TTSResult.id) - ) - - return list(session.exec(statement).all()) - - -def bulk_update_pending_tts_results( - *, - session: Session, - updates: list[TTSResultUpdate], -) -> int: - """Apply terminal outcomes to results that are still PENDING; commit once. - - Rows already moved out of PENDING (by a concurrent or earlier attempt) are - left untouched so a late writer can never clobber a finished result. - - Returns: - int: Number of rows updated - """ - if not updates: - return 0 - - updates_by_id: dict[int, TTSResultUpdate] = {} - for update in updates: - updates_by_id[update.result_id] = update - - statement = select(TTSResult).where( - col(TTSResult.id).in_(list(updates_by_id.keys())), - TTSResult.status == JobStatus.PENDING.value, - ) - rows = session.exec(statement).all() - - timestamp = now() - for row in rows: - update = updates_by_id[row.id] - row.status = update.status.value - if update.object_store_url is not None: - row.object_store_url = update.object_store_url - if update.metadata is not None: - row.metadata_ = update.metadata - if update.error_message is not None: - row.error_message = update.error_message - row.updated_at = timestamp - session.add(row) - - session.commit() - - skipped = len(updates_by_id) - len(rows) - logger.info( - f"[bulk_update_pending_tts_results] Results updated | " - f"updated: {len(rows)}, skipped_not_pending: {skipped}" - ) - - return len(rows) - - def count_results_by_status( *, session: Session, diff --git a/backend/app/crud/tts_evaluations/run.py b/backend/app/crud/tts_evaluations/run.py index cd53e0575..2f9831288 100644 --- a/backend/app/crud/tts_evaluations/run.py +++ b/backend/app/crud/tts_evaluations/run.py @@ -231,18 +231,8 @@ def finalize_tts_run_status( ) -> EvaluationRun | None: """Recompute a TTS run's status from its result rows and persist it. - The run row is locked first so concurrent finalizers (sync chunks, Gemini - result processing, cron) serialize: whichever commits last counts every - result the others already committed, so a run can't be stranded in - `processing` after its final result lands. - - Args: - session: Database session - run_id: Run ID - error_message: Overrides the derived "N synthesis(es) failed" message - - Returns: - EvaluationRun | None: Updated run, or None if it doesn't exist + `processing` while any row is PENDING, `failed` when no row succeeded (every + model failed), otherwise `completed`. Only the cron calls this. """ statement = ( select(EvaluationRun).where(EvaluationRun.id == run_id).with_for_update() @@ -255,9 +245,15 @@ def finalize_tts_run_status( status_counts = count_results_by_status(session=session, run_id=run_id) pending = status_counts.get(JobStatus.PENDING.value, 0) + success = status_counts.get(JobStatus.SUCCESS.value, 0) failed = status_counts.get(JobStatus.FAILED.value, 0) - run.status = "completed" if pending == 0 else "processing" + if pending > 0: + run.status = "processing" + elif success == 0: + run.status = "failed" + else: + run.status = "completed" if error_message is None and failed > 0: error_message = f"{failed} synthesis(es) failed" @@ -272,7 +268,8 @@ def finalize_tts_run_status( logger.info( f"[finalize_tts_run_status] Run finalized | run_id: {run_id}, " - f"status: {run.status}, pending: {pending}, failed: {failed}" + f"status: {run.status}, pending: {pending}, success: {success}, " + f"failed: {failed}" ) return run diff --git a/backend/app/models/tts_evaluation.py b/backend/app/models/tts_evaluation.py index e3e3959e7..a0bbdb054 100644 --- a/backend/app/models/tts_evaluation.py +++ b/backend/app/models/tts_evaluation.py @@ -13,10 +13,7 @@ from app.core.util import now from app.models.job import JobStatus -from app.services.tts_evaluations.constants import ( - DEFAULT_TTS_MODEL, - SUPPORTED_TTS_MODELS, -) +from app.services.tts_evaluations.constants import SUPPORTED_TTS_MODELS if TYPE_CHECKING: from app.models import EvaluationDataset, EvaluationRun @@ -144,16 +141,6 @@ class TTSResult(SQLModel, table=True): ) -class TTSResultUpdate(BaseModel): - """Terminal outcome for one synthesis, applied only while the row is still PENDING.""" - - result_id: int - status: JobStatus - object_store_url: str | None = None - metadata: dict[str, Any] | None = None - error_message: str | None = None - - class TTSSampleCreate(BaseModel): """Request model for a single TTS sample.""" @@ -298,7 +285,7 @@ class TTSEvaluationRunCreate(BaseModel): run_name: str = Field(..., description="Name for this evaluation run", min_length=1) dataset_id: int = Field(..., description="ID of the TTS dataset to evaluate") models: list[str] = Field( - default_factory=lambda: [DEFAULT_TTS_MODEL], + default_factory=lambda: ["gemini-2.5-pro-preview-tts"], description="List of TTS models to use", min_length=1, ) diff --git a/backend/app/services/tts_evaluations/batch_job.py b/backend/app/services/tts_evaluations/batch_job.py index eebe87de4..97147b0b8 100644 --- a/backend/app/services/tts_evaluations/batch_job.py +++ b/backend/app/services/tts_evaluations/batch_job.py @@ -1,11 +1,10 @@ """Celery task function for TTS evaluation submission. -Creates result rows, submits batch-mode models (Gemini) to the provider Batch API -and fans sync-mode models (Sarvam, ElevenLabs) out to chunked Celery tasks. +Gemini models are submitted to the Gemini Batch API; each Sarvam/ElevenLabs model +gets its own `run_tts_sync_generation` Celery task. The cron sets the final run status. """ import logging -from dataclasses import dataclass from typing import Any from asgi_correlation_id import correlation_id @@ -14,46 +13,30 @@ from gevent import Timeout from sqlmodel import Session -from app.celery.utils import start_tts_sync_chunk -from app.core.cloud.storage import CloudStorage, get_cloud_storage +from app.celery.utils import start_tts_sync_generation +from app.core.cloud.storage import get_cloud_storage from app.core.db import engine -from app.crud.tts_evaluations.batch import ( - TTSBatchSubmissionError, - start_tts_evaluation_batch, -) +from app.crud.tts_evaluations.batch import start_tts_evaluation_batch from app.crud.tts_evaluations.dataset import get_tts_dataset_by_id from app.crud.tts_evaluations.result import ( - bulk_update_pending_tts_results, create_tts_results, get_pending_results_for_run, + update_tts_result, ) -from app.crud.tts_evaluations.run import ( - finalize_tts_run_status, - get_tts_run_by_id, - update_tts_run, -) +from app.crud.tts_evaluations.run import get_tts_run_by_id, update_tts_run +from app.models import Language from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResult, TTSResultUpdate +from app.models.tts_evaluation import TTSResult from app.services.tts_evaluations.constants import ( - KAAPI_TAG, - TTS_SYNC_CHUNK_SIZE, - split_models_by_execution_mode, + DEFAULT_TTS_LANGUAGE, + GEMINI_TTS_MODELS, + SYNC_TTS_MODELS, + LOCALE_TO_BCP47, ) -from app.services.tts_evaluations.dataset import load_sample_texts -from app.services.tts_evaluations.language import resolve_dataset_tts_language_code +from app.services.tts_evaluations.dataset import get_sample_texts_from_dataset logger = logging.getLogger(__name__) -DEFAULT_TRACE_ID = "N/A" - - -@dataclass(frozen=True) -class _SubmissionContext: - dataset_id: int - object_store_url: str | None - language_code: str - storage: CloudStorage | None - def execute_batch_submission( project_id: int, @@ -67,8 +50,9 @@ def execute_batch_submission( ) -> dict[str, Any]: """Execute TTS evaluation submission in a Celery worker. - Handles result record creation, Gemini batch submission for batch-mode - models and chunked Celery fan-out for sync-mode models. + Creates result rows, submits the Gemini batch and queues one sync task per + Sarvam/ElevenLabs model. A model that fails to start has its rows marked + FAILED; the other models carry on. Args: project_id: Project ID @@ -80,9 +64,11 @@ def execute_batch_submission( models: List of TTS model names to evaluate Returns: - dict: Result summary with batch job and sync chunk info + dict: Result summary with batch job info """ run_id = int(job_id) + batch_models = [model for model in models if model in GEMINI_TTS_MODELS] + sync_models = [model for model in models if model in SYNC_TTS_MODELS] logger.info( f"[execute_batch_submission] Starting | " @@ -91,31 +77,50 @@ def execute_batch_submission( ) try: - batch_models, sync_models = split_models_by_execution_mode(models) - - context = _load_submission_context( - run_id=run_id, - organization_id=organization_id, - project_id=project_id, - dataset_id=dataset_id, - ) - if isinstance(context, dict): - return context - - # S3 read happens with no session open. - sample_texts: list[str] = [] - if context.object_store_url and context.storage is not None: - sample_texts = load_sample_texts( - storage=context.storage, - object_store_url=context.object_store_url, - dataset_id=context.dataset_id, + with Session(engine) as session: + run = get_tts_run_by_id( + session=session, + run_id=run_id, + org_id=organization_id, + project_id=project_id, ) - else: - logger.warning( - f"[execute_batch_submission] No object_store_url | " - f"run_id: {run_id}, dataset_id: {dataset_id}" + if not run: + logger.warning( + f"[execute_batch_submission] Run not found | run_id: {run_id}" + ) + return {"success": False, "error": "Run not found"} + + dataset = get_tts_dataset_by_id( + session=session, + dataset_id=dataset_id, + org_id=organization_id, + project_id=project_id, ) + if not dataset: + logger.warning( + f"[execute_batch_submission] Dataset not found | " + f"run_id: {run_id}, dataset_id: {dataset_id}" + ) + update_tts_run( + session=session, + run_id=run_id, + status="failed", + error_message="Dataset not found", + ) + return {"success": False, "error": "Dataset not found"} + + language_code = DEFAULT_TTS_LANGUAGE + if dataset.language_id is not None: + language = session.get(Language, dataset.language_id) + if language: + language_code = LOCALE_TO_BCP47.get( + language.locale, DEFAULT_TTS_LANGUAGE + ) + storage = get_cloud_storage(session=session, project_id=project_id) + + # Read from S3 storage with no db session open + sample_texts = get_sample_texts_from_dataset(storage, dataset) if not sample_texts: logger.warning( f"[execute_batch_submission] No samples found | " @@ -126,24 +131,8 @@ def execute_batch_submission( ) return {"success": False, "error": "No samples found"} - batch_result: dict[str, Any] = {} - sync_result_ids: dict[str, list[int]] = {} - result_count = 0 - with Session(engine) as session: - run = get_tts_run_by_id( - session=session, - run_id=run_id, - org_id=organization_id, - project_id=project_id, - ) - if not run: - logger.warning( - f"[execute_batch_submission] Run not found | run_id: {run_id}" - ) - return {"success": False, "error": "Run not found"} - - create_tts_results( + results = create_tts_results( session=session, sample_texts=sample_texts, evaluation_run_id=run_id, @@ -151,73 +140,93 @@ def execute_batch_submission( project_id=project_id, models=models, ) - # One SELECT reloads every row create_tts_results' commit expired, - # instead of one lazy reload per row below. - results = get_pending_results_for_run(session=session, run_id=run_id) result_count = len(results) + update_tts_run(session=session, run_id=run_id, status="processing") + + # model -> error; these models' rows are marked FAILED once all models are started. + failed_models: dict[str, str] = {} - batch_results: list[TTSResult] = [] - for result in results: - if result.provider in batch_models: - batch_results.append(result) - elif result.provider in sync_models: - sync_result_ids.setdefault(result.provider, []).append(result.id) + # Queue sync tasks first: queuing is instant, while Gemini submission + # (JSONL upload + batch create) takes seconds per model. + trace_id = correlation_id.get() or "N/A" + for model in sync_models: + try: + start_tts_sync_generation( + project_id=project_id, + job_id=str(run_id), + trace_id=trace_id, + organization_id=organization_id, + model=model, + language_code=language_code, + ) + except Exception as e: + logger.error( + f"[execute_batch_submission] Failed to queue sync generation | " + f"run_id: {run_id}, model: {model}, error: {str(e)}", + exc_info=True, + ) + failed_models[model] = f"Failed to queue synthesis: {str(e)}" - if batch_models: + batch_result: dict[str, Any] = {} + if batch_models: + with Session(engine) as session: + batch_results: list[TTSResult] = [] + for model in batch_models: + batch_results.extend( + get_pending_results_for_run( + session=session, run_id=run_id, provider=model + ) + ) try: batch_result = start_tts_evaluation_batch( session=session, run=run, results=batch_results, - models=batch_models, org_id=organization_id, project_id=project_id, ) - except TTSBatchSubmissionError as e: - # Batch rows are already FAILED; sync models can still finish the run. - if not sync_models: - raise - logger.warning( - f"[execute_batch_submission] Batch models failed, " - f"continuing with sync models | run_id: {run_id}, " - f"error: {str(e)}" + except Exception as e: + logger.error( + f"[execute_batch_submission] Gemini batch submission failed | " + f"run_id: {run_id}, error: {str(e)}", + exc_info=True, ) - - if sync_models: - # Must precede the enqueue: a fast chunk finalizing the run would - # otherwise be overwritten back to "processing" and strand it. - update_tts_run(session=session, run_id=run_id, status="processing") - - sync_chunks: dict[str, list[str]] = {} - if sync_result_ids: - sync_chunks = _dispatch_sync_chunks( - run_id=run_id, - organization_id=organization_id, - project_id=project_id, - result_ids_by_model=sync_result_ids, - language_code=context.language_code, - ) + for model in batch_models: + failed_models[model] = f"Batch submission failed: {str(e)}" + + if failed_models: + with Session(engine) as session: + for model, error_message in failed_models.items(): + pending = get_pending_results_for_run( + session=session, run_id=run_id, provider=model + ) + for result in pending: + update_tts_result( + session=session, + result_id=result.id, + status=JobStatus.FAILED.value, + error_message=error_message, + ) + session.commit() logger.info( f"[execute_batch_submission] Submitted | run_id: {run_id}, " f"batch_jobs: {list(batch_result.get('batch_jobs', {}).keys())}, " - f"sync_models: {list(sync_chunks.keys())}" + f"sync_models: {sync_models}, failed_models: {list(failed_models.keys())}" ) return { "success": True, "run_id": run_id, "batch_jobs": batch_result.get("batch_jobs", {}), - "sync_chunks": sync_chunks, "result_count": result_count, } except (Timeout, SoftTimeLimitExceeded): - timeout_err = TimeoutError("Task exceeded soft time limit") logger.warning( f"[execute_batch_submission] TTS batch submission timed out | run_id={run_id}" ) - _mark_run_failed(run_id=run_id, error_message=str(timeout_err)) + _mark_run_failed(run_id=run_id, error_message="Task exceeded soft time limit") raise except Exception as e: @@ -230,142 +239,6 @@ def execute_batch_submission( return {"success": False, "error": str(e)} -def _load_submission_context( - *, - run_id: int, - organization_id: int, - project_id: int, - dataset_id: int, -) -> _SubmissionContext | dict[str, Any]: - """Validate run + dataset and capture what the session-free steps need. - - Returns a failure dict (already persisted on the run) when validation fails. - """ - with Session(engine) as session: - run = get_tts_run_by_id( - session=session, - run_id=run_id, - org_id=organization_id, - project_id=project_id, - ) - if not run: - logger.warning( - f"[_load_submission_context] Run not found | run_id: {run_id}" - ) - return {"success": False, "error": "Run not found"} - - dataset = get_tts_dataset_by_id( - session=session, - dataset_id=dataset_id, - org_id=organization_id, - project_id=project_id, - ) - if not dataset: - logger.warning( - f"[_load_submission_context] Dataset not found | " - f"run_id: {run_id}, dataset_id: {dataset_id}" - ) - update_tts_run( - session=session, - run_id=run_id, - status="failed", - error_message="Dataset not found", - ) - return {"success": False, "error": "Dataset not found"} - - object_store_url = dataset.object_store_url - language_code = resolve_dataset_tts_language_code( - session=session, language_id=dataset.language_id - ) - storage = ( - get_cloud_storage(session=session, project_id=project_id) - if object_store_url - else None - ) - - logger.info( - f"[_load_submission_context] Context loaded | run_id: {run_id}, " - f"dataset_id: {dataset_id}, language: {language_code}" - ) - - return _SubmissionContext( - dataset_id=dataset_id, - object_store_url=object_store_url, - language_code=language_code, - storage=storage, - ) - - -def _dispatch_sync_chunks( - *, - run_id: int, - organization_id: int, - project_id: int, - result_ids_by_model: dict[str, list[int]], - language_code: str, -) -> dict[str, list[str]]: - """Enqueue one `run_tts_sync_chunk` per (model, slice of result IDs). - - A chunk that can't be enqueued has its results marked FAILED, since no - other path would ever synthesize them. - - Returns: - dict[str, list[str]]: Celery task IDs per model - """ - trace_id = correlation_id.get() or DEFAULT_TRACE_ID - task_ids_by_model: dict[str, list[str]] = {} - failed_updates: list[TTSResultUpdate] = [] - - for model, result_ids in result_ids_by_model.items(): - task_ids: list[str] = [] - for start in range(0, len(result_ids), TTS_SYNC_CHUNK_SIZE): - chunk_ids = result_ids[start : start + TTS_SYNC_CHUNK_SIZE] - try: - celery_task_id = start_tts_sync_chunk( - project_id=project_id, - job_id=str(run_id), - trace_id=trace_id, - organization_id=organization_id, - model=model, - result_ids=chunk_ids, - language_code=language_code, - ) - except Exception as e: - logger.error( - f"[_dispatch_sync_chunks] Failed to enqueue chunk | " - f"run_id: {run_id}, model: {model}, " - f"chunk_size: {len(chunk_ids)}, error: {str(e)}", - exc_info=True, - ) - for result_id in chunk_ids: - failed_updates.append( - TTSResultUpdate( - result_id=result_id, - status=JobStatus.FAILED, - error_message=( - f"[{KAAPI_TAG}] Failed to queue synthesis for {model}: " - f"{str(e)}. Re-run the evaluation; contact Kaapi " - f"if it persists." - ), - ) - ) - continue - task_ids.append(celery_task_id) - - task_ids_by_model[model] = task_ids - logger.info( - f"[_dispatch_sync_chunks] Model dispatched | run_id: {run_id}, " - f"model: {model}, results: {len(result_ids)}, chunks: {len(task_ids)}" - ) - - if failed_updates: - with Session(engine) as session: - bulk_update_pending_tts_results(session=session, updates=failed_updates) - finalize_tts_run_status(session=session, run_id=run_id) - - return task_ids_by_model - - def _mark_run_failed(*, run_id: int, error_message: str) -> None: with Session(engine) as session: update_tts_run( diff --git a/backend/app/services/tts_evaluations/batch_result_processing.py b/backend/app/services/tts_evaluations/batch_result_processing.py index d45c63e76..3613c58bb 100644 --- a/backend/app/services/tts_evaluations/batch_result_processing.py +++ b/backend/app/services/tts_evaluations/batch_result_processing.py @@ -2,9 +2,6 @@ Processes completed Gemini TTS batch results: downloads JSONL, extracts audio, converts PCM to WAV, uploads to S3, updates DB. - -The DB session is only opened for short prefetch/write/finalize steps; the -download, audio conversion and S3 uploads run with no connection checked out. """ import base64 @@ -12,51 +9,42 @@ import uuid from typing import Any -from celery import Task -from celery.exceptions import SoftTimeLimitExceeded from gevent import Timeout +from celery.exceptions import SoftTimeLimitExceeded from sqlmodel import Session -from app.core.audio_utils import calculate_duration, pcm_to_wav from app.core.batch import BATCH_KEY, GeminiBatchProvider, GeminiClient -from app.core.cloud.storage import CloudStorage, get_cloud_storage +from app.core.cloud.storage import get_cloud_storage from app.core.db import engine from app.core.storage_utils import upload_to_object_store +from app.core.util import now from app.crud.tts_evaluations.result import ( - bulk_update_pending_tts_results, get_pending_results_for_run, + update_tts_result, ) -from app.crud.tts_evaluations.run import finalize_tts_run_status, update_tts_run from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResultUpdate -from app.services.tts_evaluations.constants import ( - TTS_AUDIO_CONTENT_TYPE, - TTS_AUDIO_FILE_EXTENSION, - TTS_AUDIO_SUBDIRECTORY, - TTS_RESULT_WRITE_CHUNK_SIZE, -) +from app.models.tts_evaluation import TTSResult +from app.core.audio_utils import calculate_duration, pcm_to_wav +from app.services.tts_evaluations.constants import TTS_AUDIO_SUBDIRECTORY logger = logging.getLogger(__name__) -DURATION_DECIMALS = 3 - def execute_tts_result_processing( project_id: int, job_id: str, task_id: str, - task_instance: Task, + task_instance: Any, organization_id: int, evaluation_run_id: int, tts_provider: str, provider_batch_id: str, **kwargs: Any, -) -> dict[str, Any]: +) -> dict: """Process completed TTS batch results in a Celery worker. Downloads batch results from Gemini, extracts audio, converts to WAV, - uploads to S3, and updates TTSResult records. Only results still PENDING - are processed, so reprocessing a batch skips rows already finished. + uploads to S3, and updates TTSResult records. Args: project_id: Project ID @@ -85,19 +73,23 @@ def execute_tts_result_processing( project_id=project_id, ) storage = get_cloud_storage(session=session, project_id=project_id) - pending_rows = get_pending_results_for_run( + + # Only PENDING rows, so reprocessing a batch skips rows already finished. + pending = get_pending_results_for_run( session=session, run_id=evaluation_run_id, provider=tts_provider ) - pending_ids: set[int] = set() - for row in pending_rows: - pending_ids.add(row.id) + result_map: dict[int, TTSResult] = {} + for result in pending: + result_map[result.id] = result logger.info( - f"[execute_tts_result_processing] Pre-fetched pending result ids | " + f"[execute_tts_result_processing] Pre-fetched result records | " f"run_id={evaluation_run_id}, provider={tts_provider}, " - f"count={len(pending_ids)}" + f"count={len(result_map)}" ) + # Download, conversion and uploads run with no session open; the detached + # rows are modified in memory and written back in one short session below. batch_provider = GeminiBatchProvider(client=gemini_client.client) results = batch_provider.download_batch_results(provider_batch_id) @@ -119,8 +111,6 @@ def execute_tts_result_processing( processed_count = 0 failed_count = 0 - skipped_count = 0 - pending_updates: list[TTSResultUpdate] = [] for batch_result in results: custom_id = batch_result[BATCH_KEY] @@ -134,38 +124,85 @@ def execute_tts_result_processing( failed_count += 1 continue - if result_id not in pending_ids: - logger.info( - f"[execute_tts_result_processing] Result not pending, skipping | " - f"result_id={result_id}" + result_record = result_map.get(result_id) + + if not result_record: + logger.warning( + f"[execute_tts_result_processing] Result record not found or " + f"not pending | result_id={result_id}" ) - skipped_count += 1 + failed_count += 1 continue - update = _build_result_update( - result_id=result_id, batch_result=batch_result, storage=storage - ) - if update.status == JobStatus.SUCCESS: - processed_count += 1 + if batch_result.get("response"): + try: + audio_b64 = _extract_audio_from_response(batch_result["response"]) + + if not audio_b64: + result_record.status = JobStatus.FAILED.value + result_record.error_message = "No audio data in response" + result_record.updated_at = now() + failed_count += 1 + continue + + # Without validate=True, garbage decodes to b"" and is stored as a 0s "success". + pcm_data = base64.b64decode(audio_b64, validate=True) + + wav_data = pcm_to_wav(pcm_data) + duration = calculate_duration(len(pcm_data)) + + audio_filename = f"{uuid.uuid4()}.wav" + audio_url = upload_to_object_store( + storage=storage, + content=wav_data, + filename=audio_filename, + subdirectory=TTS_AUDIO_SUBDIRECTORY, + content_type="audio/wav", + ) + + if not audio_url: + result_record.status = JobStatus.FAILED.value + result_record.error_message = ( + "Audio upload to object store failed" + ) + result_record.updated_at = now() + failed_count += 1 + continue + + result_record.object_store_url = audio_url + result_record.metadata_ = { + "duration_seconds": round(duration, 3), + "size_bytes": len(wav_data), + } + result_record.status = JobStatus.SUCCESS.value + result_record.updated_at = now() + processed_count += 1 + + except Exception as audio_err: + logger.warning( + f"[execute_tts_result_processing] Audio processing failed | " + f"result_id={result_id}, error={str(audio_err)}" + ) + result_record.status = JobStatus.FAILED.value + result_record.error_message = ( + f"Audio processing failed: {str(audio_err)}" + ) + result_record.updated_at = now() + failed_count += 1 else: + result_record.status = JobStatus.FAILED.value + result_record.error_message = batch_result.get("error", "Unknown error") + result_record.updated_at = now() failed_count += 1 - pending_updates.append(update) - if len(pending_updates) >= TTS_RESULT_WRITE_CHUNK_SIZE: - _write_result_updates(pending_updates) - pending_updates = [] - - _write_result_updates(pending_updates) - with Session(engine) as session: - run = finalize_tts_run_status(session=session, run_id=evaluation_run_id) - final_status = run.status if run else None + session.add_all(list(result_map.values())) + session.commit() logger.info( f"[execute_tts_result_processing] Completed | " f"run_id={evaluation_run_id}, provider={tts_provider}, " - f"processed={processed_count}, failed={failed_count}, " - f"skipped={skipped_count}, run_status={final_status}" + f"processed={processed_count}, failed={failed_count}" ) return { @@ -173,15 +210,17 @@ def execute_tts_result_processing( "run_id": evaluation_run_id, "processed": processed_count, "failed": failed_count, - "run_status": final_status, } except (Timeout, SoftTimeLimitExceeded): - timeout_err = TimeoutError("Task exceeded soft time limit") logger.warning( f"[execute_tts_result_processing] TTS result processing timed out | run_id={evaluation_run_id}" ) - _mark_run_failed(run_id=evaluation_run_id, error_message=str(timeout_err)) + _fail_pending_results( + run_id=evaluation_run_id, + model=tts_provider, + error_message="Result processing exceeded soft time limit", + ) raise except Exception as e: @@ -190,94 +229,28 @@ def execute_tts_result_processing( f"run_id={evaluation_run_id}, error={str(e)}", exc_info=True, ) - _mark_run_failed( + _fail_pending_results( run_id=evaluation_run_id, + model=tts_provider, error_message=f"Result processing failed: {str(e)}", ) return {"success": False, "error": str(e)} -def _build_result_update( - *, - result_id: int, - batch_result: dict[str, Any], - storage: CloudStorage, -) -> TTSResultUpdate: - """Convert one Gemini batch result into a terminal update (uploads the WAV).""" - if not batch_result.get("response"): - return TTSResultUpdate( - result_id=result_id, - status=JobStatus.FAILED, - error_message=batch_result.get("error", "Unknown error"), - ) - - try: - audio_b64 = _extract_audio_from_response(batch_result["response"]) - - if not audio_b64: - return TTSResultUpdate( - result_id=result_id, - status=JobStatus.FAILED, - error_message="No audio data in response", - ) - - # Without validate=True, garbage input decodes to b"" and would be stored as a 0s "success". - pcm_data = base64.b64decode(audio_b64, validate=True) - wav_data = pcm_to_wav(pcm_data) - duration = calculate_duration(len(pcm_data)) - - audio_url = upload_to_object_store( - storage=storage, - content=wav_data, - filename=f"{uuid.uuid4()}.{TTS_AUDIO_FILE_EXTENSION}", - subdirectory=TTS_AUDIO_SUBDIRECTORY, - content_type=TTS_AUDIO_CONTENT_TYPE, - ) - - if not audio_url: - return TTSResultUpdate( - result_id=result_id, - status=JobStatus.FAILED, - error_message="Audio upload to object store failed", - ) - - return TTSResultUpdate( - result_id=result_id, - status=JobStatus.SUCCESS, - object_store_url=audio_url, - metadata={ - "duration_seconds": round(duration, DURATION_DECIMALS), - "size_bytes": len(wav_data), - }, - ) - - except Exception as audio_err: - logger.warning( - f"[_build_result_update] Audio processing failed | " - f"result_id={result_id}, error={str(audio_err)}" - ) - return TTSResultUpdate( - result_id=result_id, - status=JobStatus.FAILED, - error_message=f"Audio processing failed: {str(audio_err)}", - ) - - -def _write_result_updates(updates: list[TTSResultUpdate]) -> None: - if not updates: - return +def _fail_pending_results(*, run_id: int, model: str, error_message: str) -> None: + """Fail only this model's leftover rows so the other models still decide the run.""" with Session(engine) as session: - bulk_update_pending_tts_results(session=session, updates=updates) - - -def _mark_run_failed(*, run_id: int, error_message: str) -> None: - with Session(engine) as session: - update_tts_run( - session=session, - run_id=run_id, - status="failed", - error_message=error_message, + pending = get_pending_results_for_run( + session=session, run_id=run_id, provider=model ) + for result in pending: + update_tts_result( + session=session, + result_id=result.id, + status=JobStatus.FAILED.value, + error_message=error_message, + ) + session.commit() def _extract_audio_from_response(response: dict[str, Any]) -> str | None: diff --git a/backend/app/services/tts_evaluations/constants.py b/backend/app/services/tts_evaluations/constants.py index 92e720da9..6b16716e4 100644 --- a/backend/app/services/tts_evaluations/constants.py +++ b/backend/app/services/tts_evaluations/constants.py @@ -1,124 +1,55 @@ """Shared constants for TTS evaluation services. -Kept free of `app.models` imports: `app.models.tts_evaluation` imports this module -for its request validator, so importing models here would create a cycle. +Kept free of `app.models` imports: `app.models.tts_evaluation` imports this module, +so importing models here would create a cycle. """ -from dataclasses import dataclass -from enum import StrEnum +# Gemini models go through the Gemini Batch API; Sarvam/ElevenLabs are called directly. +GEMINI_TTS_MODELS = ["gemini-2.5-pro-preview-tts", "gemini-3.1-flash-tts-preview"] +SARVAM_TTS_MODELS = ["bulbul:v3"] +ELEVENLABS_TTS_MODELS = ["eleven_v4", "eleven_v3"] -from app.core.providers import Provider - - -class TTSExecutionModeEnum(StrEnum): - """How a TTS model's synthesis is executed. - - BATCH goes through a provider Batch API polled by the cron; SYNC is called - directly from chunked Celery tasks. - """ - - BATCH = "batch" - SYNC = "sync" - - -@dataclass(frozen=True) -class TTSModelSpec: - provider: Provider - execution_mode: TTSExecutionModeEnum - default_voice: str - - -GEMINI_PRO_TTS_MODEL = "gemini-2.5-pro-preview-tts" -SARVAM_BULBUL_V3_MODEL = "bulbul:v3" -ELEVENLABS_V3_MODEL = "eleven_v3" -ELEVENLABS_V4_MODEL = "eleven_v4" - -# Default voice configuration (Gemini prebuilt voice) -DEFAULT_VOICE_NAME = "Kore" -SARVAM_DEFAULT_SPEAKER = "shubh" -# "Rahul S - Hindi Conversational": closest match to Sarvam's male speaker so the -# providers are roughly comparable in a side-by-side review. -ELEVENLABS_DEFAULT_VOICE_ID = "2cdvnKJ5TZi631y5PN1s" - -TTS_MODEL_REGISTRY: dict[str, TTSModelSpec] = { - GEMINI_PRO_TTS_MODEL: TTSModelSpec( - provider=Provider.GOOGLE_AISTUDIO, - execution_mode=TTSExecutionModeEnum.BATCH, - default_voice=DEFAULT_VOICE_NAME, - ), - SARVAM_BULBUL_V3_MODEL: TTSModelSpec( - provider=Provider.SARVAMAI, - execution_mode=TTSExecutionModeEnum.SYNC, - default_voice=SARVAM_DEFAULT_SPEAKER, - ), - ELEVENLABS_V3_MODEL: TTSModelSpec( - provider=Provider.ELEVENLABS, - execution_mode=TTSExecutionModeEnum.SYNC, - default_voice=ELEVENLABS_DEFAULT_VOICE_ID, - ), - ELEVENLABS_V4_MODEL: TTSModelSpec( - provider=Provider.ELEVENLABS, - execution_mode=TTSExecutionModeEnum.SYNC, - default_voice=ELEVENLABS_DEFAULT_VOICE_ID, - ), -} +SYNC_TTS_MODELS = SARVAM_TTS_MODELS + ELEVENLABS_TTS_MODELS # Supported TTS models for evaluation -SUPPORTED_TTS_MODELS = list(TTS_MODEL_REGISTRY.keys()) +SUPPORTED_TTS_MODELS = GEMINI_TTS_MODELS + SYNC_TTS_MODELS # Default TTS model -DEFAULT_TTS_MODEL = GEMINI_PRO_TTS_MODEL +DEFAULT_TTS_MODEL = "gemini-2.5-pro-preview-tts" + +# Default voice configuration +DEFAULT_VOICE_NAME = "Kore" +SARVAM_SPEAKER = "shubh" +# "Rahul S - Hindi Conversational": closest match to Sarvam's male speaker for side-by-side review. +ELEVENLABS_VOICE_ID = "2cdvnKJ5TZi631y5PN1s" # Default style prompt for TTS synthesis DEFAULT_STYLE_PROMPT = "Read in a calm, professional customer service tone" -# BCP-47 tag used when the dataset has no language configured. -DEFAULT_TTS_LANGUAGE_CODE = "en-IN" +DEFAULT_TTS_LANGUAGE = "en-IN" + +# Dataset locale (ISO 639-1) -> BCP-47 tag; Sarvam spells Odia "od-IN", not "or-IN". +LOCALE_TO_BCP47 = { + "en": "en-IN", + "hi": "hi-IN", + "bn": "bn-IN", + "ta": "ta-IN", + "te": "te-IN", + "mr": "mr-IN", + "gu": "gu-IN", + "kn": "kn-IN", + "ml": "ml-IN", + "pa": "pa-IN", + "or": "od-IN", +} -# Every stored evaluation clip is normalized to this format so providers compare 1:1. -TTS_SAMPLE_RATE_HZ = 24000 -TTS_SAMPLE_WIDTH_BYTES = 2 -TTS_CHANNELS = 1 +# Rows synthesized at once per sync task; kept low because eleven_v3 and eleven_v4 +# share one ElevenLabs key and its concurrent-request limit. +TTS_SYNC_MAX_WORKERS = 2 TTS_AUDIO_SUBDIRECTORY = "evaluations/tts/audio" -TTS_AUDIO_CONTENT_TYPE = "audio/wav" -TTS_AUDIO_FILE_EXTENSION = "wav" - -SARVAM_PACE = 1.0 -ELEVENLABS_OUTPUT_FORMAT = f"pcm_{TTS_SAMPLE_RATE_HZ}" - -# 25 items x 4 workers keeps a chunk well inside CELERY_TASK_SOFT_TIME_LIMIT -# even with retries on slow (~10s) ElevenLabs v3 calls. -TTS_SYNC_CHUNK_SIZE = 25 -TTS_SYNC_MAX_WORKERS = 4 -TTS_SYNC_REQUEST_TIMEOUT_SECONDS = 60 -TTS_SYNC_MAX_ATTEMPTS = 3 -TTS_SYNC_BACKOFF_BASE_SECONDS = 1.0 -TTS_SYNC_BACKOFF_MAX_SECONDS = 20.0 -TTS_RETRYABLE_STATUS_CODES = frozenset({408, 409, 429, 500, 502, 503, 504}) - -# Source tag for failures that are Kaapi-side (see error-handling convention). -KAAPI_TAG = "KAAPI" - -# Commit Gemini batch-result writes in slices so a timeout keeps earlier progress. -TTS_RESULT_WRITE_CHUNK_SIZE = 50 - - -def get_tts_model_spec(model: str) -> TTSModelSpec: - """Return the registry spec for a model; raises KeyError for unknown models.""" - return TTS_MODEL_REGISTRY[model] - -def split_models_by_execution_mode( - models: list[str], -) -> tuple[list[str], list[str]]: - """Split requested models into (batch_models, sync_models), preserving order.""" - batch_models: list[str] = [] - sync_models: list[str] = [] - for model in models: - spec = get_tts_model_spec(model) - if spec.execution_mode == TTSExecutionModeEnum.BATCH: - batch_models.append(model) - else: - sync_models.append(model) - return batch_models, sync_models +# One task synthesizes a whole model sequentially; the global 300s limit only fits ~30-60 rows. +TTS_SYNC_TASK_TIME_LIMIT_SECONDS = 1800 +# Past this, the cron treats a sync model's leftover PENDING rows as abandoned by a dead worker. +TTS_SYNC_STALE_AFTER_SECONDS = TTS_SYNC_TASK_TIME_LIMIT_SECONDS + 600 diff --git a/backend/app/services/tts_evaluations/dataset.py b/backend/app/services/tts_evaluations/dataset.py index 13c5c2cd0..02d56d0ec 100644 --- a/backend/app/services/tts_evaluations/dataset.py +++ b/backend/app/services/tts_evaluations/dataset.py @@ -5,6 +5,7 @@ import logging from typing import Any +from celery.exceptions import SoftTimeLimitExceeded from sqlmodel import Session from app.core.cloud import get_cloud_storage @@ -164,16 +165,16 @@ def _samples_to_csv(samples: list[TTSSampleCreate]) -> bytes: def get_sample_texts_from_dataset( - session: Session, + storage: CloudStorage, dataset: "EvaluationDataset", - project_id: int, ) -> list[str]: """Extract sample texts from a TTS dataset's CSV in S3. + Takes storage rather than a session so callers can read S3 with no DB session open. + Args: - session: Database session + storage: Cloud storage for the dataset's project dataset: The evaluation dataset record - project_id: Project ID Returns: List of text strings @@ -186,49 +187,19 @@ def get_sample_texts_from_dataset( return [] try: - storage = get_cloud_storage(session=session, project_id=project_id) - except Exception as e: - logger.error( - f"[get_sample_texts_from_dataset] Failed to init storage | " - f"dataset_id={dataset.id}, error={str(e)}", - exc_info=True, - ) - return [] - - return load_sample_texts( - storage=storage, - object_store_url=dataset.object_store_url, - dataset_id=dataset.id, - ) - - -def load_sample_texts( - *, - storage: CloudStorage, - object_store_url: str, - dataset_id: int, -) -> list[str]: - """Read a TTS dataset CSV from object storage; needs no DB session. - - Returns: - List of text strings (empty on read/parse failure) - """ - try: - csv_bytes = storage.stream(object_store_url).read() + csv_bytes = storage.stream(dataset.object_store_url).read() samples = parse_tts_samples_from_csv(csv_bytes) + return [s["text"] for s in samples] + except SoftTimeLimitExceeded: + # Let the task's timeout handler report it instead of "No samples found". + raise except Exception as e: logger.error( - f"[load_sample_texts] Failed to load CSV | " - f"dataset_id={dataset_id}, error={str(e)}", - exc_info=True, + f"[get_sample_texts_from_dataset] Failed to load CSV | " + f"dataset_id={dataset.id}, error={str(e)}" ) return [] - texts: list[str] = [] - for sample in samples: - texts.append(sample["text"]) - return texts - def parse_tts_samples_from_csv(csv_content: bytes) -> list[dict[str, Any]]: """Parse TTS samples from CSV content. diff --git a/backend/app/services/tts_evaluations/language.py b/backend/app/services/tts_evaluations/language.py deleted file mode 100644 index 336358629..000000000 --- a/backend/app/services/tts_evaluations/language.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Language-code resolution for TTS evaluation providers. - -Datasets store an ISO 639-1 locale (`Language.locale`, e.g. "hi"); the canonical -code carried through the TTS pipeline is a BCP-47 tag (e.g. "hi-IN"), which is -what Sarvam takes directly and what ElevenLabs' ISO code is derived from. -""" - -import logging - -from sqlmodel import Session - -from app.crud.language import get_language_locale_by_id -from app.models.llm.constants import BCP47_TO_ELEVENLABS_LANG -from app.services.tts_evaluations.constants import DEFAULT_TTS_LANGUAGE_CODE - -logger = logging.getLogger(__name__) - -# Sarvam spells Odia "od-IN" while ISO 639-1 is "or"; it is absent from the -# ElevenLabs map, so it can't be derived from the reverse lookup. -_ODIA_ISO_CODE = "or" -_ODIA_BCP47_CODE = "od-IN" - -ISO_TO_BCP47_LANG: dict[str, str] = { - iso_code: bcp47_code for bcp47_code, iso_code in BCP47_TO_ELEVENLABS_LANG.items() -} -ISO_TO_BCP47_LANG[_ODIA_ISO_CODE] = _ODIA_BCP47_CODE - - -def resolve_tts_language_code(locale: str | None) -> str: - """Map an ISO 639-1 locale to the pipeline's BCP-47 tag, defaulting to en-IN.""" - if not locale: - return DEFAULT_TTS_LANGUAGE_CODE - - language_code = ISO_TO_BCP47_LANG.get(locale.lower()) - if language_code is None: - logger.warning( - f"[resolve_tts_language_code] Unmapped locale, using default | " - f"locale: {locale}, default: {DEFAULT_TTS_LANGUAGE_CODE}" - ) - return DEFAULT_TTS_LANGUAGE_CODE - - return language_code - - -def resolve_dataset_tts_language_code(session: Session, language_id: int | None) -> str: - """Resolve a dataset's `language_id` to a BCP-47 tag (en-IN when unset/unknown).""" - if language_id is None: - return DEFAULT_TTS_LANGUAGE_CODE - - locale = get_language_locale_by_id(session=session, language_id=language_id) - if locale is None: - logger.warning( - f"[resolve_dataset_tts_language_code] Language not found, using default | " - f"language_id: {language_id}, default: {DEFAULT_TTS_LANGUAGE_CODE}" - ) - return resolve_tts_language_code(locale) - - -def to_elevenlabs_language_code(language_code: str) -> str | None: - """ElevenLabs ISO 639-1 code, or None to let it auto-detect (e.g. Odia).""" - return BCP47_TO_ELEVENLABS_LANG.get(language_code) diff --git a/backend/app/services/tts_evaluations/sync_generation.py b/backend/app/services/tts_evaluations/sync_generation.py index b4af6db77..7557c38dc 100644 --- a/backend/app/services/tts_evaluations/sync_generation.py +++ b/backend/app/services/tts_evaluations/sync_generation.py @@ -1,81 +1,117 @@ -"""Celery task function for synchronous (non-batch) TTS evaluation synthesis. +"""Celery task function for synchronous TTS synthesis (Sarvam, ElevenLabs). -One task handles one model and a slice of that model's result IDs. The DB session -is only held for short reads/writes; provider calls and S3 uploads run with no -session so a slow provider can't pin pooled connections. +One task synthesizes every PENDING row of one model, TTS_SYNC_MAX_WORKERS rows at a +time. Provider calls and uploads run with no DB session open so a slow provider can't +pin pooled connections. The task only writes result rows; the cron sets the run status. """ +import base64 +import io import logging import uuid +import wave from concurrent.futures import Future, ThreadPoolExecutor, as_completed -from dataclasses import dataclass -from typing import Any +from typing import Any, Required, TypedDict from celery import Task from celery.exceptions import SoftTimeLimitExceeded +from elevenlabs import ElevenLabs from gevent import Timeout +from sarvamai import SarvamAI from sqlmodel import Session -from app.core.cloud.storage import CloudStorage, get_cloud_storage +from app.core.audio_utils import calculate_duration, pcm_to_wav +from app.core.cloud.storage import get_cloud_storage from app.core.db import engine from app.core.providers import Provider from app.core.storage_utils import upload_to_object_store from app.crud.credentials import get_provider_credential from app.crud.tts_evaluations.result import ( - bulk_update_pending_tts_results, - list_pending_tts_results_by_ids, + get_pending_results_for_run, + update_tts_result, ) -from app.crud.tts_evaluations.run import finalize_tts_run_status from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResultUpdate +from app.models.llm.constants import BCP47_TO_ELEVENLABS_LANG +from app.services.llm.providers.eleven_ai import ElevenlabsAIProvider +from app.services.llm.providers.sarvam_ai import SarvamAIProvider from app.services.tts_evaluations.constants import ( - KAAPI_TAG, - TTS_AUDIO_CONTENT_TYPE, - TTS_AUDIO_FILE_EXTENSION, + ELEVENLABS_TTS_MODELS, + ELEVENLABS_VOICE_ID, + SARVAM_SPEAKER, + SARVAM_TTS_MODELS, TTS_AUDIO_SUBDIRECTORY, TTS_SYNC_MAX_WORKERS, - get_tts_model_spec, -) -from app.services.tts_evaluations.synthesizers import ( - TTSClient, - TTSSynthesisError, - create_tts_client, - synthesize_tts, ) logger = logging.getLogger(__name__) -DURATION_DECIMALS = 3 +# Every stored clip is 24 kHz mono 16-bit +TTS_SAMPLE_RATE = 24000 + +class TTSResultFields(TypedDict, total=False): + """Keyword arguments for `update_tts_result` produced for one synthesized row.""" -@dataclass(frozen=True) -class _PendingItem: - result_id: int - text: str + status: Required[str] + object_store_url: str + metadata: dict[str, float | int] + error_message: str -@dataclass(frozen=True) -class _ChunkContext: - items: list[_PendingItem] - provider: Provider - voice: str - client: TTSClient | None - storage: CloudStorage | None - setup_error: str | None +def synthesize_sarvam( + client: SarvamAI, text: str, model: str, language_code: str +) -> bytes: + """Synthesize text with Sarvam; returns raw 24 kHz mono 16-bit PCM.""" + response = client.text_to_speech.convert( + text=text, + model=model, + target_language_code=language_code, + speaker=SARVAM_SPEAKER, + speech_sample_rate=TTS_SAMPLE_RATE, + ) + if not response.audios: + raise ValueError("Sarvam returned no audio") + + # Long inputs come back as several base64 chunks, each a full WAV file; strip + # every header so later RIFF headers don't end up in the PCM as clicks. + pcm = b"" + for audio_b64 in response.audios: + with wave.open(io.BytesIO(base64.b64decode(audio_b64)), "rb") as wav_file: + pcm += wav_file.readframes(wav_file.getnframes()) + return pcm + + +def synthesize_elevenlabs( + client: ElevenLabs, text: str, model: str, language_code: str +) -> bytes: + """Synthesize text with ElevenLabs; returns raw 24 kHz mono 16-bit PCM.""" + chunks = client.text_to_speech.convert( + voice_id=ELEVENLABS_VOICE_ID, + text=text, + model_id=model, + output_format=f"pcm_{TTS_SAMPLE_RATE}", + # Unmapped languages (e.g. Odia) are left to ElevenLabs' auto-detect. + language_code=BCP47_TO_ELEVENLABS_LANG.get(language_code), + ) + pcm = b"".join(chunks) + if not pcm: + raise ValueError("ElevenLabs returned no audio") + return pcm -def execute_tts_sync_chunk( +def execute_tts_sync_generation( project_id: int, job_id: str, task_id: str, task_instance: Task, organization_id: int, model: str, - result_ids: list[int], language_code: str, **kwargs: Any, ) -> dict[str, Any]: - """Synthesize one chunk of sync-model TTS results in a Celery worker. + """Synthesize every PENDING Sarvam/ElevenLabs result of one model in a Celery worker. + + Only PENDING rows are picked up, so a redelivered task skips rows already finished. Args: project_id: Project ID @@ -83,367 +119,169 @@ def execute_tts_sync_chunk( task_id: Celery task ID task_instance: Celery task instance organization_id: Organization ID - model: Sync-mode TTS model (e.g. bulbul:v3, eleven_v3) - result_ids: TTSResult IDs in this chunk (all for `model`) + model: Sarvam or ElevenLabs model language_code: BCP-47 tag resolved from the dataset (e.g. hi-IN) Returns: - dict: Summary with processed/failed counts and the run status + dict: Summary with processed/failed counts """ run_id = int(job_id) + if model in SARVAM_TTS_MODELS: + provider = Provider.SARVAMAI + elif model in ELEVENLABS_TTS_MODELS: + provider = Provider.ELEVENLABS + else: + error_message = f"Unsupported sync TTS model: {model}" + logger.error(f"[execute_tts_sync_generation] {error_message} | run_id={run_id}") + _fail_pending_results(run_id=run_id, model=model, error_message=error_message) + return {"success": False, "run_id": run_id, "error": error_message} logger.info( - f"[execute_tts_sync_chunk] Starting | run_id={run_id}, model={model}, " - f"chunk_size={len(result_ids)}, language={language_code}, " - f"celery_task_id={task_id}" + f"[execute_tts_sync_generation] Starting | run_id={run_id}, model={model}, " + f"language={language_code}, celery_task_id={task_id}" ) try: - context = _prepare_chunk( - run_id=run_id, - organization_id=organization_id, - project_id=project_id, - model=model, - result_ids=result_ids, - ) - - if not context.items: - logger.info( - f"[execute_tts_sync_chunk] No pending results, skipping | " - f"run_id={run_id}, model={model}" + with Session(engine) as session: + pending = get_pending_results_for_run( + session=session, run_id=run_id, provider=model ) - run_status = _finalize_run(run_id) - return { - "success": True, - "run_id": run_id, - "processed": 0, - "failed": 0, - "run_status": run_status, - } - - if context.setup_error is not None: - failed_count = _fail_pending_results( - run_id=run_id, - result_ids=result_ids, - error_message=context.setup_error, + # Plain tuples: the ORM rows detach when this session closes. + pending_rows: list[tuple[int, str]] = [] + for result in pending: + pending_rows.append((result.id, result.sample_text)) + + credentials = get_provider_credential( + session=session, + org_id=organization_id, + project_id=project_id, + provider=provider.value, ) - run_status = _finalize_run(run_id) + storage = get_cloud_storage(session=session, project_id=project_id) + + if not isinstance(credentials, dict): + raise ValueError(f"{provider.value} credentials are not configured") + + if model in SARVAM_TTS_MODELS: + client = SarvamAIProvider.create_client(credentials) + else: + client = ElevenlabsAIProvider.create_client(credentials) + + # keep the function local to use it easily with + # internal threadpool exuector + + def synthesize_and_upload(result_id: int, text: str) -> TTSResultFields: + """Return `update_tts_result` fields; never raises, so one row can't stop the rest.""" + try: + if model in SARVAM_TTS_MODELS: + pcm = synthesize_sarvam(client, text, model, language_code) + else: + pcm = synthesize_elevenlabs(client, text, model, language_code) + wav = pcm_to_wav(pcm) + audio_url = upload_to_object_store( + storage=storage, + content=wav, + filename=f"{uuid.uuid4()}.wav", + subdirectory=TTS_AUDIO_SUBDIRECTORY, + content_type="audio/wav", + ) + except Exception as e: + logger.warning( + f"[execute_tts_sync_generation] Synthesis failed | " + f"run_id={run_id}, model={model}, result_id={result_id}, " + f"error={str(e)}" + ) + return { + "status": JobStatus.FAILED.value, + "error_message": f"{provider.value} synthesis failed: {str(e)}", + } + + if not audio_url: + return { + "status": JobStatus.FAILED.value, + "error_message": "Audio upload to object store failed", + } return { - "success": False, - "run_id": run_id, - "processed": 0, - "failed": failed_count, - "run_status": run_status, - "error": context.setup_error, + "status": JobStatus.SUCCESS.value, + "object_store_url": audio_url, + "metadata": { + "duration_seconds": round(calculate_duration(len(pcm)), 3), + "size_bytes": len(wav), + }, } - processed_count, failed_count = _synthesize_chunk( - context=context, - run_id=run_id, - model=model, - language_code=language_code, - ) - - run_status = _finalize_run(run_id) + processed = 0 + failed = 0 + # Shut down by hand: a `with` block would wait for in-flight provider calls on timeout. + executor = ThreadPoolExecutor(max_workers=TTS_SYNC_MAX_WORKERS) + try: + futures: dict[Future[TTSResultFields], int] = {} + for result_id, text in pending_rows: + future = executor.submit(synthesize_and_upload, result_id, text) + futures[future] = result_id + + for future in as_completed(futures): + fields = future.result() + with Session(engine) as session: + update_tts_result( + session=session, result_id=futures[future], **fields + ) + session.commit() + + if fields["status"] == JobStatus.SUCCESS.value: + processed += 1 + else: + failed += 1 + finally: + executor.shutdown(wait=False, cancel_futures=True) logger.info( - f"[execute_tts_sync_chunk] Completed | run_id={run_id}, model={model}, " - f"processed={processed_count}, failed={failed_count}, " - f"run_status={run_status}" + f"[execute_tts_sync_generation] Completed | run_id={run_id}, model={model}, " + f"processed={processed}, failed={failed}" ) - return { "success": True, "run_id": run_id, - "processed": processed_count, - "failed": failed_count, - "run_status": run_status, + "processed": processed, + "failed": failed, } except (Timeout, SoftTimeLimitExceeded): logger.warning( - f"[execute_tts_sync_chunk] Chunk timed out | run_id={run_id}, model={model}" + f"[execute_tts_sync_generation] Timed out | run_id={run_id}, model={model}" ) - # Nothing re-drives a sync chunk, so leftover PENDING rows would strand the run. _fail_pending_results( run_id=run_id, - result_ids=result_ids, - error_message=( - f"[{KAAPI_TAG}] Synthesis timed out before this sample was processed. " - f"Re-run the evaluation; if it persists, contact Kaapi." - ), + model=model, + error_message="Synthesis timed out before this sample was processed", ) - _finalize_run(run_id) raise except Exception as e: logger.error( - f"[execute_tts_sync_chunk] Failed | run_id={run_id}, model={model}, " + f"[execute_tts_sync_generation] Failed | run_id={run_id}, model={model}, " f"error={str(e)}", exc_info=True, ) _fail_pending_results( run_id=run_id, - result_ids=result_ids, - error_message=( - f"[{KAAPI_TAG}] Synthesis chunk failed unexpectedly: {str(e)}. " - f"Contact Kaapi if the issue persists." - ), - ) - run_status = _finalize_run(run_id) - return { - "success": False, - "run_id": run_id, - "error": str(e), - "run_status": run_status, - } - - -def _prepare_chunk( - *, - run_id: int, - organization_id: int, - project_id: int, - model: str, - result_ids: list[int], -) -> _ChunkContext: - """Load pending items, credentials, client and storage in one short session.""" - spec = get_tts_model_spec(model) - - with Session(engine) as session: - pending_rows = list_pending_tts_results_by_ids( - session=session, run_id=run_id, result_ids=result_ids - ) - items: list[_PendingItem] = [] - for row in pending_rows: - items.append(_PendingItem(result_id=row.id, text=row.sample_text)) - - if not items: - return _ChunkContext( - items=items, - provider=spec.provider, - voice=spec.default_voice, - client=None, - storage=None, - setup_error=None, - ) - - credentials = get_provider_credential( - session=session, - org_id=organization_id, - project_id=project_id, - provider=spec.provider.value, - ) - storage = get_cloud_storage(session=session, project_id=project_id) - - if not isinstance(credentials, dict): - setup_error = ( - f"[{KAAPI_TAG}] {spec.provider.value} credentials are not configured " - f"for this project. Add them under credentials and re-run the evaluation." - ) - logger.warning( - f"[_prepare_chunk] {setup_error} | run_id={run_id}, model={model}" - ) - return _ChunkContext( - items=items, - provider=spec.provider, - voice=spec.default_voice, - client=None, - storage=storage, - setup_error=setup_error, - ) - - try: - client = create_tts_client(spec.provider, credentials) - except ValueError as e: - setup_error = ( - f"[{KAAPI_TAG}] Invalid {spec.provider.value} credentials: {str(e)}. " - f"Verify the stored API key and re-run the evaluation." - ) - logger.warning( - f"[_prepare_chunk] {setup_error} | run_id={run_id}, model={model}", - exc_info=True, - ) - return _ChunkContext( - items=items, - provider=spec.provider, - voice=spec.default_voice, - client=None, - storage=storage, - setup_error=setup_error, - ) - - return _ChunkContext( - items=items, - provider=spec.provider, - voice=spec.default_voice, - client=client, - storage=storage, - setup_error=None, - ) - - -def _synthesize_chunk( - *, - context: _ChunkContext, - run_id: int, - model: str, - language_code: str, -) -> tuple[int, int]: - """Fan synthesis out over a bounded pool; persist each outcome as it lands. - - Returns: - tuple[int, int]: (processed_count, failed_count) actually written - """ - if context.client is None or context.storage is None: - raise ValueError("Chunk context is missing its client or storage") - - client = context.client - storage = context.storage - processed_count = 0 - failed_count = 0 - workers = max(1, min(TTS_SYNC_MAX_WORKERS, len(context.items))) - - # Managed by hand rather than `with`: on a gevent timeout, `with` would block - # in shutdown(wait=True) until every in-flight provider call returned. - executor = ThreadPoolExecutor(max_workers=workers) - try: - futures: list[Future[TTSResultUpdate]] = [] - for item in context.items: - future = executor.submit( - _synthesize_and_upload, - client=client, - storage=storage, - provider=context.provider, - model=model, - voice=context.voice, - language_code=language_code, - item=item, - ) - futures.append(future) - - for future in as_completed(futures): - update = future.result() - written = _write_result_updates([update]) - if not written: - continue - if update.status == JobStatus.SUCCESS: - processed_count += 1 - else: - failed_count += 1 - finally: - executor.shutdown(wait=False, cancel_futures=True) - - logger.info( - f"[_synthesize_chunk] Chunk synthesized | run_id={run_id}, model={model}, " - f"processed={processed_count}, failed={failed_count}" - ) - return processed_count, failed_count - - -def _synthesize_and_upload( - *, - client: TTSClient, - storage: CloudStorage, - provider: Provider, - model: str, - voice: str, - language_code: str, - item: _PendingItem, -) -> TTSResultUpdate: - """Synthesize + upload one item. Never raises: failures become a FAILED update.""" - try: - audio = synthesize_tts( - client=client, - provider=provider, - text=item.text, model=model, - language_code=language_code, - voice=voice, - ) - except TTSSynthesisError as e: - return TTSResultUpdate( - result_id=item.result_id, - status=JobStatus.FAILED, - error_message=e.message, + error_message=f"{provider.value} synthesis failed: {str(e)}", ) - except Exception as e: - # A worker exception would abort the whole chunk via future.result(). - error_message = ( - f"[{KAAPI_TAG}] Unexpected error during {provider.value} synthesis: " - f"{str(e)}. Contact Kaapi if the issue persists." - ) - logger.error( - f"[_synthesize_and_upload] {error_message} | " - f"result_id={item.result_id}, model={model}", - exc_info=True, - ) - return TTSResultUpdate( - result_id=item.result_id, - status=JobStatus.FAILED, - error_message=error_message, - ) - - audio_url = upload_to_object_store( - storage=storage, - content=audio.wav_bytes, - filename=f"{uuid.uuid4()}.{TTS_AUDIO_FILE_EXTENSION}", - subdirectory=TTS_AUDIO_SUBDIRECTORY, - content_type=TTS_AUDIO_CONTENT_TYPE, - ) - - if not audio_url: - error_message = ( - f"[{KAAPI_TAG}] Audio upload to object store failed. Check the " - f"project's storage configuration; contact Kaapi if it persists." - ) - logger.error( - f"[_synthesize_and_upload] {error_message} | " - f"result_id={item.result_id}, model={model}" - ) - return TTSResultUpdate( - result_id=item.result_id, - status=JobStatus.FAILED, - error_message=error_message, - ) - - return TTSResultUpdate( - result_id=item.result_id, - status=JobStatus.SUCCESS, - object_store_url=audio_url, - metadata={ - "duration_seconds": round(audio.duration_seconds, DURATION_DECIMALS), - "size_bytes": len(audio.wav_bytes), - }, - ) + return {"success": False, "run_id": run_id, "error": str(e)} -def _write_result_updates(updates: list[TTSResultUpdate]) -> int: +def _fail_pending_results(*, run_id: int, model: str, error_message: str) -> None: + """Mark this model's leftover PENDING rows FAILED; nothing re-drives a sync task.""" with Session(engine) as session: - return bulk_update_pending_tts_results(session=session, updates=updates) - - -def _fail_pending_results( - *, run_id: int, result_ids: list[int], error_message: str -) -> int: - """Mark every still-PENDING result of this chunk FAILED in a fresh session.""" - with Session(engine) as session: - pending_rows = list_pending_tts_results_by_ids( - session=session, run_id=run_id, result_ids=result_ids + pending = get_pending_results_for_run( + session=session, run_id=run_id, provider=model ) - updates: list[TTSResultUpdate] = [] - for row in pending_rows: - updates.append( - TTSResultUpdate( - result_id=row.id, - status=JobStatus.FAILED, - error_message=error_message, - ) + for result in pending: + update_tts_result( + session=session, + result_id=result.id, + status=JobStatus.FAILED.value, + error_message=error_message, ) - return bulk_update_pending_tts_results(session=session, updates=updates) - - -def _finalize_run(run_id: int) -> str | None: - with Session(engine) as session: - run = finalize_tts_run_status(session=session, run_id=run_id) - return run.status if run else None + session.commit() diff --git a/backend/app/services/tts_evaluations/synthesizers.py b/backend/app/services/tts_evaluations/synthesizers.py deleted file mode 100644 index 2bcfee7b2..000000000 --- a/backend/app/services/tts_evaluations/synthesizers.py +++ /dev/null @@ -1,403 +0,0 @@ -"""Synchronous TTS synthesizers for evaluation runs (Sarvam, ElevenLabs). - -Every synthesizer returns 24 kHz / 16-bit / mono WAV so clips from different -providers are stored and reviewed in one format. SDK and network failures are -mapped to `TTSSynthesisError`, carrying a source-tagged message and whether the -failure is transient (429 / 5xx / network), which drives the retry policy. -""" - -import base64 -import io -import logging -import wave -from dataclasses import dataclass -from typing import Any, cast - -import httpx -from elevenlabs import ElevenLabs -from elevenlabs.core.api_error import ApiError as ElevenLabsApiError -from elevenlabs.core.request_options import ( - RequestOptions as ElevenLabsRequestOptions, -) -from pydub import AudioSegment -from sarvamai import SarvamAI -from sarvamai.core.api_error import ApiError as SarvamApiError -from sarvamai.core.request_options import RequestOptions as SarvamRequestOptions -from tenacity import ( - before_sleep_log, - retry, - retry_if_exception, - stop_after_attempt, - wait_random_exponential, -) - -from app.core.audio_utils import calculate_duration, pcm_to_wav -from app.core.providers import Provider -from app.services.llm.providers.eleven_ai import ElevenlabsAIProvider -from app.services.llm.providers.sarvam_ai import SarvamAIProvider -from app.services.tts_evaluations.constants import ( - ELEVENLABS_OUTPUT_FORMAT, - KAAPI_TAG, - SARVAM_PACE, - TTS_CHANNELS, - TTS_RETRYABLE_STATUS_CODES, - TTS_SAMPLE_RATE_HZ, - TTS_SAMPLE_WIDTH_BYTES, - TTS_SYNC_BACKOFF_BASE_SECONDS, - TTS_SYNC_BACKOFF_MAX_SECONDS, - TTS_SYNC_MAX_ATTEMPTS, - TTS_SYNC_REQUEST_TIMEOUT_SECONDS, -) -from app.services.tts_evaluations.language import to_elevenlabs_language_code - -logger = logging.getLogger(__name__) - -TTSClient = SarvamAI | ElevenLabs - -SARVAM_TAG = "SARVAM" -ELEVENLABS_TAG = "ELEVENLABS" - -_BITS_PER_BYTE = 8 - -# Our tenacity policy owns retries; SDK-level retries would multiply attempts. -_SARVAM_REQUEST_OPTIONS = SarvamRequestOptions( - max_retries=0, timeout_in_seconds=TTS_SYNC_REQUEST_TIMEOUT_SECONDS -) -_ELEVENLABS_REQUEST_OPTIONS = ElevenLabsRequestOptions( - max_retries=0, timeout_in_seconds=TTS_SYNC_REQUEST_TIMEOUT_SECONDS -) - -_STATUS_HINTS: dict[int, str] = { - 400: "Review the input text, model, language and voice — the request shape or content may be invalid.", - 401: "Verify the API key is valid, not expired, and configured for this project.", - 403: "The API key lacks access to the requested model/voice — check plan and key scopes.", - 404: "Verify the model name and voice ID are correct and available on your plan.", - 408: "The provider timed out handling the request — retry in a few seconds.", - 409: "Request conflicts with current resource state — review concurrent requests before retrying.", - 413: "Text exceeds the provider's size limit — shorten the sample.", - 422: "Provider rejected the payload — check input format and parameter values against the API spec.", - 429: "Hit the provider's rate/quota — wait at least 1 min and retry. Request a quota increase or contact Kaapi if persistent.", - 500: "Typically transient — retry in a few seconds. If it persists, contact Kaapi.", - 502: "Provider gateway error — typically transient, retry in a few seconds.", - 503: "Provider temporarily down or overloaded — retry in a few seconds.", - 504: "Provider took too long — retry with a shorter sample.", -} -_DEFAULT_STATUS_HINT = "If this persists, contact Kaapi." - - -class TTSSynthesisError(Exception): - """A single synthesis failed; `message` is safe to store on the result row.""" - - def __init__(self, message: str, *, retryable: bool = False) -> None: - super().__init__(message) - self.message = message - self.retryable = retryable - - -@dataclass(frozen=True) -class SynthesizedAudio: - wav_bytes: bytes - duration_seconds: float - - -def create_tts_client(provider: Provider, credentials: dict[str, Any]) -> TTSClient: - """Build the SDK client for a sync TTS provider from stored credentials. - - Raises: - ValueError: If the credentials lack an API key or the provider isn't sync-capable - """ - if provider == Provider.SARVAMAI: - return SarvamAIProvider.create_client(credentials) - if provider == Provider.ELEVENLABS: - return ElevenlabsAIProvider.create_client(credentials) - raise ValueError(f"Provider '{provider.value}' has no synchronous TTS client") - - -def _api_error( - *, - tag: str, - status_code: int | None, - body: Any, - model: str, - function_name: str, -) -> TTSSynthesisError: - hint = _STATUS_HINTS.get(status_code or 0, _DEFAULT_STATUS_HINT) - message = f"[{tag}] TTS request failed (code: {status_code}): {body}. {hint}" - # 5xx is provider-side (alert-worthy); 4xx is caller's fault (noise if alerted) - log = logger.error if status_code and status_code >= 500 else logger.warning - log( - f"[{function_name}] {message} | provider={tag.lower()}, model={model}", - exc_info=True, - ) - return TTSSynthesisError( - message, retryable=status_code in TTS_RETRYABLE_STATUS_CODES - ) - - -def _network_error( - *, - err: httpx.TransportError, - tag: str, - model: str, - function_name: str, -) -> TTSSynthesisError: - code = type(err).__name__ - if isinstance(err, httpx.TimeoutException): - message = ( - f"[{KAAPI_TAG}] TTS request to {tag.title()} timed out (code: {code}). " - f"Retry with a shorter sample; if persistent, contact Kaapi." - ) - else: - message = ( - f"[{KAAPI_TAG}] Could not reach {tag.title()} (code: {code}): {err}. " - f"Network/DNS issue reaching the provider — check connectivity; if " - f"persistent, contact Kaapi." - ) - logger.error( - f"[{function_name}] {message} | provider={tag.lower()}, model={model}", - exc_info=True, - ) - return TTSSynthesisError(message, retryable=True) - - -def _normalize_pcm( - pcm_data: bytes, *, channels: int, sample_width: int, frame_rate: int -) -> bytes: - """Convert PCM to the evaluation format (24 kHz, 16-bit, mono) if it differs.""" - if ( - channels == TTS_CHANNELS - and sample_width == TTS_SAMPLE_WIDTH_BYTES - and frame_rate == TTS_SAMPLE_RATE_HZ - ): - return pcm_data - - segment = AudioSegment( - data=pcm_data, - sample_width=sample_width, - frame_rate=frame_rate, - channels=channels, - ) - segment = segment.set_channels(TTS_CHANNELS) - segment = segment.set_sample_width(TTS_SAMPLE_WIDTH_BYTES) - segment = segment.set_frame_rate(TTS_SAMPLE_RATE_HZ) - return bytes(segment.raw_data or b"") - - -def _wav_chunk_to_pcm(wav_bytes: bytes) -> bytes: - with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file: - channels = wav_file.getnchannels() - sample_width = wav_file.getsampwidth() - frame_rate = wav_file.getframerate() - frames = wav_file.readframes(wav_file.getnframes()) - return _normalize_pcm( - frames, channels=channels, sample_width=sample_width, frame_rate=frame_rate - ) - - -def _to_synthesized_audio(pcm_data: bytes) -> SynthesizedAudio: - wav_bytes = pcm_to_wav( - pcm_data, - sample_rate=TTS_SAMPLE_RATE_HZ, - bits_per_sample=TTS_SAMPLE_WIDTH_BYTES * _BITS_PER_BYTE, - channels=TTS_CHANNELS, - ) - duration = calculate_duration( - len(pcm_data), - sample_rate=TTS_SAMPLE_RATE_HZ, - bits_per_sample=TTS_SAMPLE_WIDTH_BYTES * _BITS_PER_BYTE, - channels=TTS_CHANNELS, - ) - return SynthesizedAudio(wav_bytes=wav_bytes, duration_seconds=duration) - - -def synthesize_sarvam( - *, - client: SarvamAI, - text: str, - model: str, - language_code: str, - speaker: str, -) -> SynthesizedAudio: - """Synthesize one text with Sarvam and return it as normalized WAV. - - Raises: - TTSSynthesisError: On any provider, network, or decoding failure - """ - try: - response = client.text_to_speech.convert( - text=text, - model=model, - target_language_code=language_code, - speaker=speaker, - pace=SARVAM_PACE, - speech_sample_rate=TTS_SAMPLE_RATE_HZ, - request_options=_SARVAM_REQUEST_OPTIONS, - ) - except SarvamApiError as e: - raise _api_error( - tag=SARVAM_TAG, - status_code=e.status_code, - body=e.body, - model=model, - function_name="synthesize_sarvam", - ) from e - except httpx.TransportError as e: - raise _network_error( - err=e, tag=SARVAM_TAG, model=model, function_name="synthesize_sarvam" - ) from e - - if not response.audios: - message = ( - f"[{SARVAM_TAG}] TTS response contains no audio data. Sarvam accepted " - f"the request but returned an empty audio list — retry later; if it " - f"persists, contact Kaapi." - ) - logger.warning( - f"[synthesize_sarvam] {message} | provider=sarvam, model={model}, " - f"request_id={response.request_id}" - ) - raise TTSSynthesisError(message) - - # Long inputs come back as several base64 chunks, each a complete WAV file. - # Joining the base64 strings (or the decoded bytes) would leave the later - # chunks' RIFF headers inside the PCM stream as audible clicks, so each chunk - # is decoded and stripped separately and the PCM re-wrapped once. - pcm_parts: list[bytes] = [] - try: - for audio_b64 in response.audios: - pcm_parts.append(_wav_chunk_to_pcm(base64.b64decode(audio_b64))) - except (ValueError, wave.Error, EOFError) as e: - message = ( - f"[{KAAPI_TAG}] Failed to decode Sarvam audio (code: {type(e).__name__}): " - f"{e}. The response was not valid base64 WAV — contact Kaapi if this persists." - ) - logger.error( - f"[synthesize_sarvam] {message} | provider=sarvam, model={model}, " - f"request_id={response.request_id}", - exc_info=True, - ) - raise TTSSynthesisError(message) from e - - logger.info( - f"[synthesize_sarvam] Synthesized | model={model}, " - f"chunks={len(response.audios)}, request_id={response.request_id}" - ) - return _to_synthesized_audio(b"".join(pcm_parts)) - - -def synthesize_elevenlabs( - *, - client: ElevenLabs, - text: str, - model: str, - language_code: str, - voice_id: str, -) -> SynthesizedAudio: - """Synthesize one text with ElevenLabs (raw 24 kHz PCM) and return it as WAV. - - Raises: - TTSSynthesisError: On any provider or network failure - """ - convert_kwargs: dict[str, Any] = {} - elevenlabs_language = to_elevenlabs_language_code(language_code) - if elevenlabs_language: - convert_kwargs["language_code"] = elevenlabs_language - - # The SDK streams lazily, so HTTP errors can surface while joining the chunks. - try: - chunks = client.text_to_speech.convert( - voice_id=voice_id, - text=text, - model_id=model, - output_format=ELEVENLABS_OUTPUT_FORMAT, - request_options=_ELEVENLABS_REQUEST_OPTIONS, - **convert_kwargs, - ) - pcm_data = b"".join(chunks) - except ElevenLabsApiError as e: - raise _api_error( - tag=ELEVENLABS_TAG, - status_code=e.status_code, - body=e.body, - model=model, - function_name="synthesize_elevenlabs", - ) from e - except httpx.TransportError as e: - raise _network_error( - err=e, - tag=ELEVENLABS_TAG, - model=model, - function_name="synthesize_elevenlabs", - ) from e - - if not pcm_data: - message = ( - f"[{ELEVENLABS_TAG}] TTS response contains no audio data. ElevenLabs " - f"accepted the request but returned an empty audio stream — retry " - f"later; if it persists, contact Kaapi." - ) - logger.warning( - f"[synthesize_elevenlabs] {message} | provider=elevenlabs, " - f"model={model}, voice_id={voice_id}" - ) - raise TTSSynthesisError(message) - - logger.info( - f"[synthesize_elevenlabs] Synthesized | model={model}, " - f"voice_id={voice_id}, pcm_bytes={len(pcm_data)}" - ) - return _to_synthesized_audio(pcm_data) - - -def _is_retryable_synthesis_error(err: BaseException) -> bool: - return isinstance(err, TTSSynthesisError) and err.retryable - - -@retry( - retry=retry_if_exception(_is_retryable_synthesis_error), - wait=wait_random_exponential( - multiplier=TTS_SYNC_BACKOFF_BASE_SECONDS, max=TTS_SYNC_BACKOFF_MAX_SECONDS - ), - stop=stop_after_attempt(TTS_SYNC_MAX_ATTEMPTS), - before_sleep=before_sleep_log(logger, logging.INFO), - reraise=True, -) -def synthesize_tts( - *, - client: TTSClient, - provider: Provider, - text: str, - model: str, - language_code: str, - voice: str, -) -> SynthesizedAudio: - """Dispatch to the provider's synthesizer, retrying transient failures. - - Raises: - TTSSynthesisError: On a non-retryable failure or after the final attempt - """ - # Dispatch on the registry provider rather than isinstance so the client - # can be substituted (e.g. a mock) without matching the SDK class. - if provider == Provider.SARVAMAI: - return synthesize_sarvam( - client=cast(SarvamAI, client), - text=text, - model=model, - language_code=language_code, - speaker=voice, - ) - if provider == Provider.ELEVENLABS: - return synthesize_elevenlabs( - client=cast(ElevenLabs, client), - text=text, - model=model, - language_code=language_code, - voice_id=voice, - ) - - message = ( - f"[{KAAPI_TAG}] No synchronous synthesizer for provider " - f"'{provider.value}'. Contact Kaapi." - ) - logger.error(f"[synthesize_tts] {message} | model={model}") - raise TTSSynthesisError(message) diff --git a/backend/app/tests/celery/test_tts_tasks.py b/backend/app/tests/celery/test_tts_tasks.py index d53affbf5..803e134d6 100644 --- a/backend/app/tests/celery/test_tts_tasks.py +++ b/backend/app/tests/celery/test_tts_tasks.py @@ -4,81 +4,61 @@ from asgi_correlation_id import correlation_id from app.celery.tasks import job_execution -from app.celery.utils import start_tts_sync_chunk +from app.celery.utils import start_tts_sync_generation -CHUNK_KWARGS: dict[str, Any] = { +SYNC_KWARGS: dict[str, Any] = { "organization_id": 3, "model": "bulbul:v3", - "result_ids": [11, 12], "language_code": "hi-IN", } -class TestStartTTSSyncChunk: - def test_enqueues_task_with_kwargs_and_trace_headers(self) -> None: - # apply_async is the broker boundary; nothing is published in tests. - with patch.object( - job_execution.run_tts_sync_chunk, - "apply_async", - return_value=MagicMock(id="celery-123"), - ) as apply_async: - task_id = start_tts_sync_chunk( - project_id=7, job_id="42", trace_id="trace-abc", **CHUNK_KWARGS - ) +def test_start_tts_sync_generation_enqueues_task_kwargs() -> None: + # apply_async is the broker boundary; nothing is published in tests. + with patch.object( + job_execution.run_tts_sync_generation, + "apply_async", + return_value=MagicMock(id="celery-123"), + ) as apply_async: + task_id = start_tts_sync_generation( + project_id=7, job_id="42", trace_id="trace-abc", **SYNC_KWARGS + ) - assert task_id == "celery-123" - call = apply_async.call_args.kwargs - assert call["kwargs"] == { - "project_id": 7, - "job_id": "42", - "trace_id": "trace-abc", - **CHUNK_KWARGS, - } - assert "otel" in call["headers"] + assert task_id == "celery-123" + assert apply_async.call_args.kwargs["kwargs"] == { + "project_id": 7, + "job_id": "42", + "trace_id": "trace-abc", + **SYNC_KWARGS, + } - def test_trace_id_defaults_to_na(self) -> None: - with patch.object( - job_execution.run_tts_sync_chunk, - "apply_async", - return_value=MagicMock(id="celery-123"), - ) as apply_async: - start_tts_sync_chunk(project_id=7, job_id="42", **CHUNK_KWARGS) - assert apply_async.call_args.kwargs["kwargs"]["trace_id"] == "N/A" +def test_run_tts_sync_generation_forwards_kwargs_and_trace() -> None: + seen: dict[str, Any] = {} + def fake_execute(**kwargs: Any) -> dict[str, Any]: + seen.update(kwargs, correlation_id=correlation_id.get()) + return {"success": True, "processed": 2} -class TestRunTTSSyncChunk: - def test_runs_chunk_with_task_context_and_trace(self) -> None: - seen: dict[str, Any] = {} + with patch( + "app.services.tts_evaluations.sync_generation.execute_tts_sync_generation", + side_effect=fake_execute, + ): + result = job_execution.run_tts_sync_generation.apply( + kwargs={ + "project_id": 7, + "job_id": "42", + "trace_id": "trace-abc", + **SYNC_KWARGS, + }, + task_id="task-xyz", + ).get() - def fake_execute(**kwargs: Any) -> dict[str, Any]: - seen.update(kwargs) - seen["correlation_id"] = correlation_id.get() - return {"success": True, "processed": 2} - - with patch( - "app.services.tts_evaluations.sync_generation.execute_tts_sync_chunk", - side_effect=fake_execute, - ): - result = job_execution.run_tts_sync_chunk.apply( - kwargs={ - "project_id": 7, - "job_id": "42", - "trace_id": "trace-abc", - **CHUNK_KWARGS, - }, - task_id="task-xyz", - ).get() - - assert result == {"success": True, "processed": 2} - assert seen["project_id"] == 7 - assert seen["job_id"] == "42" - assert seen["task_id"] == "task-xyz" - assert seen["task_instance"] is not None - assert seen["correlation_id"] == "trace-abc" - for key, value in CHUNK_KWARGS.items(): - assert seen[key] == value - - def test_task_is_on_default_queue_at_low_priority(self) -> None: - assert job_execution.run_tts_sync_chunk.queue == "default" - assert job_execution.run_tts_sync_chunk.priority == 2 + assert result == {"success": True, "processed": 2} + assert {k: seen[k] for k in ("project_id", "job_id", "task_id")} == { + "project_id": 7, + "job_id": "42", + "task_id": "task-xyz", + } + assert {k: seen[k] for k in SYNC_KWARGS} == SYNC_KWARGS + assert seen["correlation_id"] == "trace-abc" diff --git a/backend/app/tests/crud/test_language.py b/backend/app/tests/crud/test_language.py deleted file mode 100644 index 0398563fe..000000000 --- a/backend/app/tests/crud/test_language.py +++ /dev/null @@ -1,17 +0,0 @@ -from sqlmodel import Session - -from app.crud.language import get_language_by_locale, get_language_locale_by_id -from app.models.language import Language -from app.tests.utils.utils import get_non_existent_id - - -def test_get_language_locale_by_id_returns_seeded_locale(db: Session) -> None: - hindi = get_language_by_locale(session=db, locale="hi") - - assert get_language_locale_by_id(session=db, language_id=hindi.id) == "hi" - - -def test_get_language_locale_by_id_returns_none_when_missing(db: Session) -> None: - missing_id = get_non_existent_id(db, Language) - - assert get_language_locale_by_id(session=db, language_id=missing_id) is None diff --git a/backend/app/tests/crud/tts_evaluations/test_batch.py b/backend/app/tests/crud/tts_evaluations/test_batch.py deleted file mode 100644 index bc7c452f0..000000000 --- a/backend/app/tests/crud/tts_evaluations/test_batch.py +++ /dev/null @@ -1,180 +0,0 @@ -import pytest -from sqlmodel import Session, select - -from app.crud.tts_evaluations.batch import ( - TTSBatchSubmissionError, - start_tts_evaluation_batch, -) -from app.models import EvaluationRun -from app.models.batch_job import BatchJob, BatchJobType -from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResult -from app.services.tts_evaluations.constants import ( - TTS_MODEL_REGISTRY, - TTSExecutionModeEnum, - TTSModelSpec, -) -from app.core.providers import Provider -from app.tests.utils.auth import TestAuthContext -from app.tests.utils.tts_evaluation import ( - create_test_tts_result_row, - create_test_tts_run_with_dataset, - fake_gemini_batch, -) - -MODULE = "app.crud.tts_evaluations.batch" -GEMINI = "gemini-2.5-pro-preview-tts" -# A second batch-mode model so partial (per-model) failure is observable. -SECOND_BATCH_MODEL = "gemini-test-batch-tts" - - -@pytest.fixture -def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: - return create_test_tts_run_with_dataset( - db, - organization_id=user_api_key.organization_id, - project_id=user_api_key.project_id, - models=[GEMINI, "bulbul:v3"], - ) - - -@pytest.fixture -def second_batch_model(monkeypatch: pytest.MonkeyPatch) -> str: - monkeypatch.setitem( - TTS_MODEL_REGISTRY, - SECOND_BATCH_MODEL, - TTSModelSpec( - provider=Provider.GOOGLE_AISTUDIO, - execution_mode=TTSExecutionModeEnum.BATCH, - default_voice="Puck", - ), - ) - return SECOND_BATCH_MODEL - - -def _submit( - db: Session, run: EvaluationRun, results: list[TTSResult], models: list[str] -) -> dict: - return start_tts_evaluation_batch( - session=db, - run=run, - results=results, - models=models, - org_id=run.organization_id, - project_id=run.project_id, - ) - - -def _batch_jobs_for(db: Session, run_id: int) -> list[BatchJob]: - jobs = db.exec( - select(BatchJob).where(BatchJob.job_type == BatchJobType.TTS_EVALUATION.value) - ).all() - return [j for j in jobs if j.config.get("evaluation_run_id") == run_id] - - -class TestStartTTSEvaluationBatch: - def test_rejects_sync_model(self, db: Session, run: EvaluationRun) -> None: - with fake_gemini_batch(MODULE) as fake: - with pytest.raises(ValueError, match="'bulbul:v3' is not a batch-mode"): - _submit(db, run, [], [GEMINI, "bulbul:v3"]) - - assert fake.submitted == {} - - def test_submits_batch_rows_and_links_run( - self, db: Session, run: EvaluationRun - ) -> None: - gemini_rows = [create_test_tts_result_row(db, run=run) for _ in range(2)] - sync_row = create_test_tts_result_row(db, run=run, provider="bulbul:v3") - - with fake_gemini_batch(MODULE) as fake: - outcome = _submit(db, run, [*gemini_rows, sync_row], [GEMINI]) - - [job] = _batch_jobs_for(db, run.id) - assert outcome == { - "success": True, - "run_id": run.id, - "batch_jobs": { - GEMINI: { - "batch_job_id": job.id, - "provider_batch_id": f"batches/{GEMINI}", - } - }, - "result_count": 2, - } - assert [r["key"] for r in fake.submitted[f"models/{GEMINI}"]] == [ - str(r.id) for r in gemini_rows - ] - assert job.config["tts_provider"] == GEMINI - assert job.config["voice_name"] == "Kore" - assert job.provider == "google-aistudio" - db.refresh(run) - assert run.status == "processing" - assert run.batch_job_id == job.id - - def test_all_models_failing_raises_and_fails_rows( - self, db: Session, run: EvaluationRun - ) -> None: - row = create_test_tts_result_row(db, run=run) - sync_row = create_test_tts_result_row(db, run=run, provider="bulbul:v3") - - with fake_gemini_batch(MODULE, {f"models/{GEMINI}": RuntimeError("quota")}): - with pytest.raises(TTSBatchSubmissionError): - _submit(db, run, [row, sync_row], [GEMINI]) - - db.refresh(row) - db.refresh(sync_row) - db.refresh(run) - assert row.status == JobStatus.FAILED.value - assert row.error_message == f"Batch submission failed for {GEMINI}: quota" - assert sync_row.status == JobStatus.PENDING.value - assert run.batch_job_id is None - [job] = _batch_jobs_for(db, run.id) - assert job.provider_status == "failed" - - def test_model_without_rows_is_skipped( - self, db: Session, run: EvaluationRun - ) -> None: - with fake_gemini_batch(MODULE) as fake: - with pytest.raises(TTSBatchSubmissionError): - _submit(db, run, [], [GEMINI]) - - assert fake.submitted == {} - assert _batch_jobs_for(db, run.id) == [] - - def test_one_model_failing_only_fails_its_rows( - self, db: Session, run: EvaluationRun, second_batch_model: str - ) -> None: - failing_row = create_test_tts_result_row(db, run=run) - ok_row = create_test_tts_result_row(db, run=run, provider=second_batch_model) - - with fake_gemini_batch(MODULE, {f"models/{GEMINI}": RuntimeError("bad key")}): - outcome = _submit( - db, run, [failing_row, ok_row], [GEMINI, second_batch_model] - ) - - assert list(outcome["batch_jobs"]) == [second_batch_model] - assert outcome["result_count"] == 1 - db.refresh(failing_row) - db.refresh(ok_row) - db.refresh(run) - assert failing_row.status == JobStatus.FAILED.value - assert ok_row.status == JobStatus.PENDING.value - assert ( - run.batch_job_id - == outcome["batch_jobs"][second_batch_model]["batch_job_id"] - ) - - def test_run_links_first_submitted_job( - self, db: Session, run: EvaluationRun, second_batch_model: str - ) -> None: - rows = [ - create_test_tts_result_row(db, run=run), - create_test_tts_result_row(db, run=run, provider=second_batch_model), - ] - - with fake_gemini_batch(MODULE): - outcome = _submit(db, run, rows, [GEMINI, second_batch_model]) - - assert len(_batch_jobs_for(db, run.id)) == 2 - db.refresh(run) - assert run.batch_job_id == outcome["batch_jobs"][GEMINI]["batch_job_id"] diff --git a/backend/app/tests/crud/tts_evaluations/test_cron.py b/backend/app/tests/crud/tts_evaluations/test_cron.py index 48ac80498..c50747102 100644 --- a/backend/app/tests/crud/tts_evaluations/test_cron.py +++ b/backend/app/tests/crud/tts_evaluations/test_cron.py @@ -1,75 +1,71 @@ +from collections.abc import Iterator +from datetime import timedelta +from typing import Any from unittest.mock import MagicMock, patch import pytest from sqlmodel import Session from app.core.batch import BatchJobState -from app.crud.tts_evaluations.cron import poll_tts_run +from app.crud.tts_evaluations.cron import ( + poll_all_pending_tts_evaluations, + poll_tts_run, +) from app.models import EvaluationRun -from app.models.batch_job import BatchJob, BatchJobType +from app.models.batch_job import BatchJob from app.models.job import JobStatus +from app.models.tts_evaluation import TTSResult +from app.services.tts_evaluations.constants import TTS_SYNC_STALE_AFTER_SECONDS from app.tests.utils.auth import TestAuthContext from app.tests.utils.tts_evaluation import ( + create_test_tts_batch_job, create_test_tts_result_row, create_test_tts_run_with_dataset, ) +MODULE = "app.crud.tts_evaluations.cron" GEMINI = "gemini-2.5-pro-preview-tts" +SARVAM = "bulbul:v3" -@pytest.fixture -def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: +def _processing_run( + db: Session, auth: TestAuthContext, models: list[str], stale: bool = False +) -> EvaluationRun: run = create_test_tts_run_with_dataset( db, - organization_id=user_api_key.organization_id, - project_id=user_api_key.project_id, - models=[GEMINI, "bulbul:v3"], + organization_id=auth.organization_id, + project_id=auth.project_id, + models=models, ) run.status = "processing" + if stale: + run.inserted_at -= timedelta(seconds=TTS_SYNC_STALE_AFTER_SECONDS + 60) db.add(run) db.commit() return run -def _terminal_batch_job( - db: Session, run: EvaluationRun, state: BatchJobState, error: str | None = None -) -> BatchJob: - job = BatchJob( - provider="google-aistudio", - job_type=BatchJobType.TTS_EVALUATION.value, - config={"evaluation_run_id": run.id, "tts_provider": GEMINI}, - provider_batch_id="batches/1", - provider_status=state.value, - error_message=error, - total_items=1, - organization_id=run.organization_id, - project_id=run.project_id, - ) - db.add(job) - db.commit() - db.refresh(job) - return job - - -async def _poll(db: Session, run: EvaluationRun, expected_dispatches: int = 0) -> dict: - # Gemini is never reached: batch jobs are either terminal or polled via the - # patched poll_batch_status in the tests that need it. +@pytest.fixture +def dispatch() -> Iterator[MagicMock]: + # Celery enqueue of result processing is the boundary. with patch( - "app.crud.tts_evaluations.cron.start_tts_result_processing", - return_value="celery-task", + f"{MODULE}.start_tts_result_processing", return_value="celery-task" ) as dispatch: - outcome = await poll_tts_run( - session=db, run=run, batch_provider=MagicMock(), org_id=run.organization_id - ) - assert dispatch.call_count == expected_dispatches - if expected_dispatches: - assert dispatch.call_args.kwargs["evaluation_run_id"] == run.id - assert dispatch.call_args.kwargs["tts_provider"] == GEMINI - return outcome + yield dispatch -def _provider_reports(db: Session, state: BatchJobState): - def fake_poll(*, session: Session, provider: object, batch_job: BatchJob) -> None: +@pytest.fixture +def gemini_client() -> Iterator[MagicMock]: + with patch(f"{MODULE}.GeminiClient") as client: + yield client + + +def _gemini_reports(state: BatchJobState | Exception) -> Any: + """Stand-in for the Gemini status call made while polling a batch job.""" + + def fake_poll(*, session: Session, batch_job: BatchJob, **_kwargs: Any) -> None: + if isinstance(state, Exception): + raise state batch_job.provider_status = state.value session.add(batch_job) session.commit() @@ -79,113 +75,217 @@ def fake_poll(*, session: Session, provider: object, batch_job: BatchJob) -> Non ) +async def _poll(db: Session, run: EvaluationRun) -> dict[str, Any]: + return await poll_tts_run(session=db, run=run, org_id=run.organization_id) + + +def _refresh(db: Session, *rows: TTSResult | EvaluationRun) -> None: + for row in rows: + db.refresh(row) + + @pytest.mark.asyncio -class TestPollTTSRunFinalize: - async def test_succeeded_batch_with_sync_rows_pending_stays_processing( - self, db: Session, run: EvaluationRun +@pytest.mark.usefixtures("gemini_client") +class TestPollTTSRunGeminiBatches: + @pytest.mark.parametrize( + "initial_state", [BatchJobState.RUNNING, BatchJobState.SUCCEEDED] + ) + async def test_succeeded_batch_dispatches_result_processing( + self, + db: Session, + user_api_key: TestAuthContext, + dispatch: MagicMock, + initial_state: BatchJobState, ) -> None: - _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - create_test_tts_result_row(db, run=run, provider="bulbul:v3") - - outcome = await _poll(db, run) + run = _processing_run(db, user_api_key, [GEMINI]) + batch_job = create_test_tts_batch_job( + db, run=run, provider_status=initial_state.value + ) + create_test_tts_result_row(db, run=run, provider=GEMINI) - assert outcome["action"] == "completed" - assert outcome["current_status"] == "processing" - db.refresh(run) - assert run.status == "processing" + with _gemini_reports(BatchJobState.SUCCEEDED): + outcome = await _poll(db, run) - async def test_all_done_completes_with_failure_count( - self, db: Session, run: EvaluationRun + assert (outcome["action"], outcome["current_status"]) == ( + "dispatched", + "processing", + ) + dispatched = dispatch.call_args.kwargs + assert ( + dispatched["evaluation_run_id"], + dispatched["tts_provider"], + dispatched["provider_batch_id"], + ) == (run.id, GEMINI, batch_job.provider_batch_id) + + async def test_succeeded_batch_with_no_pending_rows_is_not_reprocessed( + self, db: Session, user_api_key: TestAuthContext, dispatch: MagicMock ) -> None: - _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) + run = _processing_run(db, user_api_key, [GEMINI]) + create_test_tts_batch_job( + db, run=run, provider_status=BatchJobState.SUCCEEDED.value + ) create_test_tts_result_row( - db, run=run, provider="bulbul:v3", status=JobStatus.FAILED + db, run=run, provider=GEMINI, status=JobStatus.SUCCESS ) outcome = await _poll(db, run) - assert outcome["current_status"] == "completed" - assert outcome["error"] == "1 synthesis(es) failed" - db.refresh(run) + dispatch.assert_not_called() + assert outcome["action"] == "completed" + _refresh(db, run) assert run.status == "completed" - assert run.error_message == "1 synthesis(es) failed" - async def test_failed_batch_error_takes_precedence( - self, db: Session, run: EvaluationRun + @pytest.mark.parametrize( + ("state", "batch_error", "expected_error"), + [ + ( + BatchJobState.FAILED, + "quota exceeded", + "Gemini batch ended: quota exceeded", + ), + (BatchJobState.CANCELLED, None, "Gemini batch ended: JOB_STATE_CANCELLED"), + (BatchJobState.EXPIRED, None, "Gemini batch ended: JOB_STATE_EXPIRED"), + ], + ) + async def test_unsuccessful_batch_fails_only_that_models_rows( + self, + db: Session, + user_api_key: TestAuthContext, + dispatch: MagicMock, + state: BatchJobState, + batch_error: str | None, + expected_error: str, ) -> None: - _terminal_batch_job(db, run, BatchJobState.FAILED, error="quota exceeded") - create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) + run = _processing_run(db, user_api_key, [GEMINI, SARVAM]) + create_test_tts_batch_job( + db, run=run, provider_status=state.value, error_message=batch_error + ) + gemini_row = create_test_tts_result_row(db, run=run, provider=GEMINI) + sarvam_row = create_test_tts_result_row(db, run=run, provider=SARVAM) outcome = await _poll(db, run) - assert outcome["action"] == "failed" - assert outcome["current_status"] == "completed" - assert outcome["error"] == f"{GEMINI}: quota exceeded" - db.refresh(run) - assert run.error_message == f"{GEMINI}: quota exceeded" + _refresh(db, gemini_row, sarvam_row, run) + assert (gemini_row.status, gemini_row.error_message) == ( + JobStatus.FAILED.value, + expected_error, + ) + assert sarvam_row.status == JobStatus.PENDING.value + assert run.status == outcome["current_status"] == "processing" - async def test_run_vanishing_before_finalize_falls_back_to_poll_state( - self, db: Session, run: EvaluationRun + async def test_running_batch_leaves_rows_pending( + self, db: Session, user_api_key: TestAuthContext, dispatch: MagicMock ) -> None: - _terminal_batch_job(db, run, BatchJobState.FAILED, error="expired") + run = _processing_run(db, user_api_key, [GEMINI]) + create_test_tts_batch_job( + db, run=run, provider_status=BatchJobState.PENDING.value + ) + row = create_test_tts_result_row(db, run=run, provider=GEMINI) - with patch( - "app.crud.tts_evaluations.cron.finalize_tts_run_status", return_value=None - ): + with _gemini_reports(BatchJobState.RUNNING): outcome = await _poll(db, run) - assert outcome["current_status"] == "processing" - assert outcome["error"] == f"{GEMINI}: expired" - - -@pytest.mark.asyncio -class TestPollTTSRunDispatch: - async def test_no_batch_jobs_fails_run( - self, db: Session, run: EvaluationRun + assert outcome["action"] == "no_change" + dispatch.assert_not_called() + _refresh(db, row) + assert row.status == JobStatus.PENDING.value + + async def test_gemini_client_failure_fails_only_gemini_rows( + self, + db: Session, + user_api_key: TestAuthContext, + gemini_client: MagicMock, ) -> None: - outcome = await _poll(db, run) + gemini_client.from_credentials.side_effect = ValueError("no gemini key") + run = _processing_run(db, user_api_key, [GEMINI, SARVAM]) + create_test_tts_batch_job( + db, run=run, provider_status=BatchJobState.RUNNING.value + ) + gemini_row = create_test_tts_result_row(db, run=run, provider=GEMINI) + sarvam_row = create_test_tts_result_row(db, run=run, provider=SARVAM) - assert (outcome["action"], outcome["error"]) == ( - "failed", - "No batch jobs found", + await _poll(db, run) + + _refresh(db, gemini_row, sarvam_row) + assert (gemini_row.status, gemini_row.error_message) == ( + JobStatus.FAILED.value, + "Gemini client initialization failed: no gemini key", ) - db.refresh(run) - assert run.status == "failed" + assert sarvam_row.status == JobStatus.PENDING.value - async def test_still_running_batch_is_no_change( - self, db: Session, run: EvaluationRun + async def test_batch_job_never_created_is_skipped_and_run_finalizes( + self, db: Session, user_api_key: TestAuthContext ) -> None: - _terminal_batch_job(db, run, BatchJobState.RUNNING) + # Regression: a job whose creation failed (no provider_batch_id) was polled + # each tick, and the poll error kept the run stuck in processing. + run = _processing_run(db, user_api_key, [GEMINI, SARVAM]) + create_test_tts_batch_job( + db, run=run, provider_status="failed", provider_batch_id=None + ) + create_test_tts_result_row( + db, run=run, provider=GEMINI, status=JobStatus.FAILED + ) + create_test_tts_result_row( + db, run=run, provider=SARVAM, status=JobStatus.SUCCESS + ) - with _provider_reports(db, BatchJobState.RUNNING): + with _gemini_reports(ValueError("batch name must not be None")): outcome = await _poll(db, run) - assert outcome["action"] == "no_change" - db.refresh(run) - assert run.status == "processing" + assert outcome["action"] == "completed" + _refresh(db, run) + assert (run.status, run.error_message) == ( + "completed", + "1 synthesis(es) failed", + ) - async def test_newly_succeeded_batch_dispatches_processing( - self, db: Session, run: EvaluationRun + +@pytest.mark.asyncio +class TestPollTTSRunStaleModels: + @pytest.mark.parametrize( + ("model", "expected_error"), + [(SARVAM, "Synthesis did not complete"), (GEMINI, "No batch jobs found")], + ) + @pytest.mark.parametrize("stale", [False, True], ids=["fresh", "stale"]) + async def test_pending_rows_fail_only_after_stale_window( + self, + db: Session, + user_api_key: TestAuthContext, + model: str, + expected_error: str, + stale: bool, ) -> None: - _terminal_batch_job(db, run, BatchJobState.RUNNING) - create_test_tts_result_row(db, run=run) + run = _processing_run(db, user_api_key, [model], stale=stale) + row = create_test_tts_result_row(db, run=run, provider=model) - with _provider_reports(db, BatchJobState.SUCCEEDED): - outcome = await _poll(db, run, expected_dispatches=1) + await _poll(db, run) - assert (outcome["action"], outcome["current_status"]) == ( - "dispatched", - "processing", - ) + _refresh(db, row, run) + if stale: + assert (row.status, row.error_message) == ( + JobStatus.FAILED.value, + expected_error, + ) + assert run.status == "failed" + else: + assert row.status == JobStatus.PENDING.value + assert run.status == "processing" - async def test_already_succeeded_with_pending_rows_redispatches( - self, db: Session, run: EvaluationRun + +@pytest.mark.asyncio +class TestPollAllPendingTTSEvaluations: + async def test_finalizes_sync_only_run_without_batch_job( + self, db: Session, user_api_key: TestAuthContext ) -> None: - _terminal_batch_job(db, run, BatchJobState.SUCCEEDED) - create_test_tts_result_row(db, run=run) + run = _processing_run(db, user_api_key, [SARVAM]) + assert run.batch_job_id is None + create_test_tts_result_row( + db, run=run, provider=SARVAM, status=JobStatus.SUCCESS + ) - outcome = await _poll(db, run, expected_dispatches=1) + summary = await poll_all_pending_tts_evaluations(session=db) - assert outcome["action"] == "dispatched" + [detail] = [d for d in summary["details"] if d["run_id"] == run.id] + assert detail["action"] == "completed" + _refresh(db, run) + assert run.status == "completed" diff --git a/backend/app/tests/crud/tts_evaluations/test_result.py b/backend/app/tests/crud/tts_evaluations/test_result.py deleted file mode 100644 index f7f7cdb69..000000000 --- a/backend/app/tests/crud/tts_evaluations/test_result.py +++ /dev/null @@ -1,120 +0,0 @@ -from sqlmodel import Session - -from app.crud.tts_evaluations.result import ( - bulk_update_pending_tts_results, - list_pending_tts_results_by_ids, -) -from app.models import EvaluationRun -from app.models.job import JobStatus -from app.models.tts_evaluation import TTSResultUpdate -from app.tests.utils.auth import TestAuthContext -from app.tests.utils.tts_evaluation import ( - create_test_tts_result_row, - create_test_tts_run_with_dataset, -) - - -def _make_run(db: Session, auth: TestAuthContext) -> EvaluationRun: - return create_test_tts_run_with_dataset( - db, organization_id=auth.organization_id, project_id=auth.project_id - ) - - -class TestListPendingTTSResultsByIds: - def test_returns_only_pending_rows_of_run_in_id_order( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - other_run = _make_run(db, user_api_key) - pending_a = create_test_tts_result_row(db, run=run) - done = create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - pending_b = create_test_tts_result_row(db, run=run) - not_requested = create_test_tts_result_row(db, run=run) - foreign = create_test_tts_result_row(db, run=other_run) - - rows = list_pending_tts_results_by_ids( - session=db, - run_id=run.id, - result_ids=[pending_b.id, done.id, pending_a.id, foreign.id], - ) - - assert [r.id for r in rows] == [pending_a.id, pending_b.id] - assert not_requested.id not in {r.id for r in rows} - - def test_empty_ids_returns_empty_list( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - create_test_tts_result_row(db, run=run) - - assert ( - list_pending_tts_results_by_ids(session=db, run_id=run.id, result_ids=[]) - == [] - ) - - -class TestBulkUpdatePendingTTSResults: - def test_applies_success_and_failure_fields( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - ok = create_test_tts_result_row(db, run=run) - bad = create_test_tts_result_row(db, run=run) - - updated = bulk_update_pending_tts_results( - session=db, - updates=[ - TTSResultUpdate( - result_id=ok.id, - status=JobStatus.SUCCESS, - object_store_url="s3://bucket/a.wav", - metadata={"duration_seconds": 1.5, "size_bytes": 72044}, - ), - TTSResultUpdate( - result_id=bad.id, - status=JobStatus.FAILED, - error_message="[SARVAM] boom", - ), - ], - ) - - assert updated == 2 - db.refresh(ok) - db.refresh(bad) - assert ok.status == JobStatus.SUCCESS.value - assert ok.object_store_url == "s3://bucket/a.wav" - assert ok.metadata_ == {"duration_seconds": 1.5, "size_bytes": 72044} - assert ok.error_message is None - assert bad.status == JobStatus.FAILED.value - assert bad.error_message == "[SARVAM] boom" - assert bad.object_store_url is None - - def test_rows_no_longer_pending_are_not_clobbered( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - finished = create_test_tts_result_row( - db, run=run, status=JobStatus.SUCCESS, object_store_url="s3://b/keep.wav" - ) - pending = create_test_tts_result_row(db, run=run) - - updated = bulk_update_pending_tts_results( - session=db, - updates=[ - TTSResultUpdate( - result_id=finished.id, status=JobStatus.FAILED, error_message="late" - ), - TTSResultUpdate( - result_id=pending.id, status=JobStatus.FAILED, error_message="late" - ), - ], - ) - - assert updated == 1 - db.refresh(finished) - assert finished.status == JobStatus.SUCCESS.value - assert finished.object_store_url == "s3://b/keep.wav" - assert finished.error_message is None - - def test_empty_updates_returns_zero(self, db: Session) -> None: - assert bulk_update_pending_tts_results(session=db, updates=[]) == 0 diff --git a/backend/app/tests/crud/tts_evaluations/test_run.py b/backend/app/tests/crud/tts_evaluations/test_run.py index 2b690aa76..3e2e7254a 100644 --- a/backend/app/tests/crud/tts_evaluations/test_run.py +++ b/backend/app/tests/crud/tts_evaluations/test_run.py @@ -1,3 +1,4 @@ +import pytest from sqlmodel import Session from app.crud.tts_evaluations.run import finalize_tts_run_status @@ -10,57 +11,49 @@ ) from app.tests.utils.utils import get_non_existent_id +S, P, F = JobStatus.SUCCESS, JobStatus.PENDING, JobStatus.FAILED -def _make_run(db: Session, auth: TestAuthContext) -> EvaluationRun: + +@pytest.fixture +def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: return create_test_tts_run_with_dataset( - db, organization_id=auth.organization_id, project_id=auth.project_id + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, ) class TestFinalizeTTSRunStatus: - def test_all_success_completes_without_error( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - - finalized = finalize_tts_run_status(session=db, run_id=run.id) - - assert finalized is not None - assert finalized.status == "completed" - assert finalized.error_message is None - - def test_pending_rows_keep_run_processing( - self, db: Session, user_api_key: TestAuthContext + @pytest.mark.parametrize( + ("row_statuses", "expected_status", "expected_error"), + [ + ([S, S], "completed", None), + ([S, P, F], "processing", "1 synthesis(es) failed"), + ([S, F, F], "completed", "2 synthesis(es) failed"), + ([F, F], "failed", "2 synthesis(es) failed"), + ([], "failed", None), + ], + ids=["all_success", "pending", "partial_failure", "all_failed", "no_rows"], + ) + def test_status_derived_from_rows( + self, + db: Session, + run: EvaluationRun, + row_statuses: list[JobStatus], + expected_status: str, + expected_error: str | None, ) -> None: - run = _make_run(db, user_api_key) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - create_test_tts_result_row(db, run=run, status=JobStatus.PENDING) + for status in row_statuses: + create_test_tts_result_row(db, run=run, status=status) - finalized = finalize_tts_run_status(session=db, run_id=run.id) + finalize_tts_run_status(session=db, run_id=run.id) - assert finalized.status == "processing" db.refresh(run) - assert run.status == "processing" - - def test_failed_rows_produce_count_message( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) - create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) - - finalized = finalize_tts_run_status(session=db, run_id=run.id) - - assert finalized.status == "completed" - assert finalized.error_message == "2 synthesis(es) failed" + assert (run.status, run.error_message) == (expected_status, expected_error) def test_explicit_error_message_wins_over_failed_count( - self, db: Session, user_api_key: TestAuthContext + self, db: Session, run: EvaluationRun ) -> None: - run = _make_run(db, user_api_key) create_test_tts_result_row(db, run=run, status=JobStatus.FAILED) finalized = finalize_tts_run_status( @@ -69,16 +62,6 @@ def test_explicit_error_message_wins_over_failed_count( assert finalized.error_message == "gemini: JOB_STATE_FAILED" - def test_run_with_no_results_completes( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _make_run(db, user_api_key) - - finalized = finalize_tts_run_status(session=db, run_id=run.id) - - assert finalized.status == "completed" - assert finalized.error_message is None - def test_missing_run_returns_none(self, db: Session) -> None: missing_id = get_non_existent_id(db, EvaluationRun) diff --git a/backend/app/tests/models/test_tts_evaluation.py b/backend/app/tests/models/test_tts_evaluation.py deleted file mode 100644 index 63cb9e679..000000000 --- a/backend/app/tests/models/test_tts_evaluation.py +++ /dev/null @@ -1,52 +0,0 @@ -import pytest -from pydantic import ValidationError - -from app.models.job import JobStatus -from app.models.tts_evaluation import TTSEvaluationRunCreate, TTSResultUpdate - - -class TestTTSEvaluationRunCreate: - def test_accepts_sync_and_batch_models_together(self) -> None: - run = TTSEvaluationRunCreate( - run_name="mixed", - dataset_id=1, - models=[ - "gemini-2.5-pro-preview-tts", - "bulbul:v3", - "eleven_v3", - "eleven_v4", - ], - ) - - assert run.models == [ - "gemini-2.5-pro-preview-tts", - "bulbul:v3", - "eleven_v3", - "eleven_v4", - ] - - def test_defaults_to_gemini(self) -> None: - run = TTSEvaluationRunCreate(run_name="default", dataset_id=1) - - assert run.models == ["gemini-2.5-pro-preview-tts"] - - def test_rejects_unknown_model(self) -> None: - with pytest.raises( - ValidationError, match="Unsupported model\\(s\\): eleven_v9" - ): - TTSEvaluationRunCreate( - run_name="bad", dataset_id=1, models=["bulbul:v3", "eleven_v9"] - ) - - -class TestTTSResultUpdate: - def test_optional_fields_default_to_none(self) -> None: - update = TTSResultUpdate(result_id=7, status=JobStatus.FAILED) - - assert update.object_store_url is None - assert update.metadata is None - assert update.error_message is None - - def test_rejects_unknown_status(self) -> None: - with pytest.raises(ValidationError): - TTSResultUpdate(result_id=7, status="exploded") diff --git a/backend/app/tests/services/tts_evaluations/test_batch_job.py b/backend/app/tests/services/tts_evaluations/test_batch_job.py index c9c756fda..27823fc1c 100644 --- a/backend/app/tests/services/tts_evaluations/test_batch_job.py +++ b/backend/app/tests/services/tts_evaluations/test_batch_job.py @@ -1,6 +1,6 @@ import io from collections.abc import Iterator -from dataclasses import dataclass, field +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock, patch @@ -10,14 +10,12 @@ from sqlmodel import Session, select from app.crud.language import get_language_by_locale -from app.models import EvaluationRun +from app.models import EvaluationDataset, EvaluationRun from app.models.job import JobStatus from app.models.tts_evaluation import TTSResult from app.services.tts_evaluations.batch_job import execute_batch_submission from app.tests.utils.auth import TestAuthContext from app.tests.utils.tts_evaluation import ( - TEST_DATASET_URL, - create_test_tts_result_row, create_test_tts_run_with_dataset, fake_gemini_batch, use_test_session, @@ -25,46 +23,34 @@ from app.tests.utils.utils import get_non_existent_id MODULE = "app.services.tts_evaluations.batch_job" +CRUD_BATCH = "app.crud.tts_evaluations.batch" GEMINI = "gemini-2.5-pro-preview-tts" SARVAM = "bulbul:v3" ELEVEN = "eleven_v3" -def _csv(texts: list[str]) -> bytes: - return ("text\n" + "\n".join(texts) + "\n").encode() - - -@dataclass -class Harness: - csv: bytes = field(default_factory=lambda: _csv(["one", "two"])) - enqueued: list[dict[str, Any]] = field(default_factory=list) - enqueue_failures: set[str] = field(default_factory=set) - on_stream: Any = None - - def stream(self, url: str) -> io.BytesIO: - assert url == TEST_DATASET_URL - if self.on_stream is not None: - self.on_stream() - return io.BytesIO(self.csv) - - def enqueue(self, **kwargs: Any) -> str: - if kwargs["model"] in self.enqueue_failures: - raise ConnectionError("broker unreachable") - self.enqueued.append(kwargs) - return f"task-{len(self.enqueued)}" +def _csv(count: int) -> bytes: + return ("text\n" + "".join(f"sample {i}\n" for i in range(count))).encode() @pytest.fixture -def harness(db: Session) -> Iterator[Harness]: - h = Harness() +def boundaries(db: Session) -> Iterator[SimpleNamespace]: + state = SimpleNamespace(csv=_csv(2), enqueued=[], enqueue_errors={}) storage = MagicMock() - storage.stream.side_effect = h.stream + storage.stream.side_effect = lambda _url: io.BytesIO(state.csv) + + def enqueue(**kwargs: Any) -> str: + if kwargs["model"] in state.enqueue_errors: + raise state.enqueue_errors[kwargs["model"]] + state.enqueued.append(kwargs) + return f"task-{len(state.enqueued)}" + with ( use_test_session(MODULE, db), patch(f"{MODULE}.get_cloud_storage", return_value=storage), - patch(f"{MODULE}.start_tts_sync_chunk", side_effect=h.enqueue), + patch(f"{MODULE}.start_tts_sync_generation", side_effect=enqueue), ): - yield h + yield state def _run( @@ -79,279 +65,185 @@ def _run( ) -def _execute( - run: EvaluationRun, models: list[str], dataset_id: int | None = None -) -> dict[str, Any]: +def _execute(run: EvaluationRun, dataset_id: int | None = None) -> dict[str, Any]: return execute_batch_submission( project_id=run.project_id, job_id=str(run.id), task_id="celery-task", task_instance=MagicMock(), organization_id=run.organization_id, - dataset_id=dataset_id if dataset_id is not None else run.dataset_id, - models=models, + dataset_id=dataset_id or run.dataset_id, + models=run.providers, ) -def _results( - db: Session, run: EvaluationRun, model: str | None = None -) -> list[TTSResult]: - stmt = select(TTSResult).where(TTSResult.evaluation_run_id == run.id) - if model: - stmt = stmt.where(TTSResult.provider == model) - return list(db.exec(stmt.order_by(TTSResult.id)).all()) +def _results(db: Session, run: EvaluationRun, model: str) -> list[TTSResult]: + stmt = ( + select(TTSResult) + .where(TTSResult.evaluation_run_id == run.id, TTSResult.provider == model) + .order_by(TTSResult.id) + ) + return list(db.exec(stmt).all()) -@pytest.mark.usefixtures("harness") -class TestExecuteBatchSubmission: - def test_batch_only_submits_gemini_and_links_run( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - run = _run(db, user_api_key, [GEMINI]) +def _row_states( + db: Session, run: EvaluationRun, model: str +) -> set[tuple[str, str | None]]: + return {(r.status, r.error_message) for r in _results(db, run, model)} - with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: - outcome = _execute(run, [GEMINI]) - assert outcome["success"] is True - assert outcome["result_count"] == 2 - assert outcome["sync_chunks"] == {} - assert list(outcome["batch_jobs"]) == [GEMINI] - assert len(gemini.submitted[f"models/{GEMINI}"]) == 2 - assert harness.enqueued == [] - db.refresh(run) - assert run.status == "processing" - assert run.batch_job_id == outcome["batch_jobs"][GEMINI]["batch_job_id"] - - def test_sync_only_chunks_by_25_with_default_language( - self, db: Session, user_api_key: TestAuthContext, harness: Harness +@pytest.mark.usefixtures("boundaries") +class TestExecuteBatchSubmission: + def test_mixed_run_creates_rows_and_starts_every_model( + self, db: Session, user_api_key: TestAuthContext, boundaries: SimpleNamespace ) -> None: - harness.csv = _csv([f"sample {i}" for i in range(30)]) - run = _run(db, user_api_key, [SARVAM]) - - with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: - outcome = _execute(run, [SARVAM]) - - assert outcome["success"] is True - assert outcome["batch_jobs"] == {} - assert outcome["sync_chunks"] == {SARVAM: ["task-1", "task-2"]} - assert gemini.submitted == {} - result_ids = [r.id for r in _results(db, run)] - assert [c["result_ids"] for c in harness.enqueued] == [ - result_ids[:25], - result_ids[25:], - ] - for chunk in harness.enqueued: - assert chunk["model"] == SARVAM - assert chunk["language_code"] == "en-IN" - assert chunk["job_id"] == str(run.id) - assert chunk["organization_id"] == run.organization_id - assert chunk["trace_id"] == "N/A" + run = _run(db, user_api_key, [GEMINI, SARVAM, ELEVEN]) + + with fake_gemini_batch(CRUD_BATCH) as gemini: + outcome = _execute(run) + + assert (outcome["success"], outcome["result_count"]) == (True, 6) + for model in (GEMINI, SARVAM, ELEVEN): + assert [r.sample_text for r in _results(db, run, model)] == [ + "sample 0", + "sample 1", + ] + assert _row_states(db, run, model) == {(JobStatus.PENDING.value, None)} + assert {r["key"] for r in gemini.submitted[f"models/{GEMINI}"]} == { + str(r.id) for r in _results(db, run, GEMINI) + } + assert [ + (c["model"], c["job_id"], c["language_code"]) for c in boundaries.enqueued + ] == [(SARVAM, str(run.id), "en-IN"), (ELEVEN, str(run.id), "en-IN")] db.refresh(run) assert run.status == "processing" - assert run.batch_job_id is None - def test_dataset_language_is_passed_to_chunks( - self, db: Session, user_api_key: TestAuthContext, harness: Harness + @pytest.mark.parametrize( + ("locale", "expected_language"), + [(None, "en-IN"), ("hi", "hi-IN"), ("or", "od-IN")], + ) + def test_sync_language_comes_from_dataset_locale( + self, + db: Session, + user_api_key: TestAuthContext, + boundaries: SimpleNamespace, + locale: str | None, + expected_language: str, ) -> None: - hindi = get_language_by_locale(session=db, locale="hi") - run = _run(db, user_api_key, [ELEVEN], language_id=hindi.id) + language_id = ( + get_language_by_locale(session=db, locale=locale).id if locale else None + ) + run = _run(db, user_api_key, [SARVAM], language_id=language_id) - _execute(run, [ELEVEN]) + _execute(run) - assert [c["language_code"] for c in harness.enqueued] == ["hi-IN"] + assert [c["language_code"] for c in boundaries.enqueued] == [expected_language] - def test_mixed_models_route_rows_by_execution_mode( - self, db: Session, user_api_key: TestAuthContext, harness: Harness + @pytest.mark.parametrize("models", [[GEMINI], [GEMINI, SARVAM]]) + def test_gemini_submission_failure_fails_only_gemini_rows( + self, + db: Session, + user_api_key: TestAuthContext, + boundaries: SimpleNamespace, + models: list[str], ) -> None: - models = [GEMINI, SARVAM, ELEVEN] run = _run(db, user_api_key, models) - with fake_gemini_batch("app.crud.tts_evaluations.batch") as gemini: - outcome = _execute(run, models) - - assert outcome["result_count"] == 6 - assert list(outcome["batch_jobs"]) == [GEMINI] - assert set(outcome["sync_chunks"]) == {SARVAM, ELEVEN} - gemini_keys = {r["key"] for r in gemini.submitted[f"models/{GEMINI}"]} - assert gemini_keys == {str(r.id) for r in _results(db, run, GEMINI)} - enqueued_by_model = {c["model"]: c["result_ids"] for c in harness.enqueued} - assert enqueued_by_model == { - SARVAM: [r.id for r in _results(db, run, SARVAM)], - ELEVEN: [r.id for r in _results(db, run, ELEVEN)], - } - db.refresh(run) - assert run.status == "processing" - assert run.batch_job_id is not None - - def test_batch_failure_with_sync_models_continues( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - run = _run(db, user_api_key, [GEMINI, SARVAM]) - - with fake_gemini_batch( - "app.crud.tts_evaluations.batch", - {f"models/{GEMINI}": RuntimeError("quota")}, - ): - outcome = _execute(run, [GEMINI, SARVAM]) + with patch(f"{CRUD_BATCH}.GeminiClient") as gemini_client: + gemini_client.from_credentials.side_effect = ValueError("no gemini key") + outcome = _execute(run) assert outcome["success"] is True - assert outcome["batch_jobs"] == {} - assert outcome["sync_chunks"] == {SARVAM: ["task-1"]} - assert {r.status for r in _results(db, run, GEMINI)} == {JobStatus.FAILED.value} - assert {r.status for r in _results(db, run, SARVAM)} == { - JobStatus.PENDING.value + assert _row_states(db, run, GEMINI) == { + (JobStatus.FAILED.value, "Batch submission failed: no gemini key") } + if SARVAM in models: + assert _row_states(db, run, SARVAM) == {(JobStatus.PENDING.value, None)} + assert [c["model"] for c in boundaries.enqueued] == [SARVAM] + # Left for the cron to finalize. db.refresh(run) assert run.status == "processing" - def test_batch_failure_without_sync_models_fails_run( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - run = _run(db, user_api_key, [GEMINI]) - - with fake_gemini_batch( - "app.crud.tts_evaluations.batch", - {f"models/{GEMINI}": RuntimeError("quota")}, - ): - outcome = _execute(run, [GEMINI]) - - assert outcome == { - "success": False, - "error": "Batch submission failed for all models", - } - db.refresh(run) - assert run.status == "failed" - assert run.error_message == "Batch submission failed for all models" - - def test_enqueue_failure_fails_that_models_results( - self, db: Session, user_api_key: TestAuthContext, harness: Harness + def test_enqueue_failure_fails_only_that_models_rows( + self, db: Session, user_api_key: TestAuthContext, boundaries: SimpleNamespace ) -> None: - harness.enqueue_failures = {ELEVEN} + boundaries.enqueue_errors[SARVAM] = ConnectionError("broker unreachable") run = _run(db, user_api_key, [SARVAM, ELEVEN]) - outcome = _execute(run, [SARVAM, ELEVEN]) + outcome = _execute(run) - assert outcome["sync_chunks"] == {SARVAM: ["task-1"], ELEVEN: []} - eleven_rows = _results(db, run, ELEVEN) - assert {r.status for r in eleven_rows} == {JobStatus.FAILED.value} - assert eleven_rows[0].error_message.startswith( - f"[KAAPI] Failed to queue synthesis for {ELEVEN}: broker unreachable." - ) - assert {r.status for r in _results(db, run, SARVAM)} == { - JobStatus.PENDING.value + assert outcome["success"] is True + assert _row_states(db, run, SARVAM) == { + (JobStatus.FAILED.value, "Failed to queue synthesis: broker unreachable") } + assert _row_states(db, run, ELEVEN) == {(JobStatus.PENDING.value, None)} + assert [c["model"] for c in boundaries.enqueued] == [ELEVEN] db.refresh(run) assert run.status == "processing" - assert run.error_message == "2 synthesis(es) failed" - def test_pending_rows_of_unrequested_models_are_not_enqueued( - self, db: Session, user_api_key: TestAuthContext, harness: Harness + @pytest.mark.parametrize( + ("case", "expected_error"), + [ + ("missing_dataset", "Dataset not found"), + ("empty_csv", "No samples found for dataset"), + ("no_object_store_url", "No samples found for dataset"), + ], + ) + def test_unstartable_run_is_failed_without_rows( + self, + db: Session, + user_api_key: TestAuthContext, + boundaries: SimpleNamespace, + case: str, + expected_error: str, ) -> None: - run = _run(db, user_api_key, [SARVAM]) - stray = create_test_tts_result_row(db, run=run, provider="eleven_v4") - - outcome = _execute(run, [SARVAM]) - - assert list(outcome["sync_chunks"]) == [SARVAM] - assert all(stray.id not in c["result_ids"] for c in harness.enqueued) - - def test_run_not_found(self, db: Session, user_api_key: TestAuthContext) -> None: - run = _run(db, user_api_key, [SARVAM]) - missing = EvaluationRun( - id=get_non_existent_id(db, EvaluationRun), - project_id=run.project_id, - organization_id=run.organization_id, - dataset_id=run.dataset_id, + if case == "empty_csv": + boundaries.csv = _csv(0) + run = _run( + db, + user_api_key, + [SARVAM], + object_store_url=None if case == "no_object_store_url" else "s3://b/x.csv", ) - - assert _execute(missing, [SARVAM]) == { - "success": False, - "error": "Run not found", - } - - def test_run_deleted_mid_submission( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - run = _run(db, user_api_key, [SARVAM]) - run_id = run.id - - def delete_run() -> None: - db.delete(db.get(EvaluationRun, run_id)) - db.commit() - - harness.on_stream = delete_run - - assert _execute(run, [SARVAM]) == {"success": False, "error": "Run not found"} - assert harness.enqueued == [] - - def test_dataset_not_found_fails_run( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - run = _run(db, user_api_key, [SARVAM]) - - outcome = _execute(run, [SARVAM], dataset_id=run.dataset_id + 10_000_000) - - assert outcome == {"success": False, "error": "Dataset not found"} - db.refresh(run) - assert (run.status, run.error_message) == ("failed", "Dataset not found") - - def test_dataset_without_csv_fails_run( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - run = _run(db, user_api_key, [SARVAM], object_store_url=None) - - outcome = _execute(run, [SARVAM]) - - assert outcome == {"success": False, "error": "No samples found"} - db.refresh(run) - assert (run.status, run.error_message) == ( - "failed", - "No samples found for dataset", + dataset_id = ( + get_non_existent_id(db, EvaluationDataset) + if case == "missing_dataset" + else None ) - assert _results(db, run) == [] - def test_empty_csv_fails_run( - self, db: Session, user_api_key: TestAuthContext, harness: Harness - ) -> None: - harness.csv = b"text\n" - run = _run(db, user_api_key, [SARVAM]) + outcome = _execute(run, dataset_id=dataset_id) - assert _execute(run, [SARVAM]) == { - "success": False, - "error": "No samples found", - } + assert outcome["success"] is False + db.refresh(run) + assert (run.status, run.error_message) == ("failed", expected_error) + assert _results(db, run, SARVAM) == [] + assert boundaries.enqueued == [] @pytest.mark.parametrize( - "timeout", [Timeout(), SoftTimeLimitExceeded()], ids=["gevent", "soft"] + ("error", "expected_message"), + [ + (Timeout(), "Task exceeded soft time limit"), + (SoftTimeLimitExceeded(), "Task exceeded soft time limit"), + (RuntimeError("no bucket"), "no bucket"), + ], + ids=["gevent_timeout", "soft_timeout", "unexpected"], ) - def test_timeout_fails_run_and_reraises( + def test_worker_error_fails_run( self, db: Session, user_api_key: TestAuthContext, - harness: Harness, - timeout: BaseException, - ) -> None: - run = _run(db, user_api_key, [SARVAM]) - with patch(f"{MODULE}.load_sample_texts", side_effect=timeout): - with pytest.raises(type(timeout)): - _execute(run, [SARVAM]) - - db.refresh(run) - assert (run.status, run.error_message) == ( - "failed", - "Task exceeded soft time limit", - ) - - def test_unexpected_error_fails_run( - self, db: Session, user_api_key: TestAuthContext + error: BaseException, + expected_message: str, ) -> None: run = _run(db, user_api_key, [SARVAM]) - with patch( - f"{MODULE}.get_cloud_storage", side_effect=RuntimeError("no bucket") - ): - outcome = _execute(run, [SARVAM]) + with patch(f"{MODULE}.get_cloud_storage", side_effect=error): + if isinstance(error, RuntimeError): + assert _execute(run)["success"] is False + else: + # Timeouts re-raise so Celery records the task as timed out. + with pytest.raises(type(error)): + _execute(run) - assert outcome == {"success": False, "error": "no bucket"} db.refresh(run) - assert (run.status, run.error_message) == ("failed", "no bucket") + assert (run.status, run.error_message) == ("failed", expected_message) diff --git a/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py b/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py index fb0ecd350..2dbd087f3 100644 --- a/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py +++ b/backend/app/tests/services/tts_evaluations/test_batch_result_processing.py @@ -24,6 +24,7 @@ MODULE = "app.services.tts_evaluations.batch_result_processing" GEMINI = "gemini-2.5-pro-preview-tts" +SARVAM = "bulbul:v3" # 24 kHz / 16-bit / mono: 48000 bytes is one second; the WAV adds a 44-byte header. ONE_SECOND_PCM_B64 = base64.b64encode(b"\x00\x01" * 24000).decode() @@ -73,7 +74,7 @@ def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: db, organization_id=user_api_key.organization_id, project_id=user_api_key.project_id, - models=[GEMINI, "bulbul:v3"], + models=[GEMINI, SARVAM], ) @@ -98,7 +99,7 @@ def _refresh(db: Session, *rows: TTSResult | EvaluationRun) -> None: @pytest.mark.usefixtures("boundaries") class TestExecuteTTSResultProcessing: @pytest.mark.parametrize("key", ["inlineData", "inline_data"]) - def test_success_uploads_wav_and_completes_run( + def test_success_stores_wav_without_finalizing_run( self, db: Session, run: EvaluationRun, boundaries: Boundaries, key: str ) -> None: rows = [create_test_tts_result_row(db, run=run) for _ in range(2)] @@ -106,22 +107,15 @@ def test_success_uploads_wav_and_completes_run( outcome = _execute(run) - assert outcome == { - "success": True, - "run_id": run.id, - "processed": 2, - "failed": 0, - "run_status": "completed", - } + assert (outcome["processed"], outcome["failed"]) == (2, 0) _refresh(db, run, *rows) for row in rows: assert row.status == JobStatus.SUCCESS.value assert row.object_store_url.startswith("s3://bucket/evaluations/tts/audio/") assert row.metadata_ == {"duration_seconds": 1.0, "size_bytes": 48044} - assert run.status == "completed" - boundaries.download.assert_called_once_with("batches/abc") + assert run.status == "pending" - def test_non_pending_and_unknown_results_are_skipped( + def test_only_this_models_pending_rows_are_processed( self, db: Session, run: EvaluationRun, boundaries: Boundaries ) -> None: done = create_test_tts_result_row( @@ -130,7 +124,7 @@ def test_non_pending_and_unknown_results_are_skipped( status=JobStatus.SUCCESS, object_store_url="s3://bucket/keep.wav", ) - other_model = create_test_tts_result_row(db, run=run, provider="bulbul:v3") + other_model = create_test_tts_result_row(db, run=run, provider=SARVAM) pending = create_test_tts_result_row(db, run=run) boundaries.download.return_value = [ _ok(done.id), @@ -138,75 +132,50 @@ def test_non_pending_and_unknown_results_are_skipped( _ok(pending.id), ] - outcome = _execute(run) + _execute(run) - assert (outcome["processed"], outcome["failed"]) == (1, 0) - assert outcome["run_status"] == "processing" - assert boundaries.upload.call_count == 1 _refresh(db, done, other_model, pending) assert done.object_store_url == "s3://bucket/keep.wav" assert other_model.status == JobStatus.PENDING.value assert pending.status == JobStatus.SUCCESS.value - def test_invalid_custom_id_is_counted_failed( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - boundaries.download.return_value = [ - _ok("not-a-number"), - {"custom_id": None, "response": {}}, - ] - - outcome = _execute(run) - - assert (outcome["processed"], outcome["failed"]) == (0, 2) - - def test_failure_shapes_mark_results_failed( + def test_bad_results_fail_their_row_and_others_still_succeed( self, db: Session, run: EvaluationRun, boundaries: Boundaries ) -> None: - errored, no_error_text, silent, bad_b64, odd_shape = ( - create_test_tts_result_row(db, run=run) for _ in range(5) + errored, no_error_text, silent, bad_padding, not_base64, odd_shape, ok = ( + create_test_tts_result_row(db, run=run) for _ in range(7) ) boundaries.download.return_value = [ + {"custom_id": "not-a-number", "response": _audio_response()}, {"custom_id": str(errored.id), "response": None, "error": "SAFETY block"}, {"custom_id": str(no_error_text.id)}, { "custom_id": str(silent.id), "response": {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}, }, - _ok(bad_b64.id, data="abc"), # incorrect padding -> binascii.Error + _ok(bad_padding.id, data="abc"), + # Without strict decoding this became b"" and was stored as a 0s success. + _ok(not_base64.id, data="###"), {"custom_id": str(odd_shape.id), "response": ["not", "a", "dict"]}, + _ok(ok.id), ] outcome = _execute(run) - assert (outcome["processed"], outcome["failed"]) == (0, 5) - assert outcome["run_status"] == "completed" - _refresh(db, run, errored, no_error_text, silent, bad_b64, odd_shape) + assert (outcome["processed"], outcome["failed"]) == (1, 7) + _refresh( + db, errored, no_error_text, silent, bad_padding, not_base64, odd_shape, ok + ) + assert ok.status == JobStatus.SUCCESS.value assert errored.error_message == "SAFETY block" assert no_error_text.error_message == "Unknown error" assert silent.error_message == "No audio data in response" - assert bad_b64.error_message.startswith("Audio processing failed:") - assert odd_shape.error_message.startswith("Audio processing failed:") - assert { - r.status for r in (errored, no_error_text, silent, bad_b64, odd_shape) - } == {JobStatus.FAILED.value} - assert run.error_message == "5 synthesis(es) failed" - - def test_non_base64_audio_is_not_stored_as_empty_success( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - row = create_test_tts_result_row(db, run=run) - boundaries.download.return_value = [_ok(row.id, data="###")] - - outcome = _execute(run) + for row in (bad_padding, not_base64, odd_shape): + assert row.error_message.startswith("Audio processing failed:") + for row in (errored, no_error_text, silent, bad_padding, not_base64, odd_shape): + assert (row.status, row.object_store_url) == (JobStatus.FAILED.value, None) - assert (outcome["processed"], outcome["failed"]) == (0, 1) - boundaries.upload.assert_not_called() - _refresh(db, row) - assert row.status == JobStatus.FAILED.value - assert row.error_message.startswith("Audio processing failed:") - - def test_upload_failure_marks_result_failed( + def test_upload_failure_fails_row( self, db: Session, run: EvaluationRun, boundaries: Boundaries ) -> None: boundaries.upload.side_effect = None @@ -214,76 +183,55 @@ def test_upload_failure_marks_result_failed( row = create_test_tts_result_row(db, run=run) boundaries.download.return_value = [_ok(row.id)] - outcome = _execute(run) + _execute(run) - assert outcome["failed"] == 1 _refresh(db, row) assert (row.status, row.error_message) == ( JobStatus.FAILED.value, "Audio upload to object store failed", ) - def test_empty_batch_finalizes_run( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - create_test_tts_result_row(db, run=run, status=JobStatus.SUCCESS) - - outcome = _execute(run) - - assert (outcome["processed"], outcome["run_status"]) == (0, "completed") - - def test_timeout_keeps_already_written_chunks( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + @pytest.mark.parametrize( + "timeout", [Timeout(), SoftTimeLimitExceeded()], ids=["gevent", "soft"] + ) + def test_timeout_fails_only_this_models_pending_rows_and_reraises( + self, + db: Session, + run: EvaluationRun, + boundaries: Boundaries, + timeout: BaseException, ) -> None: - rows = [create_test_tts_result_row(db, run=run) for _ in range(60)] - boundaries.download.return_value = [_ok(r.id) for r in rows] - uploads = 0 - - def upload_then_time_out(**kwargs: Any) -> str: - nonlocal uploads - uploads += 1 - if uploads == 55: - raise Timeout() - return _fake_upload(**kwargs) - - boundaries.upload.side_effect = upload_then_time_out + boundaries.download.side_effect = timeout + gemini_row = create_test_tts_result_row(db, run=run) + sarvam_row = create_test_tts_result_row(db, run=run, provider=SARVAM) - with pytest.raises(Timeout): + with pytest.raises(type(timeout)): _execute(run) - _refresh(db, run, *rows) - statuses = [r.status for r in rows] - # writes land in slices of 50, so results 51-54 were still buffered - assert statuses.count(JobStatus.SUCCESS.value) == 50 - assert statuses.count(JobStatus.PENDING.value) == 10 - assert (run.status, run.error_message) == ( - "failed", - "Task exceeded soft time limit", + _refresh(db, run, gemini_row, sarvam_row) + assert (gemini_row.status, gemini_row.error_message) == ( + JobStatus.FAILED.value, + "Result processing exceeded soft time limit", ) + assert sarvam_row.status == JobStatus.PENDING.value + assert run.status == "pending" - def test_soft_time_limit_fails_run_and_reraises( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - boundaries.download.side_effect = SoftTimeLimitExceeded() - - with pytest.raises(SoftTimeLimitExceeded): - _execute(run) - - _refresh(db, run) - assert run.status == "failed" - - def test_unexpected_error_fails_run( + def test_unexpected_error_fails_only_this_models_pending_rows( self, db: Session, run: EvaluationRun, boundaries: Boundaries ) -> None: boundaries.gemini_client.from_credentials.side_effect = ValueError( "no gemini key" ) + gemini_row = create_test_tts_result_row(db, run=run) + sarvam_row = create_test_tts_result_row(db, run=run, provider=SARVAM) outcome = _execute(run) assert outcome == {"success": False, "error": "no gemini key"} - _refresh(db, run) - assert (run.status, run.error_message) == ( - "failed", + _refresh(db, run, gemini_row, sarvam_row) + assert (gemini_row.status, gemini_row.error_message) == ( + JobStatus.FAILED.value, "Result processing failed: no gemini key", ) + assert sarvam_row.status == JobStatus.PENDING.value + assert run.status == "pending" diff --git a/backend/app/tests/services/tts_evaluations/test_constants.py b/backend/app/tests/services/tts_evaluations/test_constants.py deleted file mode 100644 index f298174fd..000000000 --- a/backend/app/tests/services/tts_evaluations/test_constants.py +++ /dev/null @@ -1,62 +0,0 @@ -import pytest - -from app.core.providers import Provider -from app.services.tts_evaluations.constants import ( - SUPPORTED_TTS_MODELS, - TTSExecutionModeEnum, - get_tts_model_spec, - split_models_by_execution_mode, -) - - -class TestGetTTSModelSpec: - @pytest.mark.parametrize( - ("model", "provider", "mode"), - [ - ( - "gemini-2.5-pro-preview-tts", - Provider.GOOGLE_AISTUDIO, - TTSExecutionModeEnum.BATCH, - ), - ("bulbul:v3", Provider.SARVAMAI, TTSExecutionModeEnum.SYNC), - ("eleven_v3", Provider.ELEVENLABS, TTSExecutionModeEnum.SYNC), - ("eleven_v4", Provider.ELEVENLABS, TTSExecutionModeEnum.SYNC), - ], - ) - def test_registry_routes_models( - self, model: str, provider: Provider, mode: TTSExecutionModeEnum - ) -> None: - spec = get_tts_model_spec(model) - - assert spec.provider == provider - assert spec.execution_mode == mode - assert spec.default_voice - - def test_unknown_model_raises_key_error(self) -> None: - with pytest.raises(KeyError): - get_tts_model_spec("unknown-tts") - - def test_supported_models_match_registry(self) -> None: - assert set(SUPPORTED_TTS_MODELS) == { - "gemini-2.5-pro-preview-tts", - "bulbul:v3", - "eleven_v3", - "eleven_v4", - } - - -class TestSplitModelsByExecutionMode: - def test_mixed_models_split_preserving_order(self) -> None: - batch, sync = split_models_by_execution_mode( - ["eleven_v3", "gemini-2.5-pro-preview-tts", "bulbul:v3"] - ) - - assert batch == ["gemini-2.5-pro-preview-tts"] - assert sync == ["eleven_v3", "bulbul:v3"] - - def test_empty_list(self) -> None: - assert split_models_by_execution_mode([]) == ([], []) - - def test_unknown_model_raises(self) -> None: - with pytest.raises(KeyError): - split_models_by_execution_mode(["nope"]) diff --git a/backend/app/tests/services/tts_evaluations/test_dataset.py b/backend/app/tests/services/tts_evaluations/test_dataset.py index 44f5240c0..cbc1b5c43 100644 --- a/backend/app/tests/services/tts_evaluations/test_dataset.py +++ b/backend/app/tests/services/tts_evaluations/test_dataset.py @@ -1,113 +1,59 @@ import io -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +import pytest +from celery.exceptions import SoftTimeLimitExceeded from sqlmodel import Session -from app.services.tts_evaluations.dataset import ( - get_sample_texts_from_dataset, - load_sample_texts, -) +from app.services.tts_evaluations.dataset import get_sample_texts_from_dataset from app.tests.utils.auth import TestAuthContext from app.tests.utils.tts_evaluation import TEST_DATASET_URL, create_test_tts_dataset -CSV_BYTES = b"text\nHello world\n \n Good morning \n" - - -def _storage(content: bytes = CSV_BYTES) -> MagicMock: - def stream(url: str) -> io.BytesIO: - if url != TEST_DATASET_URL: - raise FileNotFoundError(url) - return io.BytesIO(content) +def _storage(stream: object) -> MagicMock: storage = MagicMock() storage.stream.side_effect = stream return storage -class TestLoadSampleTexts: - def test_reads_non_blank_stripped_texts(self) -> None: - storage = _storage() - - texts = load_sample_texts( - storage=storage, object_store_url=TEST_DATASET_URL, dataset_id=1 - ) - - assert texts == ["Hello world", "Good morning"] - storage.stream.assert_called_once_with(TEST_DATASET_URL) - - def test_stream_failure_returns_empty(self) -> None: - storage = MagicMock() - storage.stream.side_effect = OSError("s3 down") - - assert ( - load_sample_texts( - storage=storage, object_store_url=TEST_DATASET_URL, dataset_id=1 - ) - == [] - ) - - def test_undecodable_csv_returns_empty(self) -> None: - texts = load_sample_texts( - storage=_storage(b"\xff\xfe\xfa"), - object_store_url=TEST_DATASET_URL, - dataset_id=1, - ) - - assert texts == [] - - -class TestGetSampleTextsFromDataset: - def test_reads_dataset_csv( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - dataset = create_test_tts_dataset( - db, - organization_id=user_api_key.organization_id, - project_id=user_api_key.project_id, - ) - - with patch( - "app.services.tts_evaluations.dataset.get_cloud_storage", - return_value=_storage(), - ): - texts = get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) - - assert texts == ["Hello world", "Good morning"] - - def test_dataset_without_url_returns_empty( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - dataset = create_test_tts_dataset( - db, - organization_id=user_api_key.organization_id, - project_id=user_api_key.project_id, - object_store_url=None, - ) - - with patch( - "app.services.tts_evaluations.dataset.get_cloud_storage" - ) as get_storage: - assert ( - get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) - == [] - ) - - get_storage.assert_not_called() - - def test_storage_init_failure_returns_empty( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - dataset = create_test_tts_dataset( - db, - organization_id=user_api_key.organization_id, - project_id=user_api_key.project_id, - ) - - with patch( - "app.services.tts_evaluations.dataset.get_cloud_storage", - side_effect=ValueError("no storage credentials"), - ): - assert ( - get_sample_texts_from_dataset(db, dataset, user_api_key.project_id) - == [] - ) +@pytest.mark.parametrize( + ("object_store_url", "stream", "expected"), + [ + ( + TEST_DATASET_URL, + lambda url: io.BytesIO(b"text\nHello world\n \n Good morning \n"), + ["Hello world", "Good morning"], + ), + (None, lambda url: io.BytesIO(b"text\nunused\n"), []), + (TEST_DATASET_URL, OSError("s3 down"), []), + ], + ids=["reads_csv", "no_url", "stream_error"], +) +def test_get_sample_texts_from_dataset( + db: Session, + user_api_key: TestAuthContext, + object_store_url: str | None, + stream: object, + expected: list[str], +) -> None: + dataset = create_test_tts_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + object_store_url=object_store_url, + ) + + assert get_sample_texts_from_dataset(_storage(stream), dataset) == expected + + +def test_get_sample_texts_from_dataset_reraises_soft_timeout( + db: Session, user_api_key: TestAuthContext +) -> None: + dataset = create_test_tts_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + with pytest.raises(SoftTimeLimitExceeded): + get_sample_texts_from_dataset(_storage(SoftTimeLimitExceeded()), dataset) diff --git a/backend/app/tests/services/tts_evaluations/test_language.py b/backend/app/tests/services/tts_evaluations/test_language.py deleted file mode 100644 index 51c156123..000000000 --- a/backend/app/tests/services/tts_evaluations/test_language.py +++ /dev/null @@ -1,75 +0,0 @@ -import logging - -import pytest -from sqlmodel import Session - -from app.crud.language import get_language_by_locale -from app.models.language import Language -from app.services.tts_evaluations.language import ( - resolve_dataset_tts_language_code, - resolve_tts_language_code, - to_elevenlabs_language_code, -) -from app.tests.utils.utils import get_non_existent_id - - -class TestResolveTTSLanguageCode: - @pytest.mark.parametrize( - ("locale", "expected"), - [ - ("hi", "hi-IN"), - ("HI", "hi-IN"), - ("ta", "ta-IN"), - ("en", "en-IN"), - # Sarvam spells Odia "od-IN", not the ISO-derived "or-IN" - ("or", "od-IN"), - ], - ) - def test_maps_iso_locale_to_bcp47(self, locale: str, expected: str) -> None: - assert resolve_tts_language_code(locale) == expected - - @pytest.mark.parametrize("locale", [None, ""]) - def test_missing_locale_defaults_to_en_in(self, locale: str | None) -> None: - assert resolve_tts_language_code(locale) == "en-IN" - - def test_unmapped_locale_defaults_with_warning( - self, caplog: pytest.LogCaptureFixture - ) -> None: - with caplog.at_level(logging.WARNING): - assert resolve_tts_language_code("xx") == "en-IN" - - assert "Unmapped locale" in caplog.text - - -class TestResolveDatasetTTSLanguageCode: - def test_none_language_id_defaults_to_en_in_silently( - self, db: Session, caplog: pytest.LogCaptureFixture - ) -> None: - with caplog.at_level(logging.WARNING): - assert resolve_dataset_tts_language_code(db, None) == "en-IN" - - # An unset language is a normal dataset shape, not a misconfiguration. - assert "Language not found" not in caplog.text - - def test_seeded_language_resolves(self, db: Session) -> None: - odia = get_language_by_locale(session=db, locale="or") - - assert resolve_dataset_tts_language_code(db, odia.id) == "od-IN" - - def test_unknown_language_id_defaults_with_warning( - self, db: Session, caplog: pytest.LogCaptureFixture - ) -> None: - missing_id = get_non_existent_id(db, Language) - - with caplog.at_level(logging.WARNING): - assert resolve_dataset_tts_language_code(db, missing_id) == "en-IN" - - assert "Language not found" in caplog.text - - -class TestToElevenlabsLanguageCode: - def test_known_bcp47_maps_to_iso(self) -> None: - assert to_elevenlabs_language_code("hi-IN") == "hi" - - def test_odia_returns_none_for_auto_detect(self) -> None: - assert to_elevenlabs_language_code("od-IN") is None diff --git a/backend/app/tests/services/tts_evaluations/test_sync_generation.py b/backend/app/tests/services/tts_evaluations/test_sync_generation.py index 71f82a3c2..346107084 100644 --- a/backend/app/tests/services/tts_evaluations/test_sync_generation.py +++ b/backend/app/tests/services/tts_evaluations/test_sync_generation.py @@ -1,5 +1,6 @@ +import base64 from collections.abc import Iterator -from dataclasses import dataclass +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock, patch @@ -8,13 +9,14 @@ from gevent import Timeout from sqlmodel import Session +from app.core.audio_utils import pcm_to_wav from app.models import EvaluationRun from app.models.job import JobStatus from app.models.tts_evaluation import TTSResult -from app.services.tts_evaluations.sync_generation import execute_tts_sync_chunk -from app.services.tts_evaluations.synthesizers import ( - SynthesizedAudio, - TTSSynthesisError, +from app.services.tts_evaluations.sync_generation import ( + execute_tts_sync_generation, + synthesize_elevenlabs, + synthesize_sarvam, ) from app.tests.utils.auth import TestAuthContext from app.tests.utils.tts_evaluation import ( @@ -24,28 +26,71 @@ ) MODULE = "app.services.tts_evaluations.sync_generation" -SARVAM_MODEL = "bulbul:v3" -FAIL_TEXT = "please fail" -CRASH_TEXT = "please crash" +SARVAM = "bulbul:v3" +ELEVEN = "eleven_v3" +# 24 kHz / 16-bit / mono: 48000 bytes is one second; the WAV adds a 44-byte header. +ONE_SECOND_PCM = b"\x00\x01" * 24000 +FAIL_TEXT = "provider rejects this" -@dataclass -class Boundaries: - credentials: MagicMock - storage: MagicMock - upload: MagicMock - create_client: MagicMock - synthesize: MagicMock +def _sarvam_client(audios: list[str]) -> MagicMock: + client = MagicMock() + client.text_to_speech.convert.return_value = SimpleNamespace(audios=audios) + return client -def _fake_synthesize(*, text: str, **_kwargs: Any) -> SynthesizedAudio: - if text == FAIL_TEXT: - raise TTSSynthesisError( - "[SARVAM] TTS request failed (code: 400): bad", retryable=False +class TestSynthesizeSarvam: + def test_joins_chunks_without_wav_headers(self) -> None: + chunks = [base64.b64encode(pcm_to_wav(pcm)).decode() for pcm in (b"ab", b"cd")] + client = _sarvam_client(chunks) + + pcm = synthesize_sarvam(client, "namaste", SARVAM, "hi-IN") + + assert pcm == b"abcd" + assert ( + client.text_to_speech.convert.call_args.kwargs["target_language_code"] + == "hi-IN" + ) + + def test_no_audio_raises(self) -> None: + with pytest.raises(ValueError, match="Sarvam returned no audio"): + synthesize_sarvam(_sarvam_client([]), "namaste", SARVAM, "hi-IN") + + +class TestSynthesizeElevenLabs: + # od-IN has no ElevenLabs code; None lets ElevenLabs auto-detect. + @pytest.mark.parametrize( + ("language_code", "expected"), [("hi-IN", "hi"), ("od-IN", None)] + ) + def test_maps_language_and_joins_pcm( + self, language_code: str, expected: str | None + ) -> None: + client = MagicMock() + client.text_to_speech.convert.return_value = iter([b"ab", b"cd"]) + + pcm = synthesize_elevenlabs(client, "namaste", ELEVEN, language_code) + + assert pcm == b"abcd" + sent = client.text_to_speech.convert.call_args.kwargs + assert (sent["language_code"], sent["output_format"]) == ( + expected, + "pcm_24000", ) - if text == CRASH_TEXT: - raise KeyError("audios") - return SynthesizedAudio(wav_bytes=b"w" * 100, duration_seconds=1.23456) + + def test_no_audio_raises(self) -> None: + client = MagicMock() + client.text_to_speech.convert.return_value = iter([]) + + with pytest.raises(ValueError, match="ElevenLabs returned no audio"): + synthesize_elevenlabs(client, "namaste", ELEVEN, "hi-IN") + + +def _sarvam_convert(*, text: str, **_kwargs: Any) -> Any: + if text == FAIL_TEXT: + raise RuntimeError("bad request") + return SimpleNamespace( + audios=[base64.b64encode(pcm_to_wav(ONE_SECOND_PCM)).decode()] + ) def _fake_upload(*, subdirectory: str, filename: str, **_kwargs: Any) -> str: @@ -53,18 +98,24 @@ def _fake_upload(*, subdirectory: str, filename: str, **_kwargs: Any) -> str: @pytest.fixture -def boundaries(db: Session) -> Iterator[Boundaries]: +def boundaries(db: Session) -> Iterator[SimpleNamespace]: + sarvam = MagicMock() + sarvam.text_to_speech.convert.side_effect = _sarvam_convert + eleven = MagicMock() + eleven.text_to_speech.convert.side_effect = lambda **_kw: iter([ONE_SECOND_PCM]) with ( use_test_session(MODULE, db), + patch(f"{MODULE}.SarvamAIProvider.create_client", return_value=sarvam), + patch(f"{MODULE}.ElevenlabsAIProvider.create_client", return_value=eleven), patch( f"{MODULE}.get_provider_credential", return_value={"api_key": "k"} - ) as creds, + ) as credentials, patch(f"{MODULE}.get_cloud_storage") as storage, patch(f"{MODULE}.upload_to_object_store", side_effect=_fake_upload) as upload, - patch(f"{MODULE}.create_tts_client", return_value=MagicMock()) as create_client, - patch(f"{MODULE}.synthesize_tts", side_effect=_fake_synthesize) as synthesize, ): - yield Boundaries(creds, storage, upload, create_client, synthesize) + yield SimpleNamespace( + sarvam=sarvam, credentials=credentials, storage=storage, upload=upload + ) @pytest.fixture @@ -73,275 +124,181 @@ def run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: db, organization_id=user_api_key.organization_id, project_id=user_api_key.project_id, - models=[SARVAM_MODEL], + models=[SARVAM, ELEVEN], ) -def _pending(db: Session, run: EvaluationRun, text: str | None = None) -> TTSResult: +def _row( + db: Session, + run: EvaluationRun, + model: str = SARVAM, + status: JobStatus = JobStatus.PENDING, + text: str | None = None, +) -> TTSResult: return create_test_tts_result_row( - db, run=run, provider=SARVAM_MODEL, sample_text=text + db, run=run, provider=model, status=status, sample_text=text ) -def _execute(run: EvaluationRun, result_ids: list[int]) -> dict[str, Any]: - return execute_tts_sync_chunk( +def _execute(run: EvaluationRun, model: str = SARVAM) -> dict[str, Any]: + return execute_tts_sync_generation( project_id=run.project_id, job_id=str(run.id), task_id="celery-task", task_instance=MagicMock(), organization_id=run.organization_id, - model=SARVAM_MODEL, - result_ids=result_ids, + model=model, language_code="hi-IN", ) -def _reload(db: Session, *rows: TTSResult | EvaluationRun) -> None: +def _refresh(db: Session, *rows: TTSResult | EvaluationRun) -> None: for row in rows: db.refresh(row) @pytest.mark.usefixtures("boundaries") -class TestExecuteTTSSyncChunk: - def test_success_writes_audio_and_completes_run( - self, db: Session, run: EvaluationRun, boundaries: Boundaries +class TestExecuteTTSSyncGeneration: + @pytest.mark.parametrize("model", [SARVAM, ELEVEN]) + def test_success_stores_audio_without_finalizing_run( + self, db: Session, run: EvaluationRun, model: str ) -> None: - first = _pending(db, run) - second = _pending(db, run) - - outcome = _execute(run, [first.id, second.id]) - - assert outcome == { - "success": True, - "run_id": run.id, - "processed": 2, - "failed": 0, - "run_status": "completed", - } - _reload(db, first, second, run) - for row in (first, second): - assert row.status == JobStatus.SUCCESS.value - assert row.object_store_url.startswith("s3://bucket/evaluations/tts/audio/") - assert row.object_store_url.endswith(".wav") - assert row.metadata_ == {"duration_seconds": 1.235, "size_bytes": 100} - assert row.error_message is None - assert run.status == "completed" - assert run.error_message is None - assert boundaries.synthesize.call_args.kwargs["language_code"] == "hi-IN" - assert boundaries.synthesize.call_args.kwargs["voice"] == "shubh" - - def test_other_pending_rows_keep_run_processing( - self, db: Session, run: EvaluationRun - ) -> None: - mine = _pending(db, run) - _pending(db, run) # belongs to a different chunk - - outcome = _execute(run, [mine.id]) - - assert outcome["run_status"] == "processing" - _reload(db, run) - assert run.status == "processing" - - def test_redelivered_chunk_skips_finished_rows( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - done = create_test_tts_result_row( - db, - run=run, - provider=SARVAM_MODEL, - status=JobStatus.SUCCESS, - object_store_url="s3://bucket/original.wav", - ) - pending = _pending(db, run) + rows = [_row(db, run, model), _row(db, run, model)] - outcome = _execute(run, [done.id, pending.id]) + outcome = _execute(run, model) - assert outcome["processed"] == 1 - assert boundaries.synthesize.call_count == 1 - _reload(db, done) - assert done.object_store_url == "s3://bucket/original.wav" - - def test_row_finished_concurrently_is_not_counted_or_overwritten( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - row = _pending(db, run) - - def concurrent_writer_wins(**kwargs: Any) -> SynthesizedAudio: - # A redelivered copy of this chunk lands its write first. The main - # thread is parked in as_completed, so the session isn't shared concurrently. - db.refresh(row) - row.status = JobStatus.SUCCESS.value - row.object_store_url = "s3://bucket/other-worker.wav" - db.add(row) - db.commit() - return _fake_synthesize(**kwargs) - - boundaries.synthesize.side_effect = concurrent_writer_wins - - outcome = _execute(run, [row.id]) - - assert (outcome["processed"], outcome["failed"]) == (0, 0) - _reload(db, row) - assert row.object_store_url == "s3://bucket/other-worker.wav" - - def test_no_pending_rows_is_a_noop_that_still_finalizes( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - done = create_test_tts_result_row( - db, run=run, provider=SARVAM_MODEL, status=JobStatus.SUCCESS + assert (outcome["success"], outcome["processed"], outcome["failed"]) == ( + True, + 2, + 0, ) - - outcome = _execute(run, [done.id]) - - assert outcome == { - "success": True, - "run_id": run.id, - "processed": 0, - "failed": 0, - "run_status": "completed", - } - boundaries.synthesize.assert_not_called() - boundaries.credentials.assert_not_called() - - def test_missing_credentials_fail_the_chunk( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - boundaries.credentials.return_value = None - rows = [_pending(db, run), _pending(db, run)] - - outcome = _execute(run, [r.id for r in rows]) - - assert outcome["success"] is False - assert outcome["failed"] == 2 - assert outcome["run_status"] == "completed" - assert "sarvamai credentials are not configured" in outcome["error"] - _reload(db, run, *rows) + _refresh(db, run, *rows) for row in rows: - assert row.status == JobStatus.FAILED.value - assert row.error_message == outcome["error"] - assert run.error_message == "2 synthesis(es) failed" - boundaries.synthesize.assert_not_called() + assert row.status == JobStatus.SUCCESS.value + assert row.object_store_url.startswith("s3://bucket/evaluations/tts/audio/") + assert row.metadata_ == {"duration_seconds": 1.0, "size_bytes": 48044} + assert run.status == "pending" - def test_invalid_credentials_fail_the_chunk( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + def test_only_this_models_pending_rows_are_synthesized( + self, db: Session, run: EvaluationRun, boundaries: SimpleNamespace ) -> None: - boundaries.create_client.side_effect = ValueError( - "API Key for SarvamAI Not Set" - ) - row = _pending(db, run) - - outcome = _execute(run, [row.id]) - - assert outcome["success"] is False - assert outcome["error"].startswith( - "[KAAPI] Invalid sarvamai credentials: API Key for SarvamAI Not Set." - ) - _reload(db, row) - assert row.status == JobStatus.FAILED.value + done = _row(db, run, status=JobStatus.SUCCESS) + done.object_store_url = "s3://bucket/original.wav" + db.add(done) + db.commit() + pending = _row(db, run) + other_model = _row(db, run, model=ELEVEN) + + _execute(run) + + # A redelivered task must not re-bill the provider for finished rows. + assert boundaries.sarvam.text_to_speech.convert.call_count == 1 + _refresh(db, done, pending, other_model) + assert done.object_store_url == "s3://bucket/original.wav" + assert pending.status == JobStatus.SUCCESS.value + assert other_model.status == JobStatus.PENDING.value - def test_synthesis_errors_fail_individual_results( + def test_synthesis_failure_only_fails_that_row( self, db: Session, run: EvaluationRun ) -> None: - ok = _pending(db, run) - rejected = _pending(db, run, text=FAIL_TEXT) - crashed = _pending(db, run, text=CRASH_TEXT) + ok = _row(db, run) + rejected = _row(db, run, text=FAIL_TEXT) - outcome = _execute(run, [ok.id, rejected.id, crashed.id]) + outcome = _execute(run) - assert (outcome["success"], outcome["processed"], outcome["failed"]) == ( - True, - 1, - 2, - ) - _reload(db, ok, rejected, crashed, run) + assert (outcome["processed"], outcome["failed"]) == (1, 1) + _refresh(db, ok, rejected) assert ok.status == JobStatus.SUCCESS.value - assert rejected.status == JobStatus.FAILED.value - assert rejected.error_message == "[SARVAM] TTS request failed (code: 400): bad" - assert crashed.status == JobStatus.FAILED.value - assert crashed.error_message.startswith( - "[KAAPI] Unexpected error during sarvamai synthesis: 'audios'." + assert (rejected.status, rejected.error_message) == ( + JobStatus.FAILED.value, + "sarvamai synthesis failed: bad request", ) - assert run.status == "completed" - assert run.error_message == "2 synthesis(es) failed" - def test_upload_failure_fails_result( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + def test_upload_failure_fails_row( + self, db: Session, run: EvaluationRun, boundaries: SimpleNamespace ) -> None: boundaries.upload.side_effect = None boundaries.upload.return_value = None - row = _pending(db, run) + row = _row(db, run) - outcome = _execute(run, [row.id]) + _execute(run) - assert (outcome["processed"], outcome["failed"]) == (0, 1) - _reload(db, row) - assert row.status == JobStatus.FAILED.value - assert row.object_store_url is None - assert row.error_message.startswith( - "[KAAPI] Audio upload to object store failed." + _refresh(db, row) + assert (row.status, row.object_store_url, row.error_message) == ( + JobStatus.FAILED.value, + None, + "Audio upload to object store failed", ) - def test_gevent_timeout_fails_leftover_rows_and_reraises( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + def test_unsupported_model_fails_its_rows( + self, db: Session, run: EvaluationRun ) -> None: - boundaries.synthesize.side_effect = Timeout() - done = create_test_tts_result_row( - db, run=run, provider=SARVAM_MODEL, status=JobStatus.SUCCESS - ) - row = _pending(db, run) - - with pytest.raises(Timeout): - _execute(run, [done.id, row.id]) + row = _row(db, run, model="mystery-tts") + sarvam_row = _row(db, run) - _reload(db, done, row, run) - assert row.status == JobStatus.FAILED.value - assert row.error_message.startswith("[KAAPI] Synthesis timed out") - assert done.status == JobStatus.SUCCESS.value - assert run.status == "completed" + outcome = _execute(run, model="mystery-tts") - def test_soft_time_limit_fails_leftover_rows_and_reraises( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + assert outcome["success"] is False + _refresh(db, row, sarvam_row) + assert (row.status, row.error_message) == ( + JobStatus.FAILED.value, + "Unsupported sync TTS model: mystery-tts", + ) + assert sarvam_row.status == JobStatus.PENDING.value + + @pytest.mark.parametrize( + ("break_boundary", "error"), + [ + ( + lambda b: setattr(b.credentials, "return_value", None), + "sarvamai credentials are not configured", + ), + ( + lambda b: setattr(b.storage, "side_effect", RuntimeError("no bucket")), + "no bucket", + ), + ], + ids=["missing_credentials", "crash"], + ) + def test_setup_failure_fails_every_pending_row( + self, + db: Session, + run: EvaluationRun, + boundaries: SimpleNamespace, + break_boundary: Any, + error: str, ) -> None: - boundaries.upload.side_effect = SoftTimeLimitExceeded() - row = _pending(db, run) - - with pytest.raises(SoftTimeLimitExceeded): - _execute(run, [row.id]) + break_boundary(boundaries) + rows = [_row(db, run), _row(db, run)] - _reload(db, row, run) - assert row.status == JobStatus.FAILED.value - assert row.error_message.startswith("[KAAPI] Synthesis timed out") - assert run.status == "completed" + outcome = _execute(run) - def test_unexpected_exception_fails_leftover_rows( - self, db: Session, run: EvaluationRun, boundaries: Boundaries - ) -> None: - boundaries.upload.side_effect = RuntimeError("s3 exploded") - row = _pending(db, run) - - outcome = _execute(run, [row.id]) - - assert outcome == { - "success": False, - "run_id": run.id, - "error": "s3 exploded", - "run_status": "completed", - } - _reload(db, row) - assert row.status == JobStatus.FAILED.value - assert row.error_message.startswith( - "[KAAPI] Synthesis chunk failed unexpectedly: s3 exploded." - ) + assert (outcome["success"], outcome["error"]) == (False, error) + _refresh(db, *rows) + for row in rows: + assert (row.status, row.error_message) == ( + JobStatus.FAILED.value, + f"sarvamai synthesis failed: {error}", + ) - def test_client_factory_returning_nothing_fails_chunk( - self, db: Session, run: EvaluationRun, boundaries: Boundaries + @pytest.mark.parametrize( + "timeout", [Timeout(), SoftTimeLimitExceeded()], ids=["gevent", "soft"] + ) + def test_timeout_fails_leftover_rows_and_reraises( + self, + db: Session, + run: EvaluationRun, + boundaries: SimpleNamespace, + timeout: BaseException, ) -> None: - boundaries.create_client.return_value = None - row = _pending(db, run) + boundaries.storage.side_effect = timeout + row = _row(db, run) - outcome = _execute(run, [row.id]) + with pytest.raises(type(timeout)): + _execute(run) - assert outcome["success"] is False - assert "missing its client or storage" in outcome["error"] - _reload(db, row) - assert row.status == JobStatus.FAILED.value + _refresh(db, row) + assert (row.status, row.error_message) == ( + JobStatus.FAILED.value, + "Synthesis timed out before this sample was processed", + ) diff --git a/backend/app/tests/services/tts_evaluations/test_synthesizers.py b/backend/app/tests/services/tts_evaluations/test_synthesizers.py deleted file mode 100644 index 2b0890cdc..000000000 --- a/backend/app/tests/services/tts_evaluations/test_synthesizers.py +++ /dev/null @@ -1,368 +0,0 @@ -import base64 -import io -import logging -import wave -from collections.abc import Iterator -from types import SimpleNamespace -from unittest.mock import MagicMock - -import httpx -import pytest -from elevenlabs import ElevenLabs -from elevenlabs.core.api_error import ApiError as ElevenLabsApiError -from sarvamai import SarvamAI -from sarvamai.core.api_error import ApiError as SarvamApiError -from tenacity import wait_none - -from app.core.providers import Provider -from app.services.tts_evaluations.synthesizers import ( - SynthesizedAudio, - TTSSynthesisError, - create_tts_client, - synthesize_elevenlabs, - synthesize_sarvam, - synthesize_tts, -) - - -def _wav(pcm: bytes, *, rate: int = 24000, width: int = 2, channels: int = 1) -> bytes: - buf = io.BytesIO() - with wave.open(buf, "wb") as w: - w.setnchannels(channels) - w.setsampwidth(width) - w.setframerate(rate) - w.writeframes(pcm) - return buf.getvalue() - - -def _read_wav(wav_bytes: bytes) -> tuple[int, int, int, bytes]: - with wave.open(io.BytesIO(wav_bytes), "rb") as w: - return ( - w.getnchannels(), - w.getsampwidth(), - w.getframerate(), - w.readframes(w.getnframes()), - ) - - -def _b64(data: bytes) -> str: - return base64.b64encode(data).decode() - - -def _sarvam_client( - *, audios: list[str] | None = None, error: Exception | None = None -) -> MagicMock: - client = MagicMock() - if error is not None: - client.text_to_speech.convert.side_effect = error - else: - client.text_to_speech.convert.return_value = SimpleNamespace( - audios=audios, request_id="req-1" - ) - return client - - -def _elevenlabs_client(chunks: Iterator[bytes] | list[bytes]) -> MagicMock: - client = MagicMock() - client.text_to_speech.convert.return_value = chunks - return client - - -def _call_sarvam(client: MagicMock) -> SynthesizedAudio: - return synthesize_sarvam( - client=client, - text="namaste", - model="bulbul:v3", - language_code="hi-IN", - speaker="shubh", - ) - - -def _call_elevenlabs( - client: MagicMock, language_code: str = "hi-IN" -) -> SynthesizedAudio: - return synthesize_elevenlabs( - client=client, - text="namaste", - model="eleven_v3", - language_code=language_code, - voice_id="voice-1", - ) - - -# 24 kHz / 16-bit / mono: 48000 bytes is exactly one second. -ONE_SECOND_PCM = b"\x01\x00" * 24000 - - -class TestSynthesizeSarvam: - def test_multi_chunk_wavs_are_joined_under_one_header(self) -> None: - first = b"\x01\x00" * 100 - second = b"\x02\x00" * 50 - client = _sarvam_client(audios=[_b64(_wav(first)), _b64(_wav(second))]) - - audio = _call_sarvam(client) - - channels, width, rate, frames = _read_wav(audio.wav_bytes) - assert (channels, width, rate) == (1, 2, 24000) - assert frames == first + second - # a second RIFF header left inside the PCM would play as a click - assert audio.wav_bytes.count(b"RIFF") == 1 - assert audio.duration_seconds == pytest.approx(150 / 24000) - - def test_chunk_at_other_sample_rate_is_resampled_to_24k(self) -> None: - ten_ms_at_16k = b"\x00\x10" * 160 - client = _sarvam_client(audios=[_b64(_wav(ten_ms_at_16k, rate=16000))]) - - audio = _call_sarvam(client) - - _, _, rate, frames = _read_wav(audio.wav_bytes) - assert rate == 24000 - # 10 ms at 24 kHz is 240 frames; ratecv may drop one to rounding. - assert len(frames) // 2 in (239, 240) - assert audio.duration_seconds == pytest.approx(0.01, abs=2 / 24000) - - def test_one_second_duration(self) -> None: - client = _sarvam_client(audios=[_b64(_wav(ONE_SECOND_PCM))]) - - assert _call_sarvam(client).duration_seconds == pytest.approx(1.0) - - @pytest.mark.parametrize("audios", [[], None]) - def test_empty_audio_list_is_non_retryable(self, audios: list[str] | None) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_sarvam(_sarvam_client(audios=audios)) - - assert exc.value.retryable is False - assert exc.value.message.startswith("[SARVAM] TTS response contains no audio") - - @pytest.mark.parametrize( - "bad_chunk", - [_b64(b"definitely not a wav file"), "abc"], - ids=["not-wav", "bad-base64-padding"], - ) - def test_undecodable_audio_is_kaapi_error(self, bad_chunk: str) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_sarvam(_sarvam_client(audios=[bad_chunk])) - - assert exc.value.retryable is False - assert exc.value.message.startswith("[KAAPI] Failed to decode Sarvam audio") - - @pytest.mark.parametrize( - ("status_code", "retryable", "hint"), - [ - (400, False, "Review the input text"), - (401, False, "Verify the API key"), - (429, True, "Hit the provider's rate/quota"), - (500, True, "Typically transient"), - (503, True, "temporarily down"), - (None, False, "If this persists, contact Kaapi."), - ], - ) - def test_api_error_maps_status_to_retryability( - self, status_code: int | None, retryable: bool, hint: str - ) -> None: - error = SarvamApiError(status_code=status_code, body={"detail": "nope"}) - - with pytest.raises(TTSSynthesisError) as exc: - _call_sarvam(_sarvam_client(error=error)) - - assert exc.value.retryable is retryable - assert exc.value.message.startswith( - f"[SARVAM] TTS request failed (code: {status_code})" - ) - assert hint in exc.value.message - - def test_server_errors_log_at_error_and_client_errors_at_warning( - self, caplog: pytest.LogCaptureFixture - ) -> None: - with caplog.at_level(logging.WARNING): - for status_code in (500, 404): - with pytest.raises(TTSSynthesisError): - _call_sarvam( - _sarvam_client(error=SarvamApiError(status_code=status_code)) - ) - - levels = { - r.message.split("(code: ")[1][:3]: r.levelno - for r in caplog.records - if "code: " in r.message - } - assert levels == {"500": logging.ERROR, "404": logging.WARNING} - - def test_network_error_is_retryable(self) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_sarvam(_sarvam_client(error=httpx.ConnectError("dns failure"))) - - assert exc.value.retryable is True - assert ( - "[KAAPI] Could not reach Sarvam (code: ConnectError)" in exc.value.message - ) - - def test_timeout_is_retryable_with_timeout_message(self) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_sarvam(_sarvam_client(error=httpx.ReadTimeout("slow"))) - - assert exc.value.retryable is True - assert ( - "[KAAPI] TTS request to Sarvam timed out (code: ReadTimeout)" - in exc.value.message - ) - - -class TestSynthesizeElevenlabs: - def test_streamed_pcm_is_wrapped_as_wav(self) -> None: - client = _elevenlabs_client([ONE_SECOND_PCM[:1000], ONE_SECOND_PCM[1000:]]) - - audio = _call_elevenlabs(client) - - channels, width, rate, frames = _read_wav(audio.wav_bytes) - assert (channels, width, rate) == (1, 2, 24000) - assert frames == ONE_SECOND_PCM - assert audio.duration_seconds == pytest.approx(1.0) - - def test_language_hint_sent_for_supported_language(self) -> None: - client = _elevenlabs_client([ONE_SECOND_PCM]) - - _call_elevenlabs(client, language_code="hi-IN") - - kwargs = client.text_to_speech.convert.call_args.kwargs - assert kwargs["language_code"] == "hi" - assert kwargs["output_format"] == "pcm_24000" - - def test_language_hint_omitted_for_odia_auto_detect(self) -> None: - client = _elevenlabs_client([ONE_SECOND_PCM]) - - _call_elevenlabs(client, language_code="od-IN") - - assert "language_code" not in client.text_to_speech.convert.call_args.kwargs - - def test_empty_stream_is_non_retryable(self) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_elevenlabs(_elevenlabs_client([])) - - assert exc.value.retryable is False - assert exc.value.message.startswith( - "[ELEVENLABS] TTS response contains no audio" - ) - - @pytest.mark.parametrize( - ("status_code", "retryable"), [(422, False), (429, True), (502, True)] - ) - def test_api_error_raised_while_streaming( - self, status_code: int, retryable: bool - ) -> None: - def lazy_stream() -> Iterator[bytes]: - yield b"\x00\x00" - raise ElevenLabsApiError(status_code=status_code, body="bad") - - with pytest.raises(TTSSynthesisError) as exc: - _call_elevenlabs(_elevenlabs_client(lazy_stream())) - - assert exc.value.retryable is retryable - assert exc.value.message.startswith( - f"[ELEVENLABS] TTS request failed (code: {status_code})" - ) - - def test_network_error_is_retryable(self) -> None: - client = MagicMock() - client.text_to_speech.convert.side_effect = httpx.ConnectTimeout("t") - - with pytest.raises(TTSSynthesisError) as exc: - _call_elevenlabs(client) - - assert exc.value.retryable is True - assert "TTS request to Elevenlabs timed out" in exc.value.message - - -@pytest.fixture -def no_retry_wait(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(synthesize_tts.retry, "wait", wait_none()) - - -def _call_tts(client: MagicMock, provider: Provider) -> SynthesizedAudio: - return synthesize_tts( - client=client, - provider=provider, - text="hello", - model="bulbul:v3", - language_code="en-IN", - voice="shubh", - ) - - -@pytest.mark.usefixtures("no_retry_wait") -class TestSynthesizeTTS: - def test_dispatches_to_sarvam(self) -> None: - client = _sarvam_client(audios=[_b64(_wav(ONE_SECOND_PCM))]) - - audio = _call_tts(client, Provider.SARVAMAI) - - assert audio.duration_seconds == pytest.approx(1.0) - assert client.text_to_speech.convert.call_args.kwargs["speaker"] == "shubh" - - def test_dispatches_to_elevenlabs(self) -> None: - client = _elevenlabs_client([ONE_SECOND_PCM]) - - audio = _call_tts(client, Provider.ELEVENLABS) - - assert audio.duration_seconds == pytest.approx(1.0) - assert client.text_to_speech.convert.call_args.kwargs["voice_id"] == "shubh" - - def test_transient_failures_are_retried_until_success(self) -> None: - client = MagicMock() - client.text_to_speech.convert.side_effect = [ - SarvamApiError(status_code=503), - httpx.ConnectError("blip"), - SimpleNamespace(audios=[_b64(_wav(ONE_SECOND_PCM))], request_id="r"), - ] - - audio = _call_tts(client, Provider.SARVAMAI) - - assert audio.duration_seconds == pytest.approx(1.0) - assert client.text_to_speech.convert.call_count == 3 - - def test_gives_up_after_three_attempts(self) -> None: - client = _sarvam_client(error=SarvamApiError(status_code=429)) - - with pytest.raises(TTSSynthesisError) as exc: - _call_tts(client, Provider.SARVAMAI) - - assert exc.value.retryable is True - assert client.text_to_speech.convert.call_count == 3 - - def test_non_retryable_failure_is_not_retried(self) -> None: - client = _sarvam_client(error=SarvamApiError(status_code=400)) - - with pytest.raises(TTSSynthesisError): - _call_tts(client, Provider.SARVAMAI) - - assert client.text_to_speech.convert.call_count == 1 - - def test_unknown_provider_is_rejected(self) -> None: - with pytest.raises(TTSSynthesisError) as exc: - _call_tts(MagicMock(), Provider.GOOGLE_AISTUDIO) - - assert exc.value.retryable is False - assert ( - "No synchronous synthesizer for provider 'google-aistudio'" - in exc.value.message - ) - - -class TestCreateTTSClient: - def test_sarvam_client(self) -> None: - assert isinstance( - create_tts_client(Provider.SARVAMAI, {"api_key": "k"}), SarvamAI - ) - - def test_elevenlabs_client(self) -> None: - assert isinstance( - create_tts_client(Provider.ELEVENLABS, {"api_key": "k"}), ElevenLabs - ) - - def test_missing_api_key_raises(self) -> None: - with pytest.raises(ValueError, match="API Key"): - create_tts_client(Provider.SARVAMAI, {}) - - def test_batch_only_provider_raises(self) -> None: - with pytest.raises(ValueError, match="has no synchronous TTS client"): - create_tts_client(Provider.GOOGLE_AISTUDIO, {"api_key": "k"}) diff --git a/backend/app/tests/utils/tts_evaluation.py b/backend/app/tests/utils/tts_evaluation.py index dadeb4c3f..81d79626f 100644 --- a/backend/app/tests/utils/tts_evaluation.py +++ b/backend/app/tests/utils/tts_evaluation.py @@ -10,10 +10,11 @@ from app.core.util import now from app.crud.tts_evaluations import create_tts_run from app.models import EvaluationDataset, EvaluationRun +from app.models.batch_job import BatchJob, BatchJobType from app.models.job import JobStatus from app.models.stt_evaluation import EvaluationType from app.models.tts_evaluation import TTSResult -from app.services.tts_evaluations.constants import GEMINI_PRO_TTS_MODEL +from app.services.tts_evaluations.constants import DEFAULT_TTS_MODEL from app.tests.utils.utils import random_lower_string TEST_DATASET_URL = "s3://test-bucket/tts_datasets/test.csv" @@ -45,14 +46,9 @@ def use_test_session(module_path: str, db: Session) -> Iterator[Session]: class FakeGeminiBatchProvider: - """Stands in for the Gemini Batch API (the network boundary of a TTS batch). + """Stands in for the Gemini Batch API; records each submitted JSONL per model path.""" - `failures` maps a model path ("models/") to the exception its - create_batch raises; every call's JSONL is recorded per model path. - """ - - def __init__(self, failures: dict[str, Exception] | None = None) -> None: - self.failures = failures or {} + def __init__(self) -> None: self.submitted: dict[str, list[dict[str, Any]]] = {} def __call__( @@ -64,8 +60,6 @@ def __call__( def create_batch( self, jsonl_data: list[dict[str, Any]], config: dict[str, Any] ) -> dict[str, Any]: - if self._model in self.failures: - raise self.failures[self._model] self.submitted[self._model] = jsonl_data return { "provider_batch_id": f"batches/{self._model.split('/')[-1]}", @@ -76,10 +70,8 @@ def create_batch( @contextmanager -def fake_gemini_batch( - module_path: str, failures: dict[str, Exception] | None = None -) -> Iterator[FakeGeminiBatchProvider]: - fake = FakeGeminiBatchProvider(failures) +def fake_gemini_batch(module_path: str) -> Iterator[FakeGeminiBatchProvider]: + fake = FakeGeminiBatchProvider() with ( patch(f"{module_path}.GeminiClient") as gemini_client, patch(f"{module_path}.GeminiBatchProvider", side_effect=fake), @@ -137,7 +129,7 @@ def create_test_tts_run_with_dataset( dataset_name=dataset.name, org_id=organization_id, project_id=project_id, - models=models or [GEMINI_PRO_TTS_MODEL], + models=models or [DEFAULT_TTS_MODEL], ) @@ -145,7 +137,7 @@ def create_test_tts_result_row( db: Session, *, run: EvaluationRun, - provider: str = GEMINI_PRO_TTS_MODEL, + provider: str = DEFAULT_TTS_MODEL, status: JobStatus = JobStatus.PENDING, sample_text: str | None = None, object_store_url: str | None = None, @@ -165,3 +157,29 @@ def create_test_tts_result_row( db.commit() db.refresh(result) return result + + +def create_test_tts_batch_job( + db: Session, + *, + run: EvaluationRun, + model: str = DEFAULT_TTS_MODEL, + provider_status: str, + provider_batch_id: str | None = "batches/test", + error_message: str | None = None, +) -> BatchJob: + batch_job = BatchJob( + provider="google-aistudio", + job_type=BatchJobType.TTS_EVALUATION.value, + config={"evaluation_run_id": run.id, "tts_provider": model}, + provider_batch_id=provider_batch_id, + provider_status=provider_status, + error_message=error_message, + total_items=1, + organization_id=run.organization_id, + project_id=run.project_id, + ) + db.add(batch_job) + db.commit() + db.refresh(batch_job) + return batch_job diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index a95f0384d..d0c4c4feb 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -32,14 +32,15 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes + graph wiring only), `iteration_state.py` (`EvaluationIterationState`, `advance_round_state` — the pure per-round bookkeeping/stop arithmetic, `build_iteration_report`), `iteration_checkpointer.py` (`get_evaluation_iteration_checkpointer`, psycopg pool) - `services/stt_evaluations/`, `services/tts_evaluations/` -- TTS (`services/tts_evaluations/`): `constants.py` holds `TTS_MODEL_REGISTRY` (model → `TTSModelSpec`: provider, `TTSExecutionModeEnum` BATCH/SYNC, default voice); `SUPPORTED_TTS_MODELS` (the request validator's allow-list) is derived from it. Models: `gemini-2.5-pro-preview-tts` (BATCH), `bulbul:v3` (Sarvam, SYNC), `eleven_v3` / `eleven_v4` (ElevenLabs, SYNC). `language.py` resolves the dataset's `language_id` → ISO locale → BCP-47 (`hi` → `hi-IN`, `or` → `od-IN`), defaulting to `en-IN` when unset/unmapped; ElevenLabs gets the ISO code via `BCP47_TO_ELEVENLABS_LANG` (omitted → auto-detect). `synthesizers.py` (`synthesize_sarvam`, `synthesize_elevenlabs`, retrying `synthesize_tts`) normalizes every clip to 24 kHz/16-bit/mono WAV; Sarvam's multi-chunk responses are decoded and header-stripped per chunk before re-wrapping once. Failures raise `TTSSynthesisError` (source-tagged message + `retryable` for 429/5xx/network; SDK retries disabled so tenacity owns the policy). `crud/tts_evaluations/run.py::finalize_tts_run_status` is the single run-finalizer (locks the run row `FOR UPDATE`, counts results: `completed` when no PENDING else `processing`, `"N synthesis(es) failed"`), shared by sync chunks, Gemini result processing, and the TTS cron. +- TTS (`services/tts_evaluations/`): `constants.py` is flat module-level constants: `GEMINI_TTS_MODELS` (`gemini-2.5-pro-preview-tts`, `gemini-3.1-flash-tts-preview`; Gemini Batch), `SARVAM_TTS_MODELS` (`bulbul:v3`) and `ELEVENLABS_TTS_MODELS` (`eleven_v4`, `eleven_v3`) — the latter two synthesized directly; `SUPPORTED_TTS_MODELS` (the request validator's allow-list) is their concatenation. Language: submission maps the dataset's `language_id` → `Language.locale` → BCP-47 via `LOCALE_TO_BCP47` (`hi` → `hi-IN`, `or` → `od-IN`), defaulting to `DEFAULT_TTS_LANGUAGE` (`en-IN`); Sarvam gets the BCP-47 tag, ElevenLabs the ISO code via `BCP47_TO_ELEVENLABS_LANG` (unmapped → auto-detect). `sync_generation.py` holds `synthesize_sarvam` / `synthesize_elevenlabs` (raw 24 kHz/16-bit/mono PCM; Sarvam's multi-chunk WAVs are header-stripped per chunk) — retries are the SDKs' built-in ones; a failed sample is marked FAILED with `" synthesis failed: "`. `crud/tts_evaluations/run.py::finalize_tts_run_status` is the single run-status rule (locks the run row `FOR UPDATE`, counts results: `processing` while any row is PENDING, `failed` when no row succeeded, else `completed` with `"N synthesis(es) failed"` if some failed) and is called only by the TTS cron; workers only write `tts_result` rows, so one model failing never fails the run and the final status lags by up to one `CRON_INTERVAL_MINUTES`. - `crud/evaluations/` — `core.py`, `batch.py`, `fast.py` (orchestration + S3/OpenAI IO only; the pieces below carry the rest), `fast_results.py` (pure per-item S3 unit shapes, usage sums, failure threshold), `fast_chunks.py` (`batch_job` bookkeeping + chunk queries/cleanup), `fast_cosine.py` (pure v1 cosine scoring), `fast_traces.py` (pure `TraceData` building), `judge_stage.py` (run-level v2 judge: `resolve_config_prompt`, `judge_rows` pool, per-metric summary rollup), `judge_prompts.py` (rubric text, moved out of `score.py`), `retry.py` (shared OpenAI retry policy), `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. -- TTS submission (`run_tts_batch_submission` → `services/tts_evaluations/batch_job.py::execute_batch_submission`) creates one `tts_result` per text×model, then splits models by execution mode: BATCH models go to `start_tts_evaluation_batch` (Gemini Batch; sets `batch_job_id` so the cron polls it), SYNC models fan out `ceil(n / TTS_SYNC_CHUNK_SIZE)` `run_tts_sync_chunk` tasks per model (`celery/utils.py::start_tts_sync_chunk`, priority 2) → `services/tts_evaluations/sync_generation.py::execute_tts_sync_chunk`. Sync-only runs leave `batch_job_id` NULL, so the Gemini cron never picks them up; the last chunk to finish finalizes the run. A chunk only synthesizes rows still PENDING (idempotent on redelivery), runs provider calls + S3 uploads in a bounded thread pool with no DB session held, and writes each outcome in its own short session; on timeout/crash its leftover PENDING rows are marked FAILED so the run can't strand. If every batch model fails in a mixed run, the run continues with its sync models. -- TTS DB-connection discipline: submission, sync chunks and Gemini result processing (`batch_result_processing.py`) never hold a session across network I/O — short sessions for prefetch, session-free download/convert/upload, result writes committed in `TTS_RESULT_WRITE_CHUNK_SIZE` slices, fresh session for failure marking. `core/batch/operations.py::start_batch_job` commits (ending the read transaction opened by `create_batch_job`'s refresh) before `provider.create_batch` for every caller. The cron's `poll_batch_status` loop still runs inside the cron session's open transaction. +- TTS submission (`run_tts_batch_submission` → `services/tts_evaluations/batch_job.py::execute_batch_submission`) creates one `tts_result` per text×model and sets the run `processing` before any model work; Gemini rows go to `start_tts_evaluation_batch` (Gemini Batch), and each Sarvam/ElevenLabs model gets one `run_tts_sync_generation` task (`celery/utils.py::start_tts_sync_generation`, priority 2, own `soft_time_limit`/`gevent_timeout` of `TTS_SYNC_TASK_TIME_LIMIT_SECONDS`) → `services/tts_evaluations/sync_generation.py::execute_tts_sync_generation`, which synthesizes that model's PENDING rows `TTS_SYNC_MAX_WORKERS` (2) at a time in a thread pool (idempotent on redelivery), with clients from `SarvamAIProvider.create_client` / `ElevenlabsAIProvider.create_client`, and writes each outcome in its own short session; on timeout/crash it marks its leftover PENDING rows FAILED. A Gemini submission failure or a sync task that can't be queued marks only that model's rows FAILED; the other models carry on. +- TTS cron (`crud/tts_evaluations/cron.py::poll_all_pending_tts_evaluations`) has its own loop (not `cron_utils.poll_all_pending_evaluations_by_type`): it picks up every `processing` TTS run, including sync-only runs with NULL `batch_job_id`, and per run (`poll_tts_run`) builds the Gemini client only when the run has `batch_job` rows (a client failure fails only that run's Gemini rows), polls batches and dispatches `run_tts_result_processing` on SUCCEEDED, fails a model's PENDING rows when its batch ends FAILED/CANCELLED/EXPIRED, and — once the run is older than `TTS_SYNC_STALE_AFTER_SECONDS` — fails leftover PENDING rows of sync models (`"Synthesis did not complete"`) and of Gemini models with no batch job. It then calls `finalize_tts_run_status`. A poll exception leaves the run `processing` for the next tick. Gemini result processing does not finalize; on failure it fails only its model's PENDING rows. +- TTS DB-connection discipline: submission, sync generation and Gemini result processing (`batch_result_processing.py`) never hold a session across network I/O — short sessions for prefetch, session-free CSV read/download/convert/upload (Gemini rows are modified detached and written back in one short session), fresh session for failure marking. `core/batch/operations.py::start_batch_job` commits (ending the read transaction opened by `create_batch_job`'s refresh) before `provider.create_batch` for every caller. The cron's `poll_batch_status` loop still runs inside the cron session's open transaction. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. - Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. From bfffa676349747a7b8f750dde7b3f3a6f077ebb7 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Thu, 8 Oct 2026 08:34:40 +0530 Subject: [PATCH 3/3] chore: remove liberal time limit for sync job tts tasks --- backend/app/celery/tasks/job_execution.py | 14 ++------------ backend/app/crud/tts_evaluations/cron.py | 8 ++++++-- backend/app/services/tts_evaluations/constants.py | 7 +++---- .../app/tests/crud/tts_evaluations/test_cron.py | 7 +++++-- docs/wiki/modules/evaluations.md | 4 ++-- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/backend/app/celery/tasks/job_execution.py b/backend/app/celery/tasks/job_execution.py index dc087c7c6..375ebc89d 100644 --- a/backend/app/celery/tasks/job_execution.py +++ b/backend/app/celery/tasks/job_execution.py @@ -26,7 +26,6 @@ from app.celery.celery_app import celery_app from app.celery.utils import gevent_timeout from app.core.config import settings -from app.services.tts_evaluations.constants import TTS_SYNC_TASK_TIME_LIMIT_SECONDS if TYPE_CHECKING: from app.services.notifications.eval_completion import ( @@ -430,17 +429,8 @@ def run_tts_result_processing( ) -# Same hard-minus-soft grace as the global limits, on top of the longer per-model soft limit. -@celery_app.task( - bind=True, - queue="default", - priority=2, - soft_time_limit=TTS_SYNC_TASK_TIME_LIMIT_SECONDS, - time_limit=TTS_SYNC_TASK_TIME_LIMIT_SECONDS - + settings.CELERY_TASK_TIME_LIMIT - - settings.CELERY_TASK_SOFT_TIME_LIMIT, -) -@gevent_timeout(TTS_SYNC_TASK_TIME_LIMIT_SECONDS, "run_tts_sync_generation") +@celery_app.task(bind=True, queue="default", priority=2) +@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_tts_sync_generation") def run_tts_sync_generation( self, project_id: int, job_id: str, trace_id: str, **kwargs ): diff --git a/backend/app/crud/tts_evaluations/cron.py b/backend/app/crud/tts_evaluations/cron.py index a451341b8..62b0d5d33 100644 --- a/backend/app/crud/tts_evaluations/cron.py +++ b/backend/app/crud/tts_evaluations/cron.py @@ -14,6 +14,7 @@ from app.celery.utils import start_tts_result_processing from app.core.batch import BatchJobState, GeminiBatchProvider, GeminiClient +from app.core.config import settings from app.core.util import now from app.crud.evaluations.cron_utils import ( TERMINAL_STATES, @@ -34,7 +35,7 @@ from app.services.tts_evaluations.constants import ( GEMINI_TTS_MODELS, SYNC_TTS_MODELS, - TTS_SYNC_STALE_AFTER_SECONDS, + TTS_SYNC_STALE_GRACE_SECONDS, ) logger = logging.getLogger(__name__) @@ -181,7 +182,10 @@ async def poll_tts_run( batch_models = [model for model in models if model in GEMINI_TTS_MODELS] sync_models = [model for model in models if model in SYNC_TTS_MODELS] # Both timestamps are naive UTC (`now()` strips tzinfo before storing). - is_stale = now() - run.inserted_at > timedelta(seconds=TTS_SYNC_STALE_AFTER_SECONDS) + stale_after = timedelta( + seconds=settings.CELERY_TASK_TIME_LIMIT + TTS_SYNC_STALE_GRACE_SECONDS + ) + is_stale = now() - run.inserted_at > stale_after any_dispatched = False batch_jobs: list[BatchJob] = [] diff --git a/backend/app/services/tts_evaluations/constants.py b/backend/app/services/tts_evaluations/constants.py index 6b16716e4..aa483800f 100644 --- a/backend/app/services/tts_evaluations/constants.py +++ b/backend/app/services/tts_evaluations/constants.py @@ -49,7 +49,6 @@ TTS_AUDIO_SUBDIRECTORY = "evaluations/tts/audio" -# One task synthesizes a whole model sequentially; the global 300s limit only fits ~30-60 rows. -TTS_SYNC_TASK_TIME_LIMIT_SECONDS = 1800 -# Past this, the cron treats a sync model's leftover PENDING rows as abandoned by a dead worker. -TTS_SYNC_STALE_AFTER_SECONDS = TTS_SYNC_TASK_TIME_LIMIT_SECONDS + 600 +# Once a run is older than the Celery hard time limit plus this grace, no sync worker can +# still be writing its rows, so the cron fails any that are left PENDING. +TTS_SYNC_STALE_GRACE_SECONDS = 600 diff --git a/backend/app/tests/crud/tts_evaluations/test_cron.py b/backend/app/tests/crud/tts_evaluations/test_cron.py index c50747102..5e82a1275 100644 --- a/backend/app/tests/crud/tts_evaluations/test_cron.py +++ b/backend/app/tests/crud/tts_evaluations/test_cron.py @@ -15,7 +15,8 @@ from app.models.batch_job import BatchJob from app.models.job import JobStatus from app.models.tts_evaluation import TTSResult -from app.services.tts_evaluations.constants import TTS_SYNC_STALE_AFTER_SECONDS +from app.core.config import settings +from app.services.tts_evaluations.constants import TTS_SYNC_STALE_GRACE_SECONDS from app.tests.utils.auth import TestAuthContext from app.tests.utils.tts_evaluation import ( create_test_tts_batch_job, @@ -39,7 +40,9 @@ def _processing_run( ) run.status = "processing" if stale: - run.inserted_at -= timedelta(seconds=TTS_SYNC_STALE_AFTER_SECONDS + 60) + run.inserted_at -= timedelta( + seconds=settings.CELERY_TASK_TIME_LIMIT + TTS_SYNC_STALE_GRACE_SECONDS + 60 + ) db.add(run) db.commit() return run diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index d0c4c4feb..23da35bda 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,8 +38,8 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. -- TTS submission (`run_tts_batch_submission` → `services/tts_evaluations/batch_job.py::execute_batch_submission`) creates one `tts_result` per text×model and sets the run `processing` before any model work; Gemini rows go to `start_tts_evaluation_batch` (Gemini Batch), and each Sarvam/ElevenLabs model gets one `run_tts_sync_generation` task (`celery/utils.py::start_tts_sync_generation`, priority 2, own `soft_time_limit`/`gevent_timeout` of `TTS_SYNC_TASK_TIME_LIMIT_SECONDS`) → `services/tts_evaluations/sync_generation.py::execute_tts_sync_generation`, which synthesizes that model's PENDING rows `TTS_SYNC_MAX_WORKERS` (2) at a time in a thread pool (idempotent on redelivery), with clients from `SarvamAIProvider.create_client` / `ElevenlabsAIProvider.create_client`, and writes each outcome in its own short session; on timeout/crash it marks its leftover PENDING rows FAILED. A Gemini submission failure or a sync task that can't be queued marks only that model's rows FAILED; the other models carry on. -- TTS cron (`crud/tts_evaluations/cron.py::poll_all_pending_tts_evaluations`) has its own loop (not `cron_utils.poll_all_pending_evaluations_by_type`): it picks up every `processing` TTS run, including sync-only runs with NULL `batch_job_id`, and per run (`poll_tts_run`) builds the Gemini client only when the run has `batch_job` rows (a client failure fails only that run's Gemini rows), polls batches and dispatches `run_tts_result_processing` on SUCCEEDED, fails a model's PENDING rows when its batch ends FAILED/CANCELLED/EXPIRED, and — once the run is older than `TTS_SYNC_STALE_AFTER_SECONDS` — fails leftover PENDING rows of sync models (`"Synthesis did not complete"`) and of Gemini models with no batch job. It then calls `finalize_tts_run_status`. A poll exception leaves the run `processing` for the next tick. Gemini result processing does not finalize; on failure it fails only its model's PENDING rows. +- TTS submission (`run_tts_batch_submission` → `services/tts_evaluations/batch_job.py::execute_batch_submission`) creates one `tts_result` per text×model and sets the run `processing` before any model work; Gemini rows go to `start_tts_evaluation_batch` (Gemini Batch), and each Sarvam/ElevenLabs model gets one `run_tts_sync_generation` task (`celery/utils.py::start_tts_sync_generation`, priority 2, global Celery time limits like every other task) → `services/tts_evaluations/sync_generation.py::execute_tts_sync_generation`, which synthesizes that model's PENDING rows `TTS_SYNC_MAX_WORKERS` (2) at a time in a thread pool (idempotent on redelivery), with clients from `SarvamAIProvider.create_client` / `ElevenlabsAIProvider.create_client`, and writes each outcome in its own short session; on timeout/crash it marks its leftover PENDING rows FAILED. A Gemini submission failure or a sync task that can't be queued marks only that model's rows FAILED; the other models carry on. +- TTS cron (`crud/tts_evaluations/cron.py::poll_all_pending_tts_evaluations`) has its own loop (not `cron_utils.poll_all_pending_evaluations_by_type`): it picks up every `processing` TTS run, including sync-only runs with NULL `batch_job_id`, and per run (`poll_tts_run`) builds the Gemini client only when the run has `batch_job` rows (a client failure fails only that run's Gemini rows), polls batches and dispatches `run_tts_result_processing` on SUCCEEDED, fails a model's PENDING rows when its batch ends FAILED/CANCELLED/EXPIRED, and — once the run is older than `CELERY_TASK_TIME_LIMIT` + `TTS_SYNC_STALE_GRACE_SECONDS` — fails leftover PENDING rows of sync models (`"Synthesis did not complete"`) and of Gemini models with no batch job. It then calls `finalize_tts_run_status`. A poll exception leaves the run `processing` for the next tick. Gemini result processing does not finalize; on failure it fails only its model's PENDING rows. - TTS DB-connection discipline: submission, sync generation and Gemini result processing (`batch_result_processing.py`) never hold a session across network I/O — short sessions for prefetch, session-free CSV read/download/convert/upload (Gemini rows are modified detached and written back in one short session), fresh session for failure marking. `core/batch/operations.py::start_batch_job` commits (ending the read transaction opened by `create_batch_job`'s refresh) before `provider.create_batch` for every caller. The cron's `poll_batch_status` loop still runs inside the cron session's open transaction. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. - Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM.