|
4 | 4 | ordered by the per-task `priority`: |
5 | 5 |
|
6 | 6 | 9 LLM call + LLM chain (run_llm_job, run_llm_chain_job, run_response_job) |
7 | | - 6 Fast evaluation (run_evaluation_fast) |
| 7 | + 6 Fast evaluation (run_evaluation_fast_chunk, run_evaluation_fast_aggregate) |
8 | 8 | 2 Everything else (doctransform, collections, STT/TTS evaluation, assessment) |
9 | 9 | 1 Notifications (send_eval_completion_notification) |
10 | 10 |
|
@@ -352,32 +352,57 @@ def run_tts_result_processing( |
352 | 352 |
|
353 | 353 |
|
354 | 354 | @celery_app.task(bind=True, queue="default", priority=6) |
355 | | -@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_evaluation_fast") |
356 | | -def run_evaluation_fast( |
357 | | - self: Task, eval_run_id: int, trace_id: str = DEFAULT_TRACE_ID |
| 355 | +@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_evaluation_fast_chunk") |
| 356 | +def run_evaluation_fast_chunk( |
| 357 | + self: Task, |
| 358 | + eval_run_id: int, |
| 359 | + chunk_index: int, |
| 360 | + trace_id: str = DEFAULT_TRACE_ID, |
358 | 361 | ) -> None: |
359 | | - """Run the fast evaluation pipeline for one EvaluationRun. |
| 362 | + """Run one responses chunk of a fast EvaluationRun. |
| 363 | +
|
| 364 | + Idempotent: the chunk is skipped when its (eval_run, chunk_index) batch_job |
| 365 | + already has a raw_output_url, so Celery redelivery never re-charges OpenAI. |
| 366 | + """ |
| 367 | + from app.services.evaluations.fast import execute_fast_evaluation_chunk |
| 368 | + |
| 369 | + _set_trace(trace_id) |
| 370 | + logger.info( |
| 371 | + f"[run_evaluation_fast_chunk] Starting fast eval chunk | " |
| 372 | + f"eval_run_id={eval_run_id} | chunk_index={chunk_index} | " |
| 373 | + f"task_id={current_task.request.id}" |
| 374 | + ) |
| 375 | + |
| 376 | + return _run_with_otel_parent( |
| 377 | + self, |
| 378 | + lambda: execute_fast_evaluation_chunk( |
| 379 | + eval_run_id=eval_run_id, chunk_index=chunk_index |
| 380 | + ), |
| 381 | + ) |
360 | 382 |
|
361 | | - Idempotency: each stage is skipped on retry when its `batch_job` marker is |
362 | | - already set on the EvaluationRun, so Celery redelivery never re-calls |
363 | | - OpenAI for work that already succeeded. |
364 | 383 |
|
365 | | - Args: |
366 | | - eval_run_id: ID of the EvaluationRun (run_mode="fast"). |
367 | | - trace_id: Correlation id from the enqueueing request, propagated into |
368 | | - the worker for log correlation. |
| 384 | +@celery_app.task(bind=True, queue="default", priority=6) |
| 385 | +@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_evaluation_fast_aggregate") |
| 386 | +def run_evaluation_fast_aggregate( |
| 387 | + self: Task, eval_run_id: int, trace_id: str = DEFAULT_TRACE_ID |
| 388 | +) -> None: |
| 389 | + """Fan-in: merge the response chunks then run embeddings + scoring. |
| 390 | +
|
| 391 | + Enqueued once by the cron barrier after every chunk has completed. |
| 392 | + Idempotent: each stage is skipped on retry when its `batch_job` marker is |
| 393 | + set, so redelivery never re-does completed work. |
369 | 394 | """ |
370 | | - from app.services.evaluations.fast import execute_fast_evaluation |
| 395 | + from app.services.evaluations.fast import execute_fast_evaluation_aggregate |
371 | 396 |
|
372 | 397 | _set_trace(trace_id) |
373 | 398 | logger.info( |
374 | | - f"[run_evaluation_fast] Starting fast evaluation task | " |
| 399 | + f"[run_evaluation_fast_aggregate] Starting fast eval aggregate | " |
375 | 400 | f"eval_run_id={eval_run_id} | task_id={current_task.request.id}" |
376 | 401 | ) |
377 | 402 |
|
378 | 403 | return _run_with_otel_parent( |
379 | 404 | self, |
380 | | - lambda: execute_fast_evaluation(eval_run_id=eval_run_id), |
| 405 | + lambda: execute_fast_evaluation_aggregate(eval_run_id=eval_run_id), |
381 | 406 | ) |
382 | 407 |
|
383 | 408 |
|
|
0 commit comments