feat(eval): add --backend flag and cross-domain combined metrics to topic relevance eval

backend/app/evaluation/README.md

@@ -259,6 +259,14 @@ The predictions CSV includes `scope_score` (the LLM-assigned score) and `error_m
 python3 app/evaluation/topic_relevance/run.py
 ```
 
+Use `--backend` to run only one of the two validators (`topic_relevance` or `topic_relevance_llm`) instead of both:
+
+```bash
+python3 app/evaluation/topic_relevance/run.py --backend topic_relevance_llm
+```
+
+`topic_relevance` (the `LLMCritic`-based validator) is only imported when that backend actually runs, so `--backend topic_relevance_llm` works without installing the `llm_critic` hub validator.
+
 > **Note:** Requires `OPENAI_API_KEY` to be set. Uses `gpt-4o-mini` by default (`DEFAULT_CONFIG` in the script).
 
 ---

backend/app/evaluation/common/helper.py

@@ -80,12 +80,7 @@ def build_validator_config(validator: Any, **fields: Any) -> dict[str, Any]:
     return config
 
 
-def compute_binary_metrics(y_true, y_pred):
-    tp = sum((yt == 1 and yp == 1) for yt, yp in zip(y_true, y_pred, strict=True))
-    tn = sum((yt == 0 and yp == 0) for yt, yp in zip(y_true, y_pred, strict=True))
-    fp = sum((yt == 0 and yp == 1) for yt, yp in zip(y_true, y_pred, strict=True))
-    fn = sum((yt == 1 and yp == 0) for yt, yp in zip(y_true, y_pred, strict=True))
-
+def _confusion_rates(tp: int, tn: int, fp: int, fn: int) -> dict[str, float]:
     precision = tp / (tp + fp) if tp + fp else 0.0
     recall = tp / (tp + fn) if tp + fn else 0.0
     f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
@@ -94,17 +89,67 @@ def compute_binary_metrics(y_true, y_pred):
     accuracy = (tp + tn) / total if total else 0.0
 
     return {
-        "true_positive": tp,
-        "true_negative": tn,
-        "false_positive": fp,
-        "false_negative": fn,
         "accuracy": round(accuracy, 2),
         "precision": round(precision, 2),
         "recall": round(recall, 2),
         "f1": round(f1, 2),
     }
 
 
+def compute_binary_metrics(y_true, y_pred):
+    tp = sum((yt == 1 and yp == 1) for yt, yp in zip(y_true, y_pred, strict=True))
+    tn = sum((yt == 0 and yp == 0) for yt, yp in zip(y_true, y_pred, strict=True))
+    fp = sum((yt == 0 and yp == 1) for yt, yp in zip(y_true, y_pred, strict=True))
+    fn = sum((yt == 1 and yp == 0) for yt, yp in zip(y_true, y_pred, strict=True))
+
+    return {
+        "true_positive": tp,
+        "true_negative": tn,
+        "false_positive": fp,
+        "false_negative": fn,
+        **_confusion_rates(tp, tn, fp, fn),
+    }
+
+
+def combine_binary_metrics(
+    metrics_list: list[dict[str, Any]], labels: list[str] | None = None
+) -> dict[str, Any]:
+    """
+    Combine multiple compute_binary_metrics() results (e.g. one per dataset or
+    domain) into a single aggregate by summing their confusion-matrix counts
+    and re-deriving accuracy/precision/recall/f1 from the totals.
+
+    Pure arithmetic over already-computed metrics — does not re-run any
+    validator, so combining results costs no extra API/LLM calls.
+
+    `labels` (one per entry in metrics_list) namespaces category_metrics keys
+    (e.g. "education/Misc") so identically-named categories from different
+    sources don't collide; defaults to the entry's index if omitted.
+    """
+    tp = sum(m["true_positive"] for m in metrics_list)
+    tn = sum(m["true_negative"] for m in metrics_list)
+    fp = sum(m["false_positive"] for m in metrics_list)
+    fn = sum(m["false_negative"] for m in metrics_list)
+
+    combined: dict[str, Any] = {
+        "true_positive": tp,
+        "true_negative": tn,
+        "false_positive": fp,
+        "false_negative": fn,
+        **_confusion_rates(tp, tn, fp, fn),
+    }
+
+    category_metrics = {}
+    for i, m in enumerate(metrics_list):
+        label = labels[i] if labels else str(i)
+        for category, category_metric in m.get("category_metrics", {}).items():
+            category_metrics[f"{label}/{category}"] = category_metric
+    if category_metrics:
+        combined["category_metrics"] = category_metrics
+
+    return combined
+
+
 class Profiler:
     def __enter__(self):
         self.latencies = []

backend/app/evaluation/topic_relevance/run.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
 
+import argparse
 from pathlib import Path
 
 import pandas as pd
 from guardrails.validators import FailResult
 
 from app.core.config import settings
-from app.core.validators.topic_relevance import TopicRelevance
 from app.core.validators.topic_relevance_llm import TopicRelevanceLLM
 from app.evaluation.common.helper import (
     Profiler,
     build_evaluation_report,
     build_validator_config,
+    combine_binary_metrics,
     compute_binary_metrics,
     write_csv,
     write_json,
@@ -34,15 +35,30 @@
     },
 ]
 
+
+# v2 = forbidden-topics-only scoring (see prompts/topic_relevance_llm/v2.md);
+# matches the education/healthcare configs, which now list forbidden topics only.
+TOPIC_RELEVANCE_LLM_PROMPT_SCHEMA_VERSION = 2
+
+
+def _build_topic_relevance(topic_config: str):
+    # Imported lazily: TopicRelevance pulls in guardrails.hub's LLMCritic, which
+    # requires the llm_critic hub validator to be installed. Deferring the import
+    # means selecting only --backend topic_relevance_llm doesn't need that install.
+    from app.core.validators.topic_relevance import TopicRelevance
+
+    return TopicRelevance(
+        topic_config=topic_config,
+        prompt_schema_version=1,
+        llm_callable=settings.DEFAULT_LLM_CALLABLE,
+    )
+
+
 BACKENDS = [
     {
         "name": "topic_relevance",
         "out_dir": OUTPUTS_DIR / "topic_relevance",
-        "build": lambda tc: TopicRelevance(
-            topic_config=tc,
-            prompt_schema_version=1,
-            llm_callable=settings.DEFAULT_LLM_CALLABLE,
-        ),
+        "build": _build_topic_relevance,
         "config_fields": lambda v: {
             "llm_callable": v.llm_callable,
             "prompt_schema_version": v.prompt_schema_version,
@@ -55,16 +71,21 @@
             system_prompt=tc,
             llm_callable=settings.DEFAULT_LLM_CALLABLE,
             threshold=settings.TOPIC_RELEVANCE_LLM_THRESHOLD,
+            prompt_schema_version=TOPIC_RELEVANCE_LLM_PROMPT_SCHEMA_VERSION,
         ),
+        # TopicRelevanceLLM doesn't store prompt_schema_version on the instance
+        # (it's only used to pick the prompt template at construction time), so
+        # it's recorded here from the constant rather than read off the validator.
         "config_fields": lambda v: {
             "llm_callable": v.llm_callable,
             "threshold": v.threshold,
+            "prompt_schema_version": TOPIC_RELEVANCE_LLM_PROMPT_SCHEMA_VERSION,
         },
     },
 ]
 
 
-def run_evaluation(dataset: dict, backend: dict) -> None:
+def run_evaluation(dataset: dict, backend: dict) -> dict:
     domain = dataset["domain"]
     topic_config = (DATASETS_DIR / dataset["topic_config"]).read_text()
     dataset_path = DATASETS_DIR / dataset["dataset"]
@@ -125,11 +146,46 @@ def run_evaluation(dataset: dict, backend: dict) -> None:
 
     print(f"Completed {backend['name']} {domain} evaluation")
 
+    return metrics
+
 
 def main() -> None:
-    for backend in BACKENDS:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--backend",
+        choices=[backend["name"] for backend in BACKENDS],
+        help="Only run this backend (default: run all backends)",
+    )
+    args = parser.parse_args()
+
+    backends = BACKENDS
+    if args.backend:
+        backends = [b for b in BACKENDS if b["name"] == args.backend]
+
+    for backend in backends:
+        domain_metrics = []
+        domain_labels = []
         for dataset in DATASETS:
-            run_evaluation(dataset, backend)
+            domain_metrics.append(run_evaluation(dataset, backend))
+            domain_labels.append(dataset["domain"])
+
+        combined = combine_binary_metrics(domain_metrics, labels=domain_labels)
+        write_json(
+            {
+                "guardrail": backend["name"],
+                "domains": domain_labels,
+                "num_samples": sum(
+                    m["true_positive"]
+                    + m["true_negative"]
+                    + m["false_positive"]
+                    + m["false_negative"]
+                    for m in domain_metrics
+                ),
+                **combined,
+            },
+            backend["out_dir"] / "combined-metrics.json",
+        )
+        print(f"\nWrote combined metrics for {backend['name']} across {domain_labels}")
 
 
 if __name__ == "__main__":