diff --git a/backend/app/evaluation/README.md b/backend/app/evaluation/README.md index 7a62ecf..d983ba5 100644 --- a/backend/app/evaluation/README.md +++ b/backend/app/evaluation/README.md @@ -13,7 +13,6 @@ backend/app/evaluation/ ├── datasets/ # Evaluation datasets (downloaded separately) │ ├── ban_list_testing_dataset.csv │ ├── gender_bias_assumption_dataset.csv -│ ├── lexical_slur_testing_dataset.csv │ ├── multi_validator_whatsapp_dataset.csv │ ├── pii_detection_testing_dataset.csv │ ├── sharechat_toxic_dataset.csv @@ -22,20 +21,16 @@ backend/app/evaluation/ │ │ ├── education_topic_config.txt │ │ ├── healthcare-topic-relevance-dataset.csv │ │ └── healthcare_topic_config.txt -│ └── toxicity/ # Toxicity evaluation datasets -│ ├── toxicity_test_hasoc.csv -│ └── toxicity_test_sharechat.csv +│ └── toxicity/ # Toxicity evaluation dataset +│ └── toxicity_test_combined.csv # Standardized input consumed by run.py (see Toxicity section for its source mapping) ├── gender_assumption_bias/ │ └── run.py # Gender assumption bias evaluation script -├── lexical_slur/ -│ └── run.py # Lexical slur evaluation script ├── multiple_validators/ │ ├── config.json # Multi-validator run configuration │ └── run.py # End-to-end multi-validator evaluation script ├── outputs/ # Generated outputs (created at runtime) │ ├── ban_list/ │ ├── gender_assumption_bias/ -│ ├── lexical_slur/ │ ├── multi_validator_whatsapp/ │ ├── multiple_validators/ │ ├── pii_remover/ @@ -49,7 +44,7 @@ backend/app/evaluation/ ├── topic_relevance/ │ └── run.py # Topic relevance evaluation script └── toxicity/ - └── run.py # Toxicity evaluation script (LlamaGuard7B, NSFWText, ProfanityFree) + └── run.py # Toxicity evaluation script (LlamaGuard7B, NSFWText, ProfanityFree, LexicalSlur) ``` ## Prerequisites @@ -92,7 +87,7 @@ Validators that use LLM-as-judge approach will require credentials for LLM provi ## Running All Evaluations -To run all individual validator evaluations in sequence (lexical slur, PII, gender assumption bias, ban list, topic relevance, toxicity): +To run all individual validator evaluations in sequence (PII, gender assumption bias, ban list, topic relevance, toxicity): ```bash bash scripts/run_all_evaluations.sh @@ -110,34 +105,6 @@ Run any individual evaluation from the `backend/` directory: python3 app/evaluation//run.py ``` -### Lexical Slur (`uli_slur_match`) - -**Script:** `app/evaluation/lexical_slur/run.py` - -**Dataset:** `datasets/lexical_slur_testing_dataset.csv` - -Expected columns in input csv: - -- `commentText` — text to validate -- `label` — ground truth (`1` = abusive, `0` = not abusive) - -**What it does:** Runs each row through the `LexicalSlur` validator and records a binary prediction (`1` if `FailResult`, `0` otherwise). Computes binary classification metrics against the ground truth labels. - -**Output:** - -``` -outputs/lexical_slur/predictions.csv -outputs/lexical_slur/metrics.json -``` - -**Run:** - -```bash -python3 app/evaluation/lexical_slur/run.py -``` - ---- - ### PII Remover (`pii_remover`) **Script:** `app/evaluation/pii/run.py` @@ -271,38 +238,55 @@ python3 app/evaluation/topic_relevance/run.py --backend topic_relevance_llm --- -### Toxicity (`llamaguard_7b`, `nsfw_text`, `profanity_free`) +### Toxicity (`llamaguard_7b`, `nsfw_text`, `profanity_free`, `lexical_slur`) **Script:** `app/evaluation/toxicity/run.py` -**Datasets:** -- `datasets/toxicity/toxicity_test_hasoc.csv` (The HASOC (Hate Speech and Offensive Content) dataset is a multilingual benchmark (mainly Hindi, English, and code-mixed text) used to train and evaluate models for detecting hate speech, offensive language, and abusive content in social media.) -- `datasets/toxicity/toxicity_test_sharechat.csv` +**Dataset:** `datasets/toxicity/toxicity_test_combined.csv` — one standardized file combining three original sources (HASOC, ShareChat, and a lexical-slur set), tagged by origin. -Expected columns — HASOC dataset: +Expected columns: -- `text` — tweet/comment text to validate -- `task1` — ground truth label (`HOF` = hate/offensive/profanity → `1`, `NOT` → `0`) -- `lang` — language code (informational) +- `text` — text to validate +- `label` — binary ground truth (`1` = toxic/abusive, `0` = not) +- `language` — language label (informational) +- `dataset` — which original source the row came from (`hasoc`, `sharechat`, or `lexical`) — used for the per-source metrics breakdown and the `--sources` filter, not for validation itself -Expected columns — ShareChat dataset: +This file was assembled from three original per-source CSVs, each normalized into the schema above via a one-off local data-prep step (not a script checked into the repo) using this column mapping: -- `commentText` — comment text to validate -- `label` — binary ground truth (`1` = toxic, `0` = not toxic) -- `language` — language label (informational) +- HASOC: `text` → `text`, `task1` (`HOF`/`NOT`) → `label` (`1`/`0`), `lang` → `language` +- ShareChat: `commentText` → `text`, `label` (already `1`/`0`) → `label`, `language` → `language` +- Lexical: `commentText` → `text`, `label` (already `1.0`/`0.0`) → `label`, `language` → `language` + +Each row is tagged with `dataset` = its source name. To add or refresh a source, re-apply this same mapping and re-concatenate into `toxicity_test_combined.csv`. -**What it does:** Runs three validators — `LlamaGuard7B`, `NSFWText`, and `ProfanityFree` — across both datasets independently. For each validator, a binary prediction is recorded (`1` if `FailResult`, `0` otherwise) and compared against the ground truth label to compute classification metrics. +> HASOC (Hate Speech and Offensive Content) is a multilingual benchmark (mainly Hindi, English, and code-mixed text) for detecting hate speech, offensive language, and abusive content in social media. -**Output per dataset:** +**What it does:** Runs four validators — `LlamaGuard7B`, `NSFWText`, `ProfanityFree`, and `LexicalSlur` — against the combined dataset. For each validator, a binary prediction is recorded (`1` if `FailResult`, `0` otherwise) and compared against the ground truth label to compute classification metrics, further broken down per original `dataset` source (`source_metrics`). A `combined_pred` column (logical OR across whichever validators ran, computed from their already-produced `*_pred` columns without re-running anything) is also recorded, with its own `combined` entry (and `source_metrics` breakdown) in the metrics JSON — useful for seeing each validator's standalone effectiveness alongside what running them together would catch, both overall and per source. + +`LexicalSlur` (`uli_slur_match`) is evaluated here rather than in its own script — this eval covers it over a superset of the rows its standalone eval used, so to reproduce that older run exactly, use `--validators lexical_slur --sources lexical`. + +Use `--validators` to run only a subset of validators instead of all four: + +```bash +python3 app/evaluation/toxicity/run.py --validators lexical_slur profanity_free +``` + +Use `--sources` to run only rows from a subset of original sources instead of all three: + +```bash +python3 app/evaluation/toxicity/run.py --sources hasoc lexical +``` + +The two flags can be combined. `--validators` is useful for skipping `llamaguard_7b` or `nsfw_text` when only exercising `lexical_slur`/`profanity_free` locally. + +**Output:** ``` -outputs/toxicity/predictions_hasoc.csv -outputs/toxicity/metrics_hasoc.json -outputs/toxicity/predictions_sharechat.csv -outputs/toxicity/metrics_sharechat.json +outputs/toxicity/predictions.csv +outputs/toxicity/metrics.json ``` -Each predictions CSV contains the source text, ground truth (`y_true`), and one `*_pred` column per validator. Each metrics JSON contains accuracy, precision, recall, F1, and performance stats broken down per validator. +`predictions.csv` contains the source text, its origin `dataset`, `language`, ground truth (`y_true`), one `*_pred` column per validator that ran, and a `combined_pred` column. `metrics.json` contains accuracy, precision, recall, F1, and performance stats per validator (each with a `source_metrics` breakdown by origin dataset), plus a `combined` entry for the OR-combined prediction (with its own `source_metrics` breakdown). **Run:** @@ -310,7 +294,7 @@ Each predictions CSV contains the source text, ground truth (`y_true`), and one python3 app/evaluation/toxicity/run.py ``` -> **Note:** `LlamaGuard7B` uses remote inferencing — requires a valid `GUARDRAILS_HUB_API_KEY` and internet access. `NSFWText` downloads the `textdetox/xlmr-large-toxicity-classifier` model on first run. +> **Note:** `llamaguard_7b` is not yet migrated off the sunset Guardrails Hub install path (see `backend/app/core/validators/README.md`) — running it will currently fail regardless of credentials. Use `--validators` to exclude it until that migration lands. `NSFWText` downloads the `textdetox/xlmr-large-toxicity-classifier` model on first run. --- @@ -379,7 +363,7 @@ The output CSV contains `ID`, `text`, `validators_present`, and `response` (the ### Binary Classification Metrics (`metrics.json`) -Used by lexical slur, gender assumption bias, ban list, and topic relevance evaluations. +Used by the gender assumption bias, ban list, topic relevance, and toxicity evaluations. | Metric | Description | | ------------------ | --------------------------------------------------------- | @@ -450,13 +434,12 @@ Each evaluation script expects a specific filename — files must be named exact | Validator | Expected filename | | ---------------------- | --------------------------------------------------------------------------------------------------------------------- | -| Lexical Slur | `lexical_slur_testing_dataset.csv` | | PII Remover | `pii_detection_testing_dataset.csv` | | Gender Assumption Bias | `gender_bias_assumption_dataset.csv` | | Ban List | `ban_list_testing_dataset.csv` | | Multiple Validators | `multi_validator_whatsapp_dataset.csv` | | Topic Relevance | `topic_relevance/education-topic-relevance-dataset.csv`, `topic_relevance/healthcare-topic-relevance-dataset.csv` | -| Toxicity | `toxicity/toxicity_test_hasoc.csv`, `toxicity/toxicity_test_sharechat.csv` | +| Toxicity | `toxicity/toxicity_test_combined.csv` (see Toxicity section for the raw sources and column mapping it's derived from) | Topic relevance also requires plain-text topic config files alongside each dataset: diff --git a/backend/app/evaluation/lexical_slur/run.py b/backend/app/evaluation/lexical_slur/run.py deleted file mode 100644 index 7cd74a0..0000000 --- a/backend/app/evaluation/lexical_slur/run.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -import pandas as pd -from guardrails.validators import FailResult - -from app.core.validators.lexical_slur import LexicalSlur -from app.evaluation.common.helper import ( - build_evaluation_report, - build_validator_config, - Profiler, - compute_binary_metrics, - write_csv, - write_json, -) - -BASE_DIR = Path(__file__).resolve().parent.parent -OUT_DIR = BASE_DIR / "outputs" / "lexical_slur" - -df = pd.read_csv(BASE_DIR / "datasets" / "lexical_slur_testing_dataset.csv") - -validator = LexicalSlur() - -config = build_validator_config( - validator, - severity=validator.severity, - languages=validator.languages, - num_slurs_loaded=len(validator.slur_list), -) - -with Profiler() as p: - df["result"] = ( - df["commentText"] - .astype(str) - .apply(lambda x: p.record(lambda t: validator.validate(t, metadata=None), x)) - ) - -df["y_pred"] = df["result"].apply(lambda r: int(isinstance(r, FailResult))) -df["y_true"] = df["label"] - -metrics = compute_binary_metrics(df["y_true"], df["y_pred"]) - -# ---- Save outputs ---- -write_csv(df.drop(columns=["result"]), OUT_DIR / "predictions.csv") - -write_json( - build_evaluation_report( - guardrail="lexical_slur", - num_samples=len(df), - profiler=p, - config=config, - metrics=metrics, - ), - OUT_DIR / "metrics.json", -) diff --git a/backend/app/evaluation/toxicity/run.py b/backend/app/evaluation/toxicity/run.py index 619c47a..d31b435 100644 --- a/backend/app/evaluation/toxicity/run.py +++ b/backend/app/evaluation/toxicity/run.py @@ -1,10 +1,9 @@ +import argparse from pathlib import Path import pandas as pd -from guardrails.hub import LlamaGuard7B # noqa: not yet migrated, see backend README -from guardrails_ai.nsfw_text import NSFWText -from guardrails_ai.profanity_free import ProfanityFree from guardrails.validators import FailResult +from app.core.validators.lexical_slur import LexicalSlur from app.evaluation.common.helper import ( build_evaluation_report, build_validator_config, @@ -17,36 +16,54 @@ BASE_DIR = Path(__file__).resolve().parent.parent OUT_DIR = BASE_DIR / "outputs" / "toxicity" -DATASETS = { - "hasoc": { - "path": BASE_DIR / "datasets" / "toxicity" / "toxicity_test_hasoc.csv", - "text_col": "text", - "label_col": "task1", - "label_map": {"HOF": 1, "NOT": 0}, - }, - "sharechat": { - "path": BASE_DIR / "datasets" / "toxicity" / "toxicity_test_sharechat.csv", - "text_col": "commentText", - "label_col": "label", - "label_map": None, # already binary int - }, -} +DATASET_PATH = BASE_DIR / "datasets" / "toxicity" / "toxicity_test_combined.csv" +# Standardized columns: text, label (0/1), language, dataset (source name). +SOURCE_NAMES = ["hasoc", "sharechat", "lexical"] + + +def _build_llamaguard_7b(): + # Imported lazily: llamaguard_7b is not yet migrated off guardrails.hub (see + # backend/app/core/validators/README.md), and its local .guardrails/hub_registry.json + # entry still points at the now-defunct guardrails_grhub_llamaguard_7b package, so a + # top-level import would break every run of this script, not just ones selecting it. + from guardrails.hub import ( + LlamaGuard7B, + ) # noqa: not yet migrated, see backend README + + return LlamaGuard7B(on_fail="noop") + + +def _build_nsfw_text(): + # Imported lazily so selecting other validators via --validators doesn't require + # the nsfw_text package (and its model download) to be usable. + from guardrails_ai.nsfw_text import NSFWText + + return NSFWText( + threshold=0.8, + validation_method="sentence", + device="cpu", + model_name="textdetox/xlmr-large-toxicity-classifier", + on_fail="noop", + use_local=True, + ) + + +def _build_profanity_free(): + # Imported lazily so selecting other validators via --validators doesn't require + # the profanity_free package to be usable. + from guardrails_ai.profanity_free import ProfanityFree + + return ProfanityFree(on_fail="noop") + VALIDATORS = { "llamaguard_7b": { - "build": lambda: LlamaGuard7B(on_fail="noop"), - "config_fields": {}, + "build": _build_llamaguard_7b, + "config_fields": lambda v: {}, }, "nsfw_text": { - "build": lambda: NSFWText( - threshold=0.8, - validation_method="sentence", - device="cpu", - model_name="textdetox/xlmr-large-toxicity-classifier", - on_fail="noop", - use_local=True, - ), - "config_fields": { + "build": _build_nsfw_text, + "config_fields": lambda v: { "threshold": 0.8, "validation_method": "sentence", "device": "cpu", @@ -55,41 +72,47 @@ }, }, "profanity_free": { - "build": lambda: ProfanityFree(on_fail="noop"), - "config_fields": {}, + "build": _build_profanity_free, + "config_fields": lambda v: {}, + }, + "lexical_slur": { + "build": lambda: LexicalSlur(on_fail="noop"), + "config_fields": lambda v: { + "severity": v.severity, + "languages": v.languages, + "num_slurs_loaded": len(v.slur_list), + }, }, } -def run_dataset(dataset_name: str, dataset_cfg: dict): - df = pd.read_csv(dataset_cfg["path"]) - text_col = dataset_cfg["text_col"] - label_col = dataset_cfg["label_col"] - label_map = dataset_cfg["label_map"] +def _source_metrics(df: pd.DataFrame, pred_col: str) -> dict: + return { + str(source): compute_binary_metrics(group["y_true"], group[pred_col]) + for source, group in df.groupby("dataset", dropna=False) + } - if label_map is not None: - df["y_true"] = df[label_col].map(label_map) - unmapped = df.loc[df["y_true"].isna(), label_col].unique().tolist() - if unmapped: - raise ValueError( - f"[{dataset_name}] label_col '{label_col}' contains values not in label_map: {unmapped}" - ) - else: - df["y_true"] = df[label_col].astype(int) - missing_text = df[text_col].isna() +def run_evaluation(validators: dict, sources: list[str] | None): + df = pd.read_csv(DATASET_PATH) + df["y_true"] = df["label"].astype(int) + + if sources: + df = df[df["dataset"].isin(sources)].reset_index(drop=True) + + missing_text = df["text"].isna() if missing_text.any(): df = df[~missing_text].copy() all_metrics = {} - for validator_name, spec in VALIDATORS.items(): - print(f" Running {validator_name} on {dataset_name}...") + for validator_name, spec in validators.items(): + print(f"Running {validator_name}...") validator = spec["build"]() - config = build_validator_config(validator, **spec["config_fields"]) + config = build_validator_config(validator, **spec["config_fields"](validator)) with Profiler() as p: - df[f"{validator_name}_result"] = df[text_col].apply( + df[f"{validator_name}_result"] = df["text"].apply( lambda x: p.record(lambda t: validator.validate(t, metadata={}), x) ) @@ -101,9 +124,9 @@ def run_dataset(dataset_name: str, dataset_cfg: dict): df["llamaguard_7b_latency_ms"] = p.latencies metrics = compute_binary_metrics(df["y_true"], df[f"{validator_name}_pred"]) + metrics["source_metrics"] = _source_metrics(df, f"{validator_name}_pred") all_metrics[validator_name] = build_evaluation_report( guardrail=validator_name, - dataset=dataset_name, num_samples=len(df), profiler=p, config=config, @@ -112,19 +135,54 @@ def run_dataset(dataset_name: str, dataset_cfg: dict): df = df.drop(columns=[f"{validator_name}_result"]) - pred_cols = ["y_true"] + [f"{v}_pred" for v in VALIDATORS] + validator_pred_cols = [f"{name}_pred" for name in validators] + df["combined_pred"] = df[validator_pred_cols].max(axis=1) + combined_metrics = compute_binary_metrics(df["y_true"], df["combined_pred"]) + combined_metrics["source_metrics"] = _source_metrics(df, "combined_pred") + all_metrics["combined"] = { + "guardrail": "combined", + "validators": list(validators.keys()), + "num_samples": len(df), + "metrics": combined_metrics, + } + + pred_cols = ["y_true"] + validator_pred_cols + ["combined_pred"] latency_cols = ( ["llamaguard_7b_latency_ms"] if "llamaguard_7b_latency_ms" in df.columns else [] ) write_csv( - df[[text_col, *pred_cols, *latency_cols]], - OUT_DIR / f"predictions_{dataset_name}.csv", + df[["text", "dataset", "language", *pred_cols, *latency_cols]], + OUT_DIR / "predictions.csv", ) - write_json(all_metrics, OUT_DIR / f"metrics_{dataset_name}.json") + write_json(all_metrics, OUT_DIR / "metrics.json") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--validators", + nargs="+", + choices=list(VALIDATORS.keys()), + help="Only run these validators (default: run all validators)", + ) + parser.add_argument( + "--sources", + nargs="+", + choices=SOURCE_NAMES, + help="Only run rows from these original dataset sources (default: run all)", + ) + args = parser.parse_args() + + selected_validators = ( + {name: VALIDATORS[name] for name in args.validators} + if args.validators + else VALIDATORS + ) + + run_evaluation(selected_validators, args.sources) + print("Done. Results saved to", OUT_DIR) -for dataset_name, dataset_cfg in DATASETS.items(): - print(f"Evaluating dataset: {dataset_name}") - run_dataset(dataset_name, dataset_cfg) -print("Done. Results saved to", OUT_DIR) +if __name__ == "__main__": + main() diff --git a/backend/scripts/run_all_evaluations.sh b/backend/scripts/run_all_evaluations.sh index 0da2402..96a7c90 100755 --- a/backend/scripts/run_all_evaluations.sh +++ b/backend/scripts/run_all_evaluations.sh @@ -6,7 +6,6 @@ BACKEND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" EVAL_DIR="$BACKEND_DIR/app/evaluation" RUNNERS=( - "$EVAL_DIR/lexical_slur/run.py" "$EVAL_DIR/pii/run.py" "$EVAL_DIR/gender_assumption_bias/run.py" "$EVAL_DIR/ban_list/run.py"