diff --git a/backend/app/api/API_USAGE.md b/backend/app/api/API_USAGE.md index e04d003..f8542d4 100644 --- a/backend/app/api/API_USAGE.md +++ b/backend/app/api/API_USAGE.md @@ -522,6 +522,7 @@ From `validators.json`: - `profanity_free` - `nsfw_text` - `answer_relevance_custom_llm` +- `content_safety` Source of truth: - `backend/app/core/validators/validators.json` diff --git a/backend/app/core/enum.py b/backend/app/core/enum.py index 55b568b..c4b1b4c 100644 --- a/backend/app/core/enum.py +++ b/backend/app/core/enum.py @@ -61,3 +61,4 @@ class ValidatorType(Enum): ProfanityFree = "profanity_free" NSFWText = "nsfw_text" AnswerRelevanceCustomLLM = "answer_relevance_custom_llm" + ContentSafety = "content_safety" diff --git a/backend/app/core/validators/README.md b/backend/app/core/validators/README.md index b8fc87e..39acc58 100644 --- a/backend/app/core/validators/README.md +++ b/backend/app/core/validators/README.md @@ -16,6 +16,7 @@ Current validator manifest: - `profanity_free` (source: `hub://guardrails/profanity_free`) - `nsfw_text` (source: `hub://guardrails/nsfw_text`) - `answer_relevance_custom_llm` (source: `local`) +- `content_safety` (source: `local`) - wraps `uli_slur_match` and the `profanity_free` model ## Configuration Model @@ -559,6 +560,60 @@ Notes / limitations: - `on_fail=fix` has no programmatic fix for irrelevant answers — `safe_text` will be `""` and `metadata.reason` will identify this validator. - If `custom_prompt_id` is deleted after being referenced, the guardrail will return a 404 at resolution time. +### 11) Content Safety Validator (`content_safety`) + +Code: + +- Config: `backend/app/core/validators/config/content_safety_safety_validator_config.py` +- Runtime validator: `backend/app/core/validators/content_safety.py` + +What it does: + +- Runs the lexical slur check (`uli_slur_match`) and a profanity check as a single validator, so one config replaces stacking both. +- Step 1: redacts slurs from the curated list with `[REDACTED_SLUR]`. +- Step 2: scores the redacted text with `alt-profanity-check` (the model behind `profanity_free`) and fails it if the score is at or above `profanity_threshold`. +- Returns one pass/fail with both errors joined, e.g. `Mentioned toxic words: randi; Text contains profanity`. + +Why this is used: + +- One validator config for content safety instead of separate `uli_slur_match` and `profanity_free` configs. +- Slur-first ordering keeps the message when a listed slur is the only problem: `"That guy is a faggot, ignore him"` becomes `"that guy is a [REDACTED_SLUR], ignore him"`, whereas `profanity_free` alone would return `""`. +- A configurable profanity threshold avoids `profanity_free`'s Hinglish false positives (see limitations). + +Recommendation: + +- `input` and `output` + - Why `input`: catches slurs and profanity in user messages. + - Why `output`: prevents model-generated slurs or profanity from reaching users. + +Parameters / customization: + +- `slur_severity: "low" | "medium" | "high" | "all"` (default: `all`) — minimum slur severity to redact; same meaning as `severity` on `uli_slur_match` +- `profanity_threshold: float` (0–1, default: `0.9`) — profanity probability at or above which the text fails +- `on_fail` + +Example stored config: + +```json +{ + "name": "content safety input", + "type": "content_safety", + "stage": "input", + "on_fail_action": "fix", + "slur_severity": "all", + "profanity_threshold": 0.9 +} +``` + +Notes / limitations: + +- **Why 0.9, not `profanity_free`'s 0.5:** the profanity model is English-only. It reads `ho` as the slur "hoe" (score 0.883) and `tu` as 0.535, so at 0.5 common Hinglish such as `namaste, kaise ho`, `kya ho raha hai` and `tu kahan hai` is blanked. English profanity scores 0.95–1.0, so it is still caught at 0.9. Trade-off: `screw you` (0.893) passes. Lower the threshold for stricter English filtering. +- Hindi slurs are caught only by the slur list; the profanity model doesn't know them. +- If profanity remains after slur redaction, there is no programmatic fix: with `on_fail=fix`, `safe_text` is `""` and `metadata.reason` identifies this validator. +- Slur redaction lowercases the whole text and strips emoji (inherited from `uli_slur_match`). +- `languages` is not accepted: `uli_slur_match` stores it but never uses it. +- Error messages never include the user's text, so they are safe to show with `on_fail=rephrase` and to store in validator logs. + ## Example Config Payloads Example: create validator config (stored shape) @@ -589,6 +644,7 @@ Default stage strategy: - Input guardrails: `pii_remover`, `uli_slur_match`, `ban_list`, `topic_relevance` (when scope enforcement is needed), `profanity_free`, `llamaguard_7b` - Output guardrails: `pii_remover`, `uli_slur_match`, `gender_assumption_bias`, `ban_list`, `profanity_free`, `llamaguard_7b`, `answer_relevance_custom_llm` (when answer quality must be verified) +- `content_safety` can replace `uli_slur_match` + `profanity_free` at either stage; don't configure it alongside them. Tuning strategy: @@ -626,7 +682,7 @@ here don't even accept it. `BanList`'s `__init__` has no `**kwargs` at all and r `has_guardrails_endpoint=True`, so it's a no-op either way. Check the constructor and the decorator before adding the kwarg to a new validator's `build()`. The fully custom in-repo validators (`pii_remover`, `gender_assumption_bias`, `uli_slur_match`, `topic_relevance`, -`topic_relevance_llm`, `answer_relevance_custom_llm`) have no hub/remote code path at all and +`topic_relevance_llm`, `answer_relevance_custom_llm`, `content_safety`) have no hub/remote code path at all and don't need it. ## Related Files @@ -643,6 +699,8 @@ don't need it. - `backend/app/core/validators/config/profanity_free_safety_validator_config.py` - `backend/app/core/validators/config/answer_relevance_custom_llm_safety_validator_config.py` - `backend/app/core/validators/answer_relevance_custom_llm.py` +- `backend/app/core/validators/config/content_safety_safety_validator_config.py` +- `backend/app/core/validators/content_safety.py` - `backend/app/models/config/llm_prompt_config.py` - `backend/app/crud/llm_prompt_config.py` - `backend/app/schemas/guardrail_config.py` diff --git a/backend/app/core/validators/config/content_safety_safety_validator_config.py b/backend/app/core/validators/config/content_safety_safety_validator_config.py new file mode 100644 index 0000000..11256f3 --- /dev/null +++ b/backend/app/core/validators/config/content_safety_safety_validator_config.py @@ -0,0 +1,26 @@ +from typing import Literal + +from pydantic import Field + +from app.core.enum import SlurSeverity +from app.core.validators.config.base_validator_config import BaseValidatorConfig +from app.core.validators.content_safety import ( + DEFAULT_PROFANITY_THRESHOLD, + ContentSafety, +) + + +class ContentSafetySafetyValidatorConfig(BaseValidatorConfig): + type: Literal["content_safety"] + slur_severity: Literal[ + "low", "medium", "high", "all" + ] = "all" # minimum slur severity to flag + # Profanity probability at or above which the text fails and is blanked. + profanity_threshold: float = Field(default=DEFAULT_PROFANITY_THRESHOLD, ge=0, le=1) + + def build(self): + return ContentSafety( + slur_severity=SlurSeverity(self.slur_severity), + profanity_threshold=self.profanity_threshold, + on_fail=self.resolve_on_fail(), + ) diff --git a/backend/app/core/validators/content_safety.py b/backend/app/core/validators/content_safety.py new file mode 100644 index 0000000..d35a1ff --- /dev/null +++ b/backend/app/core/validators/content_safety.py @@ -0,0 +1,66 @@ +from collections.abc import Callable + +from guardrails import OnFailAction +from guardrails.validators import ( + FailResult, + PassResult, + ValidationResult, + Validator, + register_validator, +) +from profanity_check import predict_prob + +from app.core.enum import SlurSeverity +from app.core.validators.lexical_slur import LexicalSlur + +PROFANITY_ERROR_MESSAGE = "Text contains profanity" +# ProfanityFree uses the model's fixed 0.5 cutoff, which fails common Hinglish: +# the English model weights "ho" (as in "kaise ho") at 0.883 and "tu" at 0.535. +DEFAULT_PROFANITY_THRESHOLD = 0.9 + + +@register_validator(name="content-safety", data_type="string") +class ContentSafety(Validator): + """ + Runs lexical slur redaction followed by profanity detection as one validator. + + The slur check runs first so a redacted slur doesn't trip the profanity check, + which has no partial fix and would otherwise blank the whole text. Profanity is + scored with alt-profanity-check (the model behind ProfanityFree) against a + configurable threshold instead of ProfanityFree's fixed 0.5. + """ + + def __init__( + self, + slur_severity: SlurSeverity = SlurSeverity.All, + profanity_threshold: float = DEFAULT_PROFANITY_THRESHOLD, + on_fail: Callable | OnFailAction | None = OnFailAction.FIX, + ): + # Inner on_fail is never invoked; only this wrapper's on_fail applies. + self._slur = LexicalSlur(severity=slur_severity, on_fail=OnFailAction.NOOP) + self.profanity_threshold = profanity_threshold + super().__init__( + on_fail=on_fail, + slur_severity=slur_severity, + profanity_threshold=profanity_threshold, + ) + + def _validate(self, value: str, metadata: dict | None = None) -> ValidationResult: + metadata = metadata or {} + errors = [] + current = value + + slur_result = self._slur.validate(current, metadata) + if isinstance(slur_result, FailResult): + errors.append(slur_result.error_message) + current = slur_result.fix_value + + # Fixed message: never echo user text into logs or rephrase responses. + if predict_prob([current])[0] >= self.profanity_threshold: + errors.append(PROFANITY_ERROR_MESSAGE) + current = "" + + if not errors: + return PassResult() + + return FailResult(error_message="; ".join(errors), fix_value=current) diff --git a/backend/app/core/validators/validators.json b/backend/app/core/validators/validators.json index 6e08bb7..a1bbbb9 100644 --- a/backend/app/core/validators/validators.json +++ b/backend/app/core/validators/validators.json @@ -49,6 +49,16 @@ "type": "nsfw_text", "version": "0.1.0", "source": "pypi://guardrails-ai-nsfw-text" + }, + { + "type": "answer_relevance_custom_llm", + "version": "0.1.0", + "source": "local" + }, + { + "type": "content_safety", + "version": "0.1.0", + "source": "local" } ] } \ No newline at end of file diff --git a/backend/app/schemas/guardrail_config.py b/backend/app/schemas/guardrail_config.py index c31450f..48060fd 100644 --- a/backend/app/schemas/guardrail_config.py +++ b/backend/app/schemas/guardrail_config.py @@ -14,6 +14,9 @@ from app.core.validators.config.ban_list_safety_validator_config import ( BanListSafetyValidatorConfig, ) +from app.core.validators.config.content_safety_safety_validator_config import ( + ContentSafetySafetyValidatorConfig, +) from app.core.validators.config.gender_assumption_bias_safety_validator_config import ( GenderAssumptionBiasSafetyValidatorConfig, ) @@ -46,6 +49,7 @@ Union[ AnswerRelevanceCustomLLMSafetyValidatorConfig, BanListSafetyValidatorConfig, + ContentSafetySafetyValidatorConfig, GenderAssumptionBiasSafetyValidatorConfig, LexicalSlurSafetyValidatorConfig, LLMCriticSafetyValidatorConfig, diff --git a/backend/app/tests/validators/test_content_safety.py b/backend/app/tests/validators/test_content_safety.py new file mode 100644 index 0000000..6911753 --- /dev/null +++ b/backend/app/tests/validators/test_content_safety.py @@ -0,0 +1,372 @@ +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +from guardrails import Guard, OnFailAction +from guardrails.validators import FailResult, PassResult +from pydantic import ValidationError + +from app.core.enum import SlurSeverity +from app.core.on_fail_actions import rephrase_query_on_fail +from app.core.validators.config.content_safety_safety_validator_config import ( + ContentSafetySafetyValidatorConfig, +) +from app.core.validators.content_safety import ( + DEFAULT_PROFANITY_THRESHOLD, + PROFANITY_ERROR_MESSAGE, + ContentSafety, +) +from app.core.validators.lexical_slur import LexicalSlur + +_PREDICT_PROB_PATCH = "app.core.validators.content_safety.predict_prob" +_CONTENT_SAFETY_PATCH = ( + "app.core.validators.config.content_safety_safety_validator_config.ContentSafety" +) + + +# --------------------------------------- +# Fixtures +# --------------------------------------- +@pytest.fixture +def patch_slur_load(monkeypatch): + """Use a tiny fixed slur list instead of the curated CSV.""" + + def fake_load_slur_list(_self): + df = pd.DataFrame({"label": ["badword", "highslur"], "severity": ["L", "H"]}) + return df["label"].str.lower().tolist() + + monkeypatch.setattr(LexicalSlur, "load_slur_list", fake_load_slur_list) + + +@pytest.fixture +def profanity_score(): + """Stub the profanity model so tests don't depend on it. Set + `.return_value = [score]`; defaults to a clean score.""" + with patch(_PREDICT_PROB_PATCH) as mock: + mock.return_value = [0.0] + yield mock + + +PROFANE = [1.0] + + +# --------------------------------------- +# ContentSafety validator +# --------------------------------------- +@pytest.mark.usefixtures("patch_slur_load", "profanity_score") +def test_passes_clean_text(): + validator = ContentSafety() + + result = validator.validate("This is a perfectly fine sentence", {}) + + assert isinstance(result, PassResult) + + +@pytest.mark.usefixtures("patch_slur_load", "profanity_score") +def test_slur_only_is_redacted(): + validator = ContentSafety() + + result = validator.validate("You are a BADWORD", {}) + + assert isinstance(result, FailResult) + assert result.error_message == "Mentioned toxic words: badword" + assert result.fix_value == "you are a [REDACTED_SLUR]" + + +@pytest.mark.usefixtures("patch_slur_load") +def test_profanity_only_blanks_text(profanity_score): + validator = ContentSafety() + profanity_score.return_value = PROFANE + + result = validator.validate("some profane text", {}) + + assert isinstance(result, FailResult) + assert result.error_message == PROFANITY_ERROR_MESSAGE + assert result.fix_value == "" + + +@pytest.mark.usefixtures("patch_slur_load") +def test_slur_and_profanity_combines_errors(profanity_score): + validator = ContentSafety() + profanity_score.return_value = PROFANE + + result = validator.validate("you badword, some profane text", {}) + + assert isinstance(result, FailResult) + assert result.error_message == ( + f"Mentioned toxic words: badword; {PROFANITY_ERROR_MESSAGE}" + ) + assert result.fix_value == "" + + +@pytest.mark.usefixtures("patch_slur_load") +def test_profanity_check_runs_on_redacted_text(profanity_score): + validator = ContentSafety() + + validator.validate("You are a badword", {}) + + checked_texts = profanity_score.call_args.args[0] + assert checked_texts == ["you are a [REDACTED_SLUR]"] + + +@pytest.mark.usefixtures("patch_slur_load") +def test_error_message_does_not_echo_input(profanity_score): + text = "some profane text" + validator = ContentSafety() + profanity_score.return_value = PROFANE + + result = validator.validate(text, {}) + + assert text not in result.error_message + + +@pytest.mark.usefixtures("patch_slur_load") +def test_slur_severity_forwarded(): + validator = ContentSafety(slur_severity=SlurSeverity.High) + + assert validator._slur.severity == SlurSeverity.High + + +@pytest.mark.usefixtures("patch_slur_load") +def test_inner_slur_validator_uses_noop_on_fail(): + validator = ContentSafety() + + assert validator._slur.on_fail_descriptor == OnFailAction.NOOP + + +@pytest.mark.parametrize( + ("score", "threshold", "fails"), + [ + (0.883, DEFAULT_PROFANITY_THRESHOLD, False), # "kaise ho" with default + (0.9, DEFAULT_PROFANITY_THRESHOLD, True), # threshold is inclusive + (0.95, DEFAULT_PROFANITY_THRESHOLD, True), + (0.535, 0.5, True), # ProfanityFree's old cutoff + (0.95, 0.99, False), + ], +) +@pytest.mark.usefixtures("patch_slur_load") +def test_profanity_threshold(profanity_score, score, threshold, fails): + validator = ContentSafety(profanity_threshold=threshold) + profanity_score.return_value = [score] + + result = validator.validate("some text", {}) + + assert isinstance(result, FailResult) is fails + + +@pytest.mark.usefixtures("patch_slur_load") +def test_rephrase_message_does_not_leak_input(profanity_score): + """Profanity runs on lowercased/redacted text, so rephrase's str.replace + can't scrub a message that embeds it; ours must not contain user text.""" + text = "You badword, some profane text" + validator = ContentSafety() + profanity_score.return_value = PROFANE + + result = validator.validate(text, {}) + rephrased = rephrase_query_on_fail(text, result) + + assert "badword" in rephrased # slur name is part of the reason + assert "some profane text" not in rephrased + + +@pytest.mark.parametrize( + ("text", "expected_fix"), + [ + # Profanity-first would blank all of these (the profanity check has no + # partial fix); slur-first redacts the slurs and keeps the rest. + ("you are a bitch", "you are a [REDACTED_SLUR]"), + ( + "Stop acting like a stupid cunt", + "stop acting like a [REDACTED_SLUR] [REDACTED_SLUR]", + ), + ( + "That guy is a faggot, ignore him", + "that guy is a [REDACTED_SLUR], ignore him", + ), + ( + "What the fuck is this, you bitch", + "what the [REDACTED_SLUR] is this, you [REDACTED_SLUR]", + ), + ("you bhenchod, fuck off", "you [REDACTED_SLUR], [REDACTED_SLUR] off"), + # Hindi slurs aren't known to the profanity model; only the slur list + # catches them. + ("Woh ladki randi hai", "woh ladki [REDACTED_SLUR] hai"), + ], +) +def test_slur_redacted_before_profanity_with_real_model(text, expected_fix): + """Runs against the real curated slur list and the real profanity model.""" + validator = ContentSafety() + + result = validator.validate(text, {}) + + assert isinstance(result, FailResult) + assert PROFANITY_ERROR_MESSAGE not in result.error_message + assert result.fix_value == expected_fix + + +@pytest.mark.parametrize( + "text", + [ + "this is fucking ridiculous", + "you are such an asshole", + "piece of shit", + "go to hell", + "kill yourself", + ], +) +def test_english_profanity_still_blanked_with_real_model(text): + validator = ContentSafety() + + result = validator.validate(text, {}) + + assert isinstance(result, FailResult) + assert result.fix_value == "" + + +def test_profanity_left_after_redaction_blanks_text_with_real_model(): + """'shit' isn't on the slur list, so the profanity check still fails it.""" + validator = ContentSafety() + + result = validator.validate("You are a randi and a piece of shit", {}) + + assert isinstance(result, FailResult) + assert result.error_message.endswith(PROFANITY_ERROR_MESSAGE) + assert result.fix_value == "" + + +@pytest.mark.parametrize( + "text", + [ + "Please share the vaccination schedule for my child", + "mujhe help chahiye", + "मुझे स्कूल की फीस के बारे में जानकारी चाहिए", + "I live in Scunthorpe", + # The English model scores "ho" at 0.883 and "tu" at 0.535; these failed + # under ProfanityFree's 0.5 cutoff. + "namaste, kaise ho", + "aap kaise ho", + "kya ho raha hai", + "tu kahan hai", + ], +) +def test_clean_text_passes_with_real_model(text): + validator = ContentSafety() + + assert isinstance(validator.validate(text, {}), PassResult) + + +@pytest.mark.usefixtures("patch_slur_load", "profanity_score") +def test_guard_fix_returns_redacted_text(): + validator = ContentSafety(on_fail=OnFailAction.FIX) + + outcome = Guard().use(validator).validate("You are a badword") + + assert outcome.validated_output == "you are a [REDACTED_SLUR]" + + +# --------------------------------------- +# ContentSafetySafetyValidatorConfig +# --------------------------------------- +class TestContentSafetySafetyValidatorConfig: + def test_build_default(self): + config = ContentSafetySafetyValidatorConfig(type="content_safety") + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + result = config.build() + + assert result == mock_validator.return_value + _, kwargs = mock_validator.call_args + assert kwargs["slur_severity"] == SlurSeverity.All + assert kwargs["profanity_threshold"] == DEFAULT_PROFANITY_THRESHOLD + + def test_slur_severity_forwarded(self): + config = ContentSafetySafetyValidatorConfig( + type="content_safety", slur_severity="high" + ) + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + config.build() + + _, kwargs = mock_validator.call_args + assert kwargs["slur_severity"] == SlurSeverity.High + + def test_only_expected_kwargs_forwarded(self): + config = ContentSafetySafetyValidatorConfig(type="content_safety") + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + config.build() + + _, kwargs = mock_validator.call_args + assert set(kwargs.keys()) == { + "slur_severity", + "profanity_threshold", + "on_fail", + } + + def test_profanity_threshold_forwarded(self): + config = ContentSafetySafetyValidatorConfig( + type="content_safety", profanity_threshold=0.75 + ) + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + config.build() + + _, kwargs = mock_validator.call_args + assert kwargs["profanity_threshold"] == 0.75 + + @pytest.mark.parametrize("threshold", [-0.1, 1.5]) + def test_out_of_range_profanity_threshold_rejected(self, threshold): + with pytest.raises(ValidationError): + ContentSafetySafetyValidatorConfig( + type="content_safety", profanity_threshold=threshold + ) + + def test_on_fail_fix_resolves_to_callable(self): + config = ContentSafetySafetyValidatorConfig( + type="content_safety", on_fail="fix" + ) + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + config.build() + + _, kwargs = mock_validator.call_args + assert callable(kwargs["on_fail"]) + + def test_on_fail_exception_resolves_to_exception_action(self): + config = ContentSafetySafetyValidatorConfig( + type="content_safety", on_fail="exception" + ) + + with patch(_CONTENT_SAFETY_PATCH) as mock_validator: + config.build() + + _, kwargs = mock_validator.call_args + assert kwargs["on_fail"] == OnFailAction.EXCEPTION + + def test_invalid_slur_severity_rejected(self): + with pytest.raises(ValidationError): + ContentSafetySafetyValidatorConfig( + type="content_safety", slur_severity="extreme" + ) + + def test_languages_not_accepted(self): + """LexicalSlur ignores `languages`, so the wrapper doesn't expose it.""" + with pytest.raises(ValidationError): + ContentSafetySafetyValidatorConfig(type="content_safety", languages=["en"]) + + def test_wrong_type_literal_rejected(self): + with pytest.raises(ValidationError): + ContentSafetySafetyValidatorConfig(type="profanity_free") + + def test_on_fix_sets_reason_when_text_blanked(self): + config = ContentSafetySafetyValidatorConfig( + type="content_safety", on_fail="fix" + ) + fail_result = MagicMock(spec=FailResult) + fail_result.fix_value = "" + + config._on_fix("some input", fail_result) + + assert config._validator_metadata == { + "reason": "Empty string has been returned since the validation failed for: content_safety" + }