From 1da1b6bb60570cddb385a3bf50c1b78d32ec540e Mon Sep 17 00:00:00 2001
From: Prajna1999 <gituprajna20@gmail.com>
Date: Wed, 14 Oct 2026 13:16:22 +0530
Subject: [PATCH 1/4] feat(validators): add content_safety wrapper over
 uli_slur_match and profanity_free

Single validator that runs the lexical slur check, then ProfanityFree on the
redacted text. Slur-first keeps the rest of the message when the only problem
is a listed slur; ProfanityFree has no partial fix and would blank it.

- Config exposes only slur_severity; LexicalSlur ignores `languages`.
- Fixed error messages so user text isn't echoed into validator logs or
  rephrase responses (ProfanityFree's message embeds the full input).
- Inner validators use NOOP on_fail; only the wrapper's on_fail applies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 backend/app/core/enum.py                      |   1 +
 .../content_safety_safety_validator_config.py |  18 +
 backend/app/core/validators/content_safety.py |  56 ++++
 backend/app/schemas/guardrail_config.py       |   4 +
 .../tests/validators/test_content_safety.py   | 307 ++++++++++++++++++
 5 files changed, 386 insertions(+)
 create mode 100644 backend/app/core/validators/config/content_safety_safety_validator_config.py
 create mode 100644 backend/app/core/validators/content_safety.py
 create mode 100644 backend/app/tests/validators/test_content_safety.py

diff --git a/backend/app/core/enum.py b/backend/app/core/enum.py
index fee1a64..9e978fa 100644
--- a/backend/app/core/enum.py
+++ b/backend/app/core/enum.py
@@ -43,3 +43,4 @@ class ValidatorType(Enum):
     ProfanityFree = "profanity_free"
     NSFWText = "nsfw_text"
     AnswerRelevanceCustomLLM = "answer_relevance_custom_llm"
+    ContentSafety = "content_safety"
diff --git a/backend/app/core/validators/config/content_safety_safety_validator_config.py b/backend/app/core/validators/config/content_safety_safety_validator_config.py
new file mode 100644
index 0000000..85656f1
--- /dev/null
+++ b/backend/app/core/validators/config/content_safety_safety_validator_config.py
@@ -0,0 +1,18 @@
+from typing import Literal
+
+from app.core.enum import SlurSeverity
+from app.core.validators.config.base_validator_config import BaseValidatorConfig
+from app.core.validators.content_safety import ContentSafety
+
+
+class ContentSafetySafetyValidatorConfig(BaseValidatorConfig):
+    type: Literal["content_safety"]
+    slur_severity: Literal[
+        "low", "medium", "high", "all"
+    ] = "all"  # minimum slur severity to flag; profanity check has no knobs
+
+    def build(self):
+        return ContentSafety(
+            slur_severity=SlurSeverity(self.slur_severity),
+            on_fail=self.resolve_on_fail(),
+        )
diff --git a/backend/app/core/validators/content_safety.py b/backend/app/core/validators/content_safety.py
new file mode 100644
index 0000000..dba49a3
--- /dev/null
+++ b/backend/app/core/validators/content_safety.py
@@ -0,0 +1,56 @@
+from collections.abc import Callable
+
+from guardrails import OnFailAction
+from guardrails.validators import (
+    FailResult,
+    PassResult,
+    ValidationResult,
+    Validator,
+    register_validator,
+)
+from guardrails_ai.profanity_free import ProfanityFree
+
+from app.core.enum import SlurSeverity
+from app.core.validators.lexical_slur import LexicalSlur
+
+PROFANITY_ERROR_MESSAGE = "Text contains profanity"
+
+
+@register_validator(name="content-safety", data_type="string")
+class ContentSafety(Validator):
+    """
+    Runs lexical slur redaction followed by profanity detection as one validator.
+
+    The slur check runs first so a redacted slur doesn't trip ProfanityFree, which
+    has no partial fix and would otherwise blank the whole text.
+    """
+
+    def __init__(
+        self,
+        slur_severity: SlurSeverity = SlurSeverity.All,
+        on_fail: Callable | OnFailAction | None = OnFailAction.FIX,
+    ):
+        # Inner on_fail is never invoked; only this wrapper's on_fail applies.
+        self._slur = LexicalSlur(severity=slur_severity, on_fail=OnFailAction.NOOP)
+        self._profanity = ProfanityFree(on_fail=OnFailAction.NOOP)
+        super().__init__(on_fail=on_fail, slur_severity=slur_severity)
+
+    def _validate(self, value: str, metadata: dict | None = None) -> ValidationResult:
+        metadata = metadata or {}
+        errors = []
+        current = value
+
+        slur_result = self._slur.validate(current, metadata)
+        if isinstance(slur_result, FailResult):
+            errors.append(slur_result.error_message)
+            current = slur_result.fix_value
+
+        # ProfanityFree's own error_message embeds the full text; use a fixed one.
+        if isinstance(self._profanity.validate(current, metadata), FailResult):
+            errors.append(PROFANITY_ERROR_MESSAGE)
+            current = ""
+
+        if not errors:
+            return PassResult()
+
+        return FailResult(error_message="; ".join(errors), fix_value=current)
diff --git a/backend/app/schemas/guardrail_config.py b/backend/app/schemas/guardrail_config.py
index c31450f..48060fd 100644
--- a/backend/app/schemas/guardrail_config.py
+++ b/backend/app/schemas/guardrail_config.py
@@ -14,6 +14,9 @@
 from app.core.validators.config.ban_list_safety_validator_config import (
     BanListSafetyValidatorConfig,
 )
+from app.core.validators.config.content_safety_safety_validator_config import (
+    ContentSafetySafetyValidatorConfig,
+)
 from app.core.validators.config.gender_assumption_bias_safety_validator_config import (
     GenderAssumptionBiasSafetyValidatorConfig,
 )
@@ -46,6 +49,7 @@
     Union[
         AnswerRelevanceCustomLLMSafetyValidatorConfig,
         BanListSafetyValidatorConfig,
+        ContentSafetySafetyValidatorConfig,
         GenderAssumptionBiasSafetyValidatorConfig,
         LexicalSlurSafetyValidatorConfig,
         LLMCriticSafetyValidatorConfig,
diff --git a/backend/app/tests/validators/test_content_safety.py b/backend/app/tests/validators/test_content_safety.py
new file mode 100644
index 0000000..e093955
--- /dev/null
+++ b/backend/app/tests/validators/test_content_safety.py
@@ -0,0 +1,307 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+from guardrails import Guard, OnFailAction
+from guardrails.validators import FailResult, PassResult
+from pydantic import ValidationError
+
+from app.core.enum import SlurSeverity
+from app.core.on_fail_actions import rephrase_query_on_fail
+from app.core.validators.config.content_safety_safety_validator_config import (
+    ContentSafetySafetyValidatorConfig,
+)
+from app.core.validators.content_safety import (
+    PROFANITY_ERROR_MESSAGE,
+    ContentSafety,
+)
+from app.core.validators.lexical_slur import LexicalSlur
+
+_CONTENT_SAFETY_PATCH = (
+    "app.core.validators.config.content_safety_safety_validator_config.ContentSafety"
+)
+
+
+# ---------------------------------------
+# Fixtures
+# ---------------------------------------
+@pytest.fixture
+def patch_slur_load(monkeypatch):
+    """Use a tiny fixed slur list instead of the curated CSV."""
+
+    def fake_load_slur_list(_self):
+        df = pd.DataFrame({"label": ["badword", "highslur"], "severity": ["L", "H"]})
+        return df["label"].str.lower().tolist()
+
+    monkeypatch.setattr(LexicalSlur, "load_slur_list", fake_load_slur_list)
+
+
+def _set_profanity(validator: ContentSafety, profane: bool) -> MagicMock:
+    """Stub the inner ProfanityFree so tests don't depend on the ML model."""
+    result = (
+        FailResult(error_message="anything contains profanity.", fix_value="")
+        if profane
+        else PassResult()
+    )
+    mock = MagicMock(return_value=result)
+    validator._profanity.validate = mock
+    return mock
+
+
+# ---------------------------------------
+# ContentSafety validator
+# ---------------------------------------
+@pytest.mark.usefixtures("patch_slur_load")
+def test_passes_clean_text():
+    validator = ContentSafety()
+    _set_profanity(validator, profane=False)
+
+    result = validator.validate("This is a perfectly fine sentence", {})
+
+    assert isinstance(result, PassResult)
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_slur_only_is_redacted():
+    validator = ContentSafety()
+    _set_profanity(validator, profane=False)
+
+    result = validator.validate("You are a BADWORD", {})
+
+    assert isinstance(result, FailResult)
+    assert result.error_message == "Mentioned toxic words: badword"
+    assert result.fix_value == "you are a [REDACTED_SLUR]"
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_profanity_only_blanks_text():
+    validator = ContentSafety()
+    _set_profanity(validator, profane=True)
+
+    result = validator.validate("some profane text", {})
+
+    assert isinstance(result, FailResult)
+    assert result.error_message == PROFANITY_ERROR_MESSAGE
+    assert result.fix_value == ""
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_slur_and_profanity_combines_errors():
+    validator = ContentSafety()
+    _set_profanity(validator, profane=True)
+
+    result = validator.validate("you badword, some profane text", {})
+
+    assert isinstance(result, FailResult)
+    assert result.error_message == (
+        f"Mentioned toxic words: badword; {PROFANITY_ERROR_MESSAGE}"
+    )
+    assert result.fix_value == ""
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_profanity_check_runs_on_redacted_text():
+    validator = ContentSafety()
+    profanity_mock = _set_profanity(validator, profane=False)
+
+    validator.validate("You are a badword", {})
+
+    checked_text = profanity_mock.call_args.args[0]
+    assert checked_text == "you are a [REDACTED_SLUR]"
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_error_message_does_not_echo_input():
+    text = "some profane text"
+    validator = ContentSafety()
+    _set_profanity(validator, profane=True)
+
+    result = validator.validate(text, {})
+
+    assert text not in result.error_message
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_slur_severity_forwarded():
+    validator = ContentSafety(slur_severity=SlurSeverity.High)
+
+    assert validator._slur.severity == SlurSeverity.High
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_inner_validators_use_noop_on_fail():
+    validator = ContentSafety()
+
+    assert validator._slur.on_fail_descriptor == OnFailAction.NOOP
+    assert validator._profanity.on_fail_descriptor == OnFailAction.NOOP
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_rephrase_message_does_not_leak_input():
+    """Profanity runs on lowercased/redacted text, so rephrase's str.replace
+    can't scrub ProfanityFree's original message; ours must not contain it."""
+    text = "You badword, some profane text"
+    validator = ContentSafety()
+    _set_profanity(validator, profane=True)
+
+    result = validator.validate(text, {})
+    rephrased = rephrase_query_on_fail(text, result)
+
+    assert "badword" in rephrased  # slur name is part of the reason
+    assert "some profane text" not in rephrased
+
+
+@pytest.mark.parametrize(
+    ("text", "expected_fix"),
+    [
+        # Profanity-first would blank all of these (ProfanityFree's fix_value is
+        # always ""); slur-first redacts the slurs and keeps the rest.
+        ("you are a bitch", "you are a [REDACTED_SLUR]"),
+        (
+            "Stop acting like a stupid cunt",
+            "stop acting like a [REDACTED_SLUR] [REDACTED_SLUR]",
+        ),
+        (
+            "That guy is a faggot, ignore him",
+            "that guy is a [REDACTED_SLUR], ignore him",
+        ),
+        (
+            "What the fuck is this, you bitch",
+            "what the [REDACTED_SLUR] is this, you [REDACTED_SLUR]",
+        ),
+        ("you bhenchod, fuck off", "you [REDACTED_SLUR], [REDACTED_SLUR] off"),
+        # Hindi slurs aren't known to the profanity model; only the slur list
+        # catches them.
+        ("Woh ladki randi hai", "woh ladki [REDACTED_SLUR] hai"),
+    ],
+)
+def test_slur_redacted_before_profanity_with_real_model(text, expected_fix):
+    """Runs against the real curated slur list and the real profanity model."""
+    validator = ContentSafety()
+
+    result = validator.validate(text, {})
+
+    assert isinstance(result, FailResult)
+    assert PROFANITY_ERROR_MESSAGE not in result.error_message
+    assert result.fix_value == expected_fix
+
+
+def test_profanity_left_after_redaction_blanks_text_with_real_model():
+    """'shit' isn't on the slur list, so ProfanityFree still fails the text."""
+    validator = ContentSafety()
+
+    result = validator.validate("You are a randi and a piece of shit", {})
+
+    assert isinstance(result, FailResult)
+    assert result.error_message.endswith(PROFANITY_ERROR_MESSAGE)
+    assert result.fix_value == ""
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "Please share the vaccination schedule for my child",
+        "mujhe help chahiye",
+        "मुझे स्कूल की फीस के बारे में जानकारी चाहिए",
+        "I live in Scunthorpe",
+    ],
+)
+def test_clean_text_passes_with_real_model(text):
+    validator = ContentSafety()
+
+    assert isinstance(validator.validate(text, {}), PassResult)
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_guard_fix_returns_redacted_text():
+    validator = ContentSafety(on_fail=OnFailAction.FIX)
+    _set_profanity(validator, profane=False)
+
+    outcome = Guard().use(validator).validate("You are a badword")
+
+    assert outcome.validated_output == "you are a [REDACTED_SLUR]"
+
+
+# ---------------------------------------
+# ContentSafetySafetyValidatorConfig
+# ---------------------------------------
+class TestContentSafetySafetyValidatorConfig:
+    def test_build_default(self):
+        config = ContentSafetySafetyValidatorConfig(type="content_safety")
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            result = config.build()
+
+        assert result == mock_validator.return_value
+        _, kwargs = mock_validator.call_args
+        assert kwargs["slur_severity"] == SlurSeverity.All
+
+    def test_slur_severity_forwarded(self):
+        config = ContentSafetySafetyValidatorConfig(
+            type="content_safety", slur_severity="high"
+        )
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            config.build()
+
+        _, kwargs = mock_validator.call_args
+        assert kwargs["slur_severity"] == SlurSeverity.High
+
+    def test_only_expected_kwargs_forwarded(self):
+        config = ContentSafetySafetyValidatorConfig(type="content_safety")
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            config.build()
+
+        _, kwargs = mock_validator.call_args
+        assert set(kwargs.keys()) == {"slur_severity", "on_fail"}
+
+    def test_on_fail_fix_resolves_to_callable(self):
+        config = ContentSafetySafetyValidatorConfig(
+            type="content_safety", on_fail="fix"
+        )
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            config.build()
+
+        _, kwargs = mock_validator.call_args
+        assert callable(kwargs["on_fail"])
+
+    def test_on_fail_exception_resolves_to_exception_action(self):
+        config = ContentSafetySafetyValidatorConfig(
+            type="content_safety", on_fail="exception"
+        )
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            config.build()
+
+        _, kwargs = mock_validator.call_args
+        assert kwargs["on_fail"] == OnFailAction.EXCEPTION
+
+    def test_invalid_slur_severity_rejected(self):
+        with pytest.raises(ValidationError):
+            ContentSafetySafetyValidatorConfig(
+                type="content_safety", slur_severity="extreme"
+            )
+
+    def test_languages_not_accepted(self):
+        """LexicalSlur ignores `languages`, so the wrapper doesn't expose it."""
+        with pytest.raises(ValidationError):
+            ContentSafetySafetyValidatorConfig(type="content_safety", languages=["en"])
+
+    def test_wrong_type_literal_rejected(self):
+        with pytest.raises(ValidationError):
+            ContentSafetySafetyValidatorConfig(type="profanity_free")
+
+    def test_on_fix_sets_reason_when_text_blanked(self):
+        config = ContentSafetySafetyValidatorConfig(
+            type="content_safety", on_fail="fix"
+        )
+        fail_result = MagicMock(spec=FailResult)
+        fail_result.fix_value = ""
+
+        config._on_fix("some input", fail_result)
+
+        assert config._validator_metadata == {
+            "reason": "Empty string has been returned since the validation failed for: content_safety"
+        }

From 86106ed02912298dc075c3a4a30de2c274a25712 Mon Sep 17 00:00:00 2001
From: Prajna1999 <gituprajna20@gmail.com>
Date: Wed, 14 Oct 2026 21:34:24 +0530
Subject: [PATCH 2/4] feat(validators): make content_safety profanity cutoff
 configurable

ProfanityFree fails anything the alt-profanity-check model scores at 0.5 or
more. The model is English-only and weights "ho" (as in "kaise ho") at 0.883
and "tu" at 0.535, so common Hinglish was blanked.

Score with predict_prob directly and fail at profanity_threshold (default
0.9, range 0-1). English profanity still scores 0.95-1.0.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 .../content_safety_safety_validator_config.py |  12 +-
 backend/app/core/validators/content_safety.py |  24 +++-
 .../tests/validators/test_content_safety.py   | 135 +++++++++++++-----
 3 files changed, 127 insertions(+), 44 deletions(-)

diff --git a/backend/app/core/validators/config/content_safety_safety_validator_config.py b/backend/app/core/validators/config/content_safety_safety_validator_config.py
index 85656f1..11256f3 100644
--- a/backend/app/core/validators/config/content_safety_safety_validator_config.py
+++ b/backend/app/core/validators/config/content_safety_safety_validator_config.py
@@ -1,18 +1,26 @@
 from typing import Literal
 
+from pydantic import Field
+
 from app.core.enum import SlurSeverity
 from app.core.validators.config.base_validator_config import BaseValidatorConfig
-from app.core.validators.content_safety import ContentSafety
+from app.core.validators.content_safety import (
+    DEFAULT_PROFANITY_THRESHOLD,
+    ContentSafety,
+)
 
 
 class ContentSafetySafetyValidatorConfig(BaseValidatorConfig):
     type: Literal["content_safety"]
     slur_severity: Literal[
         "low", "medium", "high", "all"
-    ] = "all"  # minimum slur severity to flag; profanity check has no knobs
+    ] = "all"  # minimum slur severity to flag
+    # Profanity probability at or above which the text fails and is blanked.
+    profanity_threshold: float = Field(default=DEFAULT_PROFANITY_THRESHOLD, ge=0, le=1)
 
     def build(self):
         return ContentSafety(
             slur_severity=SlurSeverity(self.slur_severity),
+            profanity_threshold=self.profanity_threshold,
             on_fail=self.resolve_on_fail(),
         )
diff --git a/backend/app/core/validators/content_safety.py b/backend/app/core/validators/content_safety.py
index dba49a3..d35a1ff 100644
--- a/backend/app/core/validators/content_safety.py
+++ b/backend/app/core/validators/content_safety.py
@@ -8,12 +8,15 @@
     Validator,
     register_validator,
 )
-from guardrails_ai.profanity_free import ProfanityFree
+from profanity_check import predict_prob
 
 from app.core.enum import SlurSeverity
 from app.core.validators.lexical_slur import LexicalSlur
 
 PROFANITY_ERROR_MESSAGE = "Text contains profanity"
+# ProfanityFree uses the model's fixed 0.5 cutoff, which fails common Hinglish:
+# the English model weights "ho" (as in "kaise ho") at 0.883 and "tu" at 0.535.
+DEFAULT_PROFANITY_THRESHOLD = 0.9
 
 
 @register_validator(name="content-safety", data_type="string")
@@ -21,19 +24,26 @@ class ContentSafety(Validator):
     """
     Runs lexical slur redaction followed by profanity detection as one validator.
 
-    The slur check runs first so a redacted slur doesn't trip ProfanityFree, which
-    has no partial fix and would otherwise blank the whole text.
+    The slur check runs first so a redacted slur doesn't trip the profanity check,
+    which has no partial fix and would otherwise blank the whole text. Profanity is
+    scored with alt-profanity-check (the model behind ProfanityFree) against a
+    configurable threshold instead of ProfanityFree's fixed 0.5.
     """
 
     def __init__(
         self,
         slur_severity: SlurSeverity = SlurSeverity.All,
+        profanity_threshold: float = DEFAULT_PROFANITY_THRESHOLD,
         on_fail: Callable | OnFailAction | None = OnFailAction.FIX,
     ):
         # Inner on_fail is never invoked; only this wrapper's on_fail applies.
         self._slur = LexicalSlur(severity=slur_severity, on_fail=OnFailAction.NOOP)
-        self._profanity = ProfanityFree(on_fail=OnFailAction.NOOP)
-        super().__init__(on_fail=on_fail, slur_severity=slur_severity)
+        self.profanity_threshold = profanity_threshold
+        super().__init__(
+            on_fail=on_fail,
+            slur_severity=slur_severity,
+            profanity_threshold=profanity_threshold,
+        )
 
     def _validate(self, value: str, metadata: dict | None = None) -> ValidationResult:
         metadata = metadata or {}
@@ -45,8 +55,8 @@ def _validate(self, value: str, metadata: dict | None = None) -> ValidationResul
             errors.append(slur_result.error_message)
             current = slur_result.fix_value
 
-        # ProfanityFree's own error_message embeds the full text; use a fixed one.
-        if isinstance(self._profanity.validate(current, metadata), FailResult):
+        # Fixed message: never echo user text into logs or rephrase responses.
+        if predict_prob([current])[0] >= self.profanity_threshold:
             errors.append(PROFANITY_ERROR_MESSAGE)
             current = ""
 
diff --git a/backend/app/tests/validators/test_content_safety.py b/backend/app/tests/validators/test_content_safety.py
index e093955..6911753 100644
--- a/backend/app/tests/validators/test_content_safety.py
+++ b/backend/app/tests/validators/test_content_safety.py
@@ -12,11 +12,13 @@
     ContentSafetySafetyValidatorConfig,
 )
 from app.core.validators.content_safety import (
+    DEFAULT_PROFANITY_THRESHOLD,
     PROFANITY_ERROR_MESSAGE,
     ContentSafety,
 )
 from app.core.validators.lexical_slur import LexicalSlur
 
+_PREDICT_PROB_PATCH = "app.core.validators.content_safety.predict_prob"
 _CONTENT_SAFETY_PATCH = (
     "app.core.validators.config.content_safety_safety_validator_config.ContentSafety"
 )
@@ -36,35 +38,33 @@ def fake_load_slur_list(_self):
     monkeypatch.setattr(LexicalSlur, "load_slur_list", fake_load_slur_list)
 
 
-def _set_profanity(validator: ContentSafety, profane: bool) -> MagicMock:
-    """Stub the inner ProfanityFree so tests don't depend on the ML model."""
-    result = (
-        FailResult(error_message="anything contains profanity.", fix_value="")
-        if profane
-        else PassResult()
-    )
-    mock = MagicMock(return_value=result)
-    validator._profanity.validate = mock
-    return mock
+@pytest.fixture
+def profanity_score():
+    """Stub the profanity model so tests don't depend on it. Set
+    `.return_value = [score]`; defaults to a clean score."""
+    with patch(_PREDICT_PROB_PATCH) as mock:
+        mock.return_value = [0.0]
+        yield mock
+
+
+PROFANE = [1.0]
 
 
 # ---------------------------------------
 # ContentSafety validator
 # ---------------------------------------
-@pytest.mark.usefixtures("patch_slur_load")
+@pytest.mark.usefixtures("patch_slur_load", "profanity_score")
 def test_passes_clean_text():
     validator = ContentSafety()
-    _set_profanity(validator, profane=False)
 
     result = validator.validate("This is a perfectly fine sentence", {})
 
     assert isinstance(result, PassResult)
 
 
-@pytest.mark.usefixtures("patch_slur_load")
+@pytest.mark.usefixtures("patch_slur_load", "profanity_score")
 def test_slur_only_is_redacted():
     validator = ContentSafety()
-    _set_profanity(validator, profane=False)
 
     result = validator.validate("You are a BADWORD", {})
 
@@ -74,9 +74,9 @@ def test_slur_only_is_redacted():
 
 
 @pytest.mark.usefixtures("patch_slur_load")
-def test_profanity_only_blanks_text():
+def test_profanity_only_blanks_text(profanity_score):
     validator = ContentSafety()
-    _set_profanity(validator, profane=True)
+    profanity_score.return_value = PROFANE
 
     result = validator.validate("some profane text", {})
 
@@ -86,9 +86,9 @@ def test_profanity_only_blanks_text():
 
 
 @pytest.mark.usefixtures("patch_slur_load")
-def test_slur_and_profanity_combines_errors():
+def test_slur_and_profanity_combines_errors(profanity_score):
     validator = ContentSafety()
-    _set_profanity(validator, profane=True)
+    profanity_score.return_value = PROFANE
 
     result = validator.validate("you badword, some profane text", {})
 
@@ -100,21 +100,20 @@ def test_slur_and_profanity_combines_errors():
 
 
 @pytest.mark.usefixtures("patch_slur_load")
-def test_profanity_check_runs_on_redacted_text():
+def test_profanity_check_runs_on_redacted_text(profanity_score):
     validator = ContentSafety()
-    profanity_mock = _set_profanity(validator, profane=False)
 
     validator.validate("You are a badword", {})
 
-    checked_text = profanity_mock.call_args.args[0]
-    assert checked_text == "you are a [REDACTED_SLUR]"
+    checked_texts = profanity_score.call_args.args[0]
+    assert checked_texts == ["you are a [REDACTED_SLUR]"]
 
 
 @pytest.mark.usefixtures("patch_slur_load")
-def test_error_message_does_not_echo_input():
+def test_error_message_does_not_echo_input(profanity_score):
     text = "some profane text"
     validator = ContentSafety()
-    _set_profanity(validator, profane=True)
+    profanity_score.return_value = PROFANE
 
     result = validator.validate(text, {})
 
@@ -129,20 +128,39 @@ def test_slur_severity_forwarded():
 
 
 @pytest.mark.usefixtures("patch_slur_load")
-def test_inner_validators_use_noop_on_fail():
+def test_inner_slur_validator_uses_noop_on_fail():
     validator = ContentSafety()
 
     assert validator._slur.on_fail_descriptor == OnFailAction.NOOP
-    assert validator._profanity.on_fail_descriptor == OnFailAction.NOOP
 
 
+@pytest.mark.parametrize(
+    ("score", "threshold", "fails"),
+    [
+        (0.883, DEFAULT_PROFANITY_THRESHOLD, False),  # "kaise ho" with default
+        (0.9, DEFAULT_PROFANITY_THRESHOLD, True),  # threshold is inclusive
+        (0.95, DEFAULT_PROFANITY_THRESHOLD, True),
+        (0.535, 0.5, True),  # ProfanityFree's old cutoff
+        (0.95, 0.99, False),
+    ],
+)
 @pytest.mark.usefixtures("patch_slur_load")
-def test_rephrase_message_does_not_leak_input():
+def test_profanity_threshold(profanity_score, score, threshold, fails):
+    validator = ContentSafety(profanity_threshold=threshold)
+    profanity_score.return_value = [score]
+
+    result = validator.validate("some text", {})
+
+    assert isinstance(result, FailResult) is fails
+
+
+@pytest.mark.usefixtures("patch_slur_load")
+def test_rephrase_message_does_not_leak_input(profanity_score):
     """Profanity runs on lowercased/redacted text, so rephrase's str.replace
-    can't scrub ProfanityFree's original message; ours must not contain it."""
+    can't scrub a message that embeds it; ours must not contain user text."""
     text = "You badword, some profane text"
     validator = ContentSafety()
-    _set_profanity(validator, profane=True)
+    profanity_score.return_value = PROFANE
 
     result = validator.validate(text, {})
     rephrased = rephrase_query_on_fail(text, result)
@@ -154,8 +172,8 @@ def test_rephrase_message_does_not_leak_input():
 @pytest.mark.parametrize(
     ("text", "expected_fix"),
     [
-        # Profanity-first would blank all of these (ProfanityFree's fix_value is
-        # always ""); slur-first redacts the slurs and keeps the rest.
+        # Profanity-first would blank all of these (the profanity check has no
+        # partial fix); slur-first redacts the slurs and keeps the rest.
         ("you are a bitch", "you are a [REDACTED_SLUR]"),
         (
             "Stop acting like a stupid cunt",
@@ -186,8 +204,27 @@ def test_slur_redacted_before_profanity_with_real_model(text, expected_fix):
     assert result.fix_value == expected_fix
 
 
+@pytest.mark.parametrize(
+    "text",
+    [
+        "this is fucking ridiculous",
+        "you are such an asshole",
+        "piece of shit",
+        "go to hell",
+        "kill yourself",
+    ],
+)
+def test_english_profanity_still_blanked_with_real_model(text):
+    validator = ContentSafety()
+
+    result = validator.validate(text, {})
+
+    assert isinstance(result, FailResult)
+    assert result.fix_value == ""
+
+
 def test_profanity_left_after_redaction_blanks_text_with_real_model():
-    """'shit' isn't on the slur list, so ProfanityFree still fails the text."""
+    """'shit' isn't on the slur list, so the profanity check still fails it."""
     validator = ContentSafety()
 
     result = validator.validate("You are a randi and a piece of shit", {})
@@ -204,6 +241,12 @@ def test_profanity_left_after_redaction_blanks_text_with_real_model():
         "mujhe help chahiye",
         "मुझे स्कूल की फीस के बारे में जानकारी चाहिए",
         "I live in Scunthorpe",
+        # The English model scores "ho" at 0.883 and "tu" at 0.535; these failed
+        # under ProfanityFree's 0.5 cutoff.
+        "namaste, kaise ho",
+        "aap kaise ho",
+        "kya ho raha hai",
+        "tu kahan hai",
     ],
 )
 def test_clean_text_passes_with_real_model(text):
@@ -212,10 +255,9 @@ def test_clean_text_passes_with_real_model(text):
     assert isinstance(validator.validate(text, {}), PassResult)
 
 
-@pytest.mark.usefixtures("patch_slur_load")
+@pytest.mark.usefixtures("patch_slur_load", "profanity_score")
 def test_guard_fix_returns_redacted_text():
     validator = ContentSafety(on_fail=OnFailAction.FIX)
-    _set_profanity(validator, profane=False)
 
     outcome = Guard().use(validator).validate("You are a badword")
 
@@ -235,6 +277,7 @@ def test_build_default(self):
         assert result == mock_validator.return_value
         _, kwargs = mock_validator.call_args
         assert kwargs["slur_severity"] == SlurSeverity.All
+        assert kwargs["profanity_threshold"] == DEFAULT_PROFANITY_THRESHOLD
 
     def test_slur_severity_forwarded(self):
         config = ContentSafetySafetyValidatorConfig(
@@ -254,7 +297,29 @@ def test_only_expected_kwargs_forwarded(self):
             config.build()
 
         _, kwargs = mock_validator.call_args
-        assert set(kwargs.keys()) == {"slur_severity", "on_fail"}
+        assert set(kwargs.keys()) == {
+            "slur_severity",
+            "profanity_threshold",
+            "on_fail",
+        }
+
+    def test_profanity_threshold_forwarded(self):
+        config = ContentSafetySafetyValidatorConfig(
+            type="content_safety", profanity_threshold=0.75
+        )
+
+        with patch(_CONTENT_SAFETY_PATCH) as mock_validator:
+            config.build()
+
+        _, kwargs = mock_validator.call_args
+        assert kwargs["profanity_threshold"] == 0.75
+
+    @pytest.mark.parametrize("threshold", [-0.1, 1.5])
+    def test_out_of_range_profanity_threshold_rejected(self, threshold):
+        with pytest.raises(ValidationError):
+            ContentSafetySafetyValidatorConfig(
+                type="content_safety", profanity_threshold=threshold
+            )
 
     def test_on_fail_fix_resolves_to_callable(self):
         config = ContentSafetySafetyValidatorConfig(

From 6590e47cc242d97bdbb1dac33333e182f5b31ff2 Mon Sep 17 00:00:00 2001
From: Prajna1999 <gituprajna20@gmail.com>
Date: Thu, 15 Oct 2026 09:02:53 +0530
Subject: [PATCH 3/4] docs(validators): register content_safety in manifest and
 docs

Add content_safety to validators.json (source: local) and document it in the
validators README (supported list, details, stage guidance, related files)
and API_USAGE validator types.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 backend/app/api/API_USAGE.md                |  1 +
 backend/app/core/validators/README.md       | 60 ++++++++++++++++++++-
 backend/app/core/validators/validators.json |  5 ++
 3 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/backend/app/api/API_USAGE.md b/backend/app/api/API_USAGE.md
index e04d003..f8542d4 100644
--- a/backend/app/api/API_USAGE.md
+++ b/backend/app/api/API_USAGE.md
@@ -522,6 +522,7 @@ From `validators.json`:
 - `profanity_free`
 - `nsfw_text`
 - `answer_relevance_custom_llm`
+- `content_safety`
 
 Source of truth:
 - `backend/app/core/validators/validators.json`
diff --git a/backend/app/core/validators/README.md b/backend/app/core/validators/README.md
index b8fc87e..39acc58 100644
--- a/backend/app/core/validators/README.md
+++ b/backend/app/core/validators/README.md
@@ -16,6 +16,7 @@ Current validator manifest:
 - `profanity_free` (source: `hub://guardrails/profanity_free`)
 - `nsfw_text` (source: `hub://guardrails/nsfw_text`)
 - `answer_relevance_custom_llm` (source: `local`)
+- `content_safety` (source: `local`) - wraps `uli_slur_match` and the `profanity_free` model
 
 ## Configuration Model
 
@@ -559,6 +560,60 @@ Notes / limitations:
 - `on_fail=fix` has no programmatic fix for irrelevant answers — `safe_text` will be `""` and `metadata.reason` will identify this validator.
 - If `custom_prompt_id` is deleted after being referenced, the guardrail will return a 404 at resolution time.
 
+### 11) Content Safety Validator (`content_safety`)
+
+Code:
+
+- Config: `backend/app/core/validators/config/content_safety_safety_validator_config.py`
+- Runtime validator: `backend/app/core/validators/content_safety.py`
+
+What it does:
+
+- Runs the lexical slur check (`uli_slur_match`) and a profanity check as a single validator, so one config replaces stacking both.
+- Step 1: redacts slurs from the curated list with `[REDACTED_SLUR]`.
+- Step 2: scores the redacted text with `alt-profanity-check` (the model behind `profanity_free`) and fails it if the score is at or above `profanity_threshold`.
+- Returns one pass/fail with both errors joined, e.g. `Mentioned toxic words: randi; Text contains profanity`.
+
+Why this is used:
+
+- One validator config for content safety instead of separate `uli_slur_match` and `profanity_free` configs.
+- Slur-first ordering keeps the message when a listed slur is the only problem: `"That guy is a faggot, ignore him"` becomes `"that guy is a [REDACTED_SLUR], ignore him"`, whereas `profanity_free` alone would return `""`.
+- A configurable profanity threshold avoids `profanity_free`'s Hinglish false positives (see limitations).
+
+Recommendation:
+
+- `input` and `output`
+  - Why `input`: catches slurs and profanity in user messages.
+  - Why `output`: prevents model-generated slurs or profanity from reaching users.
+
+Parameters / customization:
+
+- `slur_severity: "low" | "medium" | "high" | "all"` (default: `all`) — minimum slur severity to redact; same meaning as `severity` on `uli_slur_match`
+- `profanity_threshold: float` (0–1, default: `0.9`) — profanity probability at or above which the text fails
+- `on_fail`
+
+Example stored config:
+
+```json
+{
+  "name": "content safety input",
+  "type": "content_safety",
+  "stage": "input",
+  "on_fail_action": "fix",
+  "slur_severity": "all",
+  "profanity_threshold": 0.9
+}
+```
+
+Notes / limitations:
+
+- **Why 0.9, not `profanity_free`'s 0.5:** the profanity model is English-only. It reads `ho` as the slur "hoe" (score 0.883) and `tu` as 0.535, so at 0.5 common Hinglish such as `namaste, kaise ho`, `kya ho raha hai` and `tu kahan hai` is blanked. English profanity scores 0.95–1.0, so it is still caught at 0.9. Trade-off: `screw you` (0.893) passes. Lower the threshold for stricter English filtering.
+- Hindi slurs are caught only by the slur list; the profanity model doesn't know them.
+- If profanity remains after slur redaction, there is no programmatic fix: with `on_fail=fix`, `safe_text` is `""` and `metadata.reason` identifies this validator.
+- Slur redaction lowercases the whole text and strips emoji (inherited from `uli_slur_match`).
+- `languages` is not accepted: `uli_slur_match` stores it but never uses it.
+- Error messages never include the user's text, so they are safe to show with `on_fail=rephrase` and to store in validator logs.
+
 ## Example Config Payloads
 
 Example: create validator config (stored shape)
@@ -589,6 +644,7 @@ Default stage strategy:
 
 - Input guardrails: `pii_remover`, `uli_slur_match`, `ban_list`, `topic_relevance` (when scope enforcement is needed), `profanity_free`, `llamaguard_7b`
 - Output guardrails: `pii_remover`, `uli_slur_match`, `gender_assumption_bias`, `ban_list`, `profanity_free`, `llamaguard_7b`, `answer_relevance_custom_llm` (when answer quality must be verified)
+- `content_safety` can replace `uli_slur_match` + `profanity_free` at either stage; don't configure it alongside them.
 
 Tuning strategy:
 
@@ -626,7 +682,7 @@ here don't even accept it. `BanList`'s `__init__` has no `**kwargs` at all and r
 `has_guardrails_endpoint=True`, so it's a no-op either way. Check the constructor and the
 decorator before adding the kwarg to a new validator's `build()`. The fully custom in-repo
 validators (`pii_remover`, `gender_assumption_bias`, `uli_slur_match`, `topic_relevance`,
-`topic_relevance_llm`, `answer_relevance_custom_llm`) have no hub/remote code path at all and
+`topic_relevance_llm`, `answer_relevance_custom_llm`, `content_safety`) have no hub/remote code path at all and
 don't need it.
 
 ## Related Files
@@ -643,6 +699,8 @@ don't need it.
 - `backend/app/core/validators/config/profanity_free_safety_validator_config.py`
 - `backend/app/core/validators/config/answer_relevance_custom_llm_safety_validator_config.py`
 - `backend/app/core/validators/answer_relevance_custom_llm.py`
+- `backend/app/core/validators/config/content_safety_safety_validator_config.py`
+- `backend/app/core/validators/content_safety.py`
 - `backend/app/models/config/llm_prompt_config.py`
 - `backend/app/crud/llm_prompt_config.py`
 - `backend/app/schemas/guardrail_config.py`
diff --git a/backend/app/core/validators/validators.json b/backend/app/core/validators/validators.json
index 6e08bb7..6c13f41 100644
--- a/backend/app/core/validators/validators.json
+++ b/backend/app/core/validators/validators.json
@@ -49,6 +49,11 @@
             "type": "nsfw_text",
             "version": "0.1.0",
             "source": "pypi://guardrails-ai-nsfw-text"
+        },
+        {
+            "type": "content_safety",
+            "version": "0.1.0",
+            "source": "local"
         }
     ]
 }
\ No newline at end of file

From f6d37b678bb06412ea378ae7ca2d7aea83a6607d Mon Sep 17 00:00:00 2001
From: Prajna1999 <gituprajna20@gmail.com>
Date: Thu, 15 Oct 2026 09:04:19 +0530
Subject: [PATCH 4/4] docs(validators): add answer_relevance_custom_llm to
 validators.json

It was the only ValidatorType missing from the manifest.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 backend/app/core/validators/validators.json | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/backend/app/core/validators/validators.json b/backend/app/core/validators/validators.json
index 6c13f41..a1bbbb9 100644
--- a/backend/app/core/validators/validators.json
+++ b/backend/app/core/validators/validators.json
@@ -50,6 +50,11 @@
             "version": "0.1.0",
             "source": "pypi://guardrails-ai-nsfw-text"
         },
+        {
+            "type": "answer_relevance_custom_llm",
+            "version": "0.1.0",
+            "source": "local"
+        },
         {
             "type": "content_safety",
             "version": "0.1.0",