feat: OHE min_frequency knob to collapse rare categories

docs/guide.md

@@ -60,7 +60,7 @@ Type inference classifies every feature as one of:
 | Inferred type | Condition | Pipeline |
 |---|---|---|
 | `numeric` | number-like with `nunique > numeric_threshold` | imputer (`median` default, or `mean`/`iterative`) with missingness indicator + scaler (`standard`/`minmax`/`robust`) |
-| `categorical_low` | not numeric, `nunique <= cardinality_threshold` | most-frequent imputer + `OneHotEncoder(drop="if_binary")` |
+| `categorical_low` | not numeric, `nunique <= cardinality_threshold` | most-frequent imputer + `OneHotEncoder(drop="if_binary", min_frequency=5)` |
 | `categorical_high` | not numeric, `nunique > cardinality_threshold` | most-frequent imputer + `TargetEncoder` (ordinal for multioutput targets) |
 | `datetime` | datetime dtype | `DatetimeEncoder` + most-frequent imputer |
 

poniard/preprocessing/core.py

@@ -128,6 +128,10 @@ class PoniardPreprocessor:
     cyclical_datetime :
         Whether to encode periodic datetime levels (hour, month, weekday, ...) as sin/cos
         pairs instead of plain integers.
+    ohe_min_frequency :
+        Minimum frequency for categories kept as separate one-hot columns. Categories rarer
+        than this collapse into sklearn's infrequent bucket. An int is an absolute count; a
+        float is a fraction of samples. ``None`` keeps every observed category.
     numeric_threshold :
         Number features with unique values above a certain threshold will be treated as numeric. If
         float, the threshold is `numeric_threshold * samples`.
@@ -160,6 +164,7 @@ def __init__(
         numeric_imputer: Literal["iterative", "mean", "median"] | TransformerMixin | None = None,
         categorical_imputer: Literal["most_frequent", "constant"] | TransformerMixin | None = None,
         cyclical_datetime: bool = False,
+        ohe_min_frequency: int | float | None = 5,
         numeric_threshold: int | float = 0.1,
         cardinality_threshold: int | float = 20,
         verbose: bool = False,
@@ -175,6 +180,7 @@ def __init__(
             "numeric_imputer": numeric_imputer,
             "categorical_imputer": categorical_imputer,
             "cyclical_datetime": cyclical_datetime,
+            "ohe_min_frequency": ohe_min_frequency,
             "numeric_threshold": numeric_threshold,
             "cardinality_threshold": cardinality_threshold,
             "verbose": verbose,
@@ -189,6 +195,7 @@ def __init__(
         self.numeric_imputer = numeric_imputer or "median"
         self.categorical_imputer = categorical_imputer or "most_frequent"
         self.cyclical_datetime = cyclical_datetime
+        self.ohe_min_frequency = ohe_min_frequency
         self.numeric_threshold = numeric_threshold
         self.cardinality_threshold = cardinality_threshold
         self.verbose = verbose
@@ -384,7 +391,12 @@ def _setup_transformers(self):
                 ("categorical_imputer", cat_imputer),
                 (
                     "one-hot_encoder",
-                    OneHotEncoder(drop="if_binary", handle_unknown="ignore", sparse_output=False),
+                    OneHotEncoder(
+                        drop="if_binary",
+                        handle_unknown="ignore",
+                        sparse_output=False,
+                        min_frequency=self.ohe_min_frequency,
+                    ),
                 ),
             ]
         )

tests/test_preprocessing.py

@@ -158,8 +158,10 @@ def test_feature_names_stable_across_type_compositions():
         }
     )
     y = np.array([0, 1, 0, 1, 0])
-    pp_full = PoniardPreprocessor().build(X=base, y=y, task="classification")
-    pp_numeric = PoniardPreprocessor().build(X=base[["A"]], y=y, task="classification")
+    pp_full = PoniardPreprocessor(ohe_min_frequency=None).build(X=base, y=y, task="classification")
+    pp_numeric = PoniardPreprocessor(ohe_min_frequency=None).build(
+        X=base[["A"]], y=y, task="classification"
+    )
     pp_full.preprocessor.fit(base, y)
     pp_numeric.preprocessor.fit(base[["A"]], y)
     full_names = pp_full.preprocessor.get_feature_names_out()
@@ -268,10 +270,48 @@ def test_inference_parity_dataframe_vs_ndarray(array, frame):
     assert from_array == from_frame
 
 
+def test_ohe_min_frequency_collapses_rare_categories():
+    X = pd.DataFrame(
+        {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]}
+    )
+    y = np.array([0, 1] * 32)
+    pp = PoniardPreprocessor(ohe_min_frequency=5).build(X=X, y=y, task="classification")
+    out = pp.preprocessor.fit_transform(X, y)
+    assert "cat_common_a" in out.columns
+    assert "cat_common_b" in out.columns
+    assert "cat_infrequent_sklearn" in out.columns
+    assert not any(c.startswith("cat_rare") for c in out.columns)
+
+
+def test_ohe_min_frequency_none_preserves_all_categories():
+    X = pd.DataFrame(
+        {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]}
+    )
+    y = np.array([0, 1] * 32)
+    pp = PoniardPreprocessor(ohe_min_frequency=None).build(X=X, y=y, task="classification")
+    out = pp.preprocessor.fit_transform(X, y)
+    assert "cat_common_a" in out.columns
+    assert any(c.startswith("cat_rare") for c in out.columns)
+    assert "cat_infrequent_sklearn" not in out.columns
+
+
+def test_ohe_min_frequency_float_is_fraction_of_samples():
+    X = pd.DataFrame(
+        {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]}
+    )
+    y = np.array([0, 1] * 32)
+    pp = PoniardPreprocessor(ohe_min_frequency=0.1).build(X=X, y=y, task="classification")
+    out = pp.preprocessor.fit_transform(X, y)
+    assert "cat_common_a" in out.columns
+    assert "cat_infrequent_sklearn" in out.columns
+
+
 def test_categorical_imputer_constant_creates_missing_category():
     X = pd.DataFrame({"cat": ["a", "b", np.nan, "a", "b"]})
     y = np.array([0, 1, 0, 1, 0])
-    pp = PoniardPreprocessor(categorical_imputer="constant").build(X=X, y=y, task="classification")
+    pp = PoniardPreprocessor(categorical_imputer="constant", ohe_min_frequency=None).build(
+        X=X, y=y, task="classification"
+    )
     out = pp.preprocessor.fit_transform(X, y)
     assert "cat_missing" in out.columns