From ba77b4557f6521aef0cf4b03bf575d44a3237667 Mon Sep 17 00:00:00 2001 From: Rafael Xavier Date: Mon, 3 Aug 2026 00:53:56 +0000 Subject: [PATCH] feat: OHE min_frequency knob to collapse rare categories - Add ohe_min_frequency (default 5) to PoniardPreprocessor; int is an absolute count, float is a fraction of samples, None keeps every category - Rare categories collapse into sklearn's infrequent bucket, composing with the existing handle_unknown='ignore' - Disable the knob in the two existing tests that assert exact OHE output on 5-row toy data; document the default in the preprocessing guide Closes P1-5 of the ML defaults improvements plan. --- docs/guide.md | 2 +- poniard/preprocessing/core.py | 14 ++++++++++- tests/test_preprocessing.py | 46 ++++++++++++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/guide.md b/docs/guide.md index c24a095..a2857ad 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -60,7 +60,7 @@ Type inference classifies every feature as one of: | Inferred type | Condition | Pipeline | |---|---|---| | `numeric` | number-like with `nunique > numeric_threshold` | imputer (`median` default, or `mean`/`iterative`) with missingness indicator + scaler (`standard`/`minmax`/`robust`) | -| `categorical_low` | not numeric, `nunique <= cardinality_threshold` | most-frequent imputer + `OneHotEncoder(drop="if_binary")` | +| `categorical_low` | not numeric, `nunique <= cardinality_threshold` | most-frequent imputer + `OneHotEncoder(drop="if_binary", min_frequency=5)` | | `categorical_high` | not numeric, `nunique > cardinality_threshold` | most-frequent imputer + `TargetEncoder` (ordinal for multioutput targets) | | `datetime` | datetime dtype | `DatetimeEncoder` + most-frequent imputer | diff --git a/poniard/preprocessing/core.py b/poniard/preprocessing/core.py index 4b28201..738a6cf 100644 --- a/poniard/preprocessing/core.py +++ b/poniard/preprocessing/core.py @@ -128,6 +128,10 @@ class PoniardPreprocessor: cyclical_datetime : Whether to encode periodic datetime levels (hour, month, weekday, ...) as sin/cos pairs instead of plain integers. + ohe_min_frequency : + Minimum frequency for categories kept as separate one-hot columns. Categories rarer + than this collapse into sklearn's infrequent bucket. An int is an absolute count; a + float is a fraction of samples. ``None`` keeps every observed category. numeric_threshold : Number features with unique values above a certain threshold will be treated as numeric. If float, the threshold is `numeric_threshold * samples`. @@ -160,6 +164,7 @@ def __init__( numeric_imputer: Literal["iterative", "mean", "median"] | TransformerMixin | None = None, categorical_imputer: Literal["most_frequent", "constant"] | TransformerMixin | None = None, cyclical_datetime: bool = False, + ohe_min_frequency: int | float | None = 5, numeric_threshold: int | float = 0.1, cardinality_threshold: int | float = 20, verbose: bool = False, @@ -175,6 +180,7 @@ def __init__( "numeric_imputer": numeric_imputer, "categorical_imputer": categorical_imputer, "cyclical_datetime": cyclical_datetime, + "ohe_min_frequency": ohe_min_frequency, "numeric_threshold": numeric_threshold, "cardinality_threshold": cardinality_threshold, "verbose": verbose, @@ -189,6 +195,7 @@ def __init__( self.numeric_imputer = numeric_imputer or "median" self.categorical_imputer = categorical_imputer or "most_frequent" self.cyclical_datetime = cyclical_datetime + self.ohe_min_frequency = ohe_min_frequency self.numeric_threshold = numeric_threshold self.cardinality_threshold = cardinality_threshold self.verbose = verbose @@ -384,7 +391,12 @@ def _setup_transformers(self): ("categorical_imputer", cat_imputer), ( "one-hot_encoder", - OneHotEncoder(drop="if_binary", handle_unknown="ignore", sparse_output=False), + OneHotEncoder( + drop="if_binary", + handle_unknown="ignore", + sparse_output=False, + min_frequency=self.ohe_min_frequency, + ), ), ] ) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 7eab294..bbafb37 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -158,8 +158,10 @@ def test_feature_names_stable_across_type_compositions(): } ) y = np.array([0, 1, 0, 1, 0]) - pp_full = PoniardPreprocessor().build(X=base, y=y, task="classification") - pp_numeric = PoniardPreprocessor().build(X=base[["A"]], y=y, task="classification") + pp_full = PoniardPreprocessor(ohe_min_frequency=None).build(X=base, y=y, task="classification") + pp_numeric = PoniardPreprocessor(ohe_min_frequency=None).build( + X=base[["A"]], y=y, task="classification" + ) pp_full.preprocessor.fit(base, y) pp_numeric.preprocessor.fit(base[["A"]], y) full_names = pp_full.preprocessor.get_feature_names_out() @@ -268,10 +270,48 @@ def test_inference_parity_dataframe_vs_ndarray(array, frame): assert from_array == from_frame +def test_ohe_min_frequency_collapses_rare_categories(): + X = pd.DataFrame( + {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]} + ) + y = np.array([0, 1] * 32) + pp = PoniardPreprocessor(ohe_min_frequency=5).build(X=X, y=y, task="classification") + out = pp.preprocessor.fit_transform(X, y) + assert "cat_common_a" in out.columns + assert "cat_common_b" in out.columns + assert "cat_infrequent_sklearn" in out.columns + assert not any(c.startswith("cat_rare") for c in out.columns) + + +def test_ohe_min_frequency_none_preserves_all_categories(): + X = pd.DataFrame( + {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]} + ) + y = np.array([0, 1] * 32) + pp = PoniardPreprocessor(ohe_min_frequency=None).build(X=X, y=y, task="classification") + out = pp.preprocessor.fit_transform(X, y) + assert "cat_common_a" in out.columns + assert any(c.startswith("cat_rare") for c in out.columns) + assert "cat_infrequent_sklearn" not in out.columns + + +def test_ohe_min_frequency_float_is_fraction_of_samples(): + X = pd.DataFrame( + {"cat": ["common_a"] * 40 + ["common_b"] * 20 + [f"rare_{i}" for i in range(4)]} + ) + y = np.array([0, 1] * 32) + pp = PoniardPreprocessor(ohe_min_frequency=0.1).build(X=X, y=y, task="classification") + out = pp.preprocessor.fit_transform(X, y) + assert "cat_common_a" in out.columns + assert "cat_infrequent_sklearn" in out.columns + + def test_categorical_imputer_constant_creates_missing_category(): X = pd.DataFrame({"cat": ["a", "b", np.nan, "a", "b"]}) y = np.array([0, 1, 0, 1, 0]) - pp = PoniardPreprocessor(categorical_imputer="constant").build(X=X, y=y, task="classification") + pp = PoniardPreprocessor(categorical_imputer="constant", ohe_min_frequency=None).build( + X=X, y=y, task="classification" + ) out = pp.preprocessor.fit_transform(X, y) assert "cat_missing" in out.columns