Implemented multiclass classification support across the C++ core and both Python wrappers, including fitting, prediction, metrics, simplification, and visualization.

docs/guide/index.md

@@ -12,9 +12,10 @@ Brush mostly consists of these components:
 data
 search_space
 working_with_programs
+multiclassification
 json
 saving_loading_populations
 locking_mechanism
 archive
 deap
-```
+```

pybrush/BrushEstimator.py

@@ -93,6 +93,12 @@ def fit(self, X, y):
 
         # Beyong this point, X is not a dataframe anymore
         X, y = check_X_y(X, y)
+        if self.mode == 'classification':
+            # The C++ core indexes probability columns with class labels.  Keep
+            # that representation internal while preserving sklearn's original
+            # labels at the public API boundary.
+            self.classes_, y = np.unique(y, return_inverse=True)
+            y = y.astype(np.float32)
 
         self.data_ = self._make_data(X, y, 
                                      feature_names=self.feature_names_,
@@ -169,6 +175,14 @@ def partial_fit(self, X, y, *,
             assert self.feature_names_ == X.columns.to_list(), \
                 "Feature names must be the same as in data from previous fit"
 
+        if self.mode == 'classification':
+            labels = np.asarray(y)
+            indices = np.searchsorted(self.classes_, labels)
+            if np.any(indices >= len(self.classes_)) \
+            or np.any(self.classes_[indices] != labels):
+                raise ValueError("partial_fit received a class not seen in fit")
+            y = indices.astype(np.float32)
+
         new_data = self._make_data(X, y, 
                                      feature_names=self.feature_names_,
                                      feature_types=self.feature_types_,
@@ -228,7 +242,10 @@ def predict(self, X):
                             validation_size=0.0,
                             )
 
-        return self.best_estimator_.program.predict(data)
+        prediction = np.asarray(self.best_estimator_.program.predict(data))
+        if self.mode == 'classification':
+            return self.classes_[prediction.astype(int)]
+        return prediction
 
     def get_params(self, deep=True):
         out = dict()
@@ -263,7 +280,8 @@ def _update_final_model(self, data=None):
         elif self.final_model_selection == "best_validation_ci":
             loss_f_dict = { # using sklearn metric, equivalent to what is used internally in brush
                 "mse": mean_squared_error, 
-                "log": log_loss, 
+                "log": log_loss,
+                "multi_log": log_loss,
                 "accuracy": accuracy_score, 
                 "balanced_accuracy": balanced_accuracy_score,
                 "average_precision_score": average_precision_score
@@ -274,11 +292,15 @@ def eval(ind, sample=None):
                 if sample is None:
                     sample = np.arange(len(y))
 
-                if self.parameters_.scorer in ["log", "average_precision_score"]:
+                if self.parameters_.scorer in ["log", "multi_log", "average_precision_score"]:
                     y_pred = np.array(ind.predict_proba(data))
                 else: # accuracy, balanced accuracy, or regression metrics
                     y_pred = np.array(ind.predict(data))
 
+                metric_kwargs = {}
+                if self.parameters_.scorer == "multi_log":
+                    metric_kwargs["labels"] = np.arange(self.parameters_.n_classes)
+
                 # y_pred = np.nan_to_num(y_pred) # Protecting the evaluation
 
                 # if user_defined, sample_weight is given by his custom weights. if
@@ -300,9 +322,10 @@ def eval(ind, sample=None):
                         # sample_weight will be indexed in the function call, so we use raw y.
                         sample_weight = [support_weights[int(label)] for label in y]
                     sample_weight = np.array(sample_weight)
-                    return loss_f(y[sample], y_pred[sample], sample_weight=sample_weight[sample])
+                    return loss_f(y[sample], y_pred[sample],
+                                  sample_weight=sample_weight[sample], **metric_kwargs)
                 else: # unbalanced metrics, ignoring weights
-                    return loss_f(y[sample], y_pred[sample])
+                    return loss_f(y[sample], y_pred[sample], **metric_kwargs)
 
             np.random.seed(0)
             val_samples = []
@@ -427,4 +450,4 @@ class BrushRegressor(BrushEstimator, RegressorMixin):
 
     def __init__(self, **kwargs):
         kwargs.pop('mode', None)
-        super().__init__(mode='regression', **kwargs)
+        super().__init__(mode='regression', **kwargs)

pybrush/EstimatorInterface.py

@@ -77,7 +77,7 @@ class EstimatorInterface():
     scorer : str, default None
         The metric to use for the "scorer" objective. If None, it will be set to
         "mse" for regression and "log" for binary classification.
-        Available options are `["mse", "log", "accuracy", "balanced_accuracy", "average_precision_score"]`
+        Available options are `["mse", "log", "multi_log", "accuracy", "balanced_accuracy", "average_precision_score"]`
     algorithm : {"nsga2island", "nsga2", "gaisland", "ga"}, default "nsga2"
         Which Evolutionary Algorithm framework to use to evolve the population.
         This is used only in DeapEstimators.
@@ -350,10 +350,13 @@ def _wrap_parameters(self, y, **extra_kwargs):
             if self.mode == "regression":
                 assert self.scorer in ['mse'], \
                     "Invalid scorer for the regression mode"
+            elif params.n_classes == 2:
+                assert self.scorer in ['log', 'balanced_accuracy', 'accuracy',
+                                       'average_precision_score'], \
+                    "Invalid scorer for binary classification"
             else:
-                assert self.scorer in ['log', 'multi_log', 'balanced_accuracy',
-                                       'accuracy', 'average_precision_score'], \
-                    "Invalid scorer for the classification mode"
+                assert self.scorer in ['multi_log', 'balanced_accuracy', 'accuracy'], \
+                    "Invalid scorer for multiclass classification"
 
         params.scorer = self.scorer
 
@@ -442,4 +445,4 @@ def __setstate__(self, state):
         # self.data_ = None
         # self.train_ = None
         # self.validation_ = None
-        # self.search_space_ = None
+        # self.search_space_ = None

pybrush/deap_api/DeapEstimator.py

@@ -156,6 +156,10 @@ def fit(self, X, y):
                         "encoding method to convert the data to a supported "
                         "format.")
 
+        if self.mode == 'classification':
+            self.classes_, y = np.unique(y, return_inverse=True)
+            y = y.astype(np.float32)
+
         self.data_ = self._make_data(X, y, 
                                      feature_names=self.feature_names_,
                                      feature_types=self.feature_types_,
@@ -267,7 +271,10 @@ def predict(self, X):
         data = Dataset(X=X, ref_dataset=self.data_, 
                               feature_names=self.feature_names_)
 
-        return self.best_estimator_.program.predict(data)
+        prediction = np.asarray(self.best_estimator_.program.predict(data))
+        if self.mode == 'classification':
+            return self.classes_[prediction.astype(int)]
+        return prediction
 
     # def _setup_population(self):
     #     """initialize programs"""
@@ -411,4 +418,4 @@ def __init__(self, **kwargs):
 
 #     def transform(self, X):
 #         """Transform X using the best estimator in the archive. """
-#         return self.predict(X)
+#         return self.predict(X)

src/bindings/bind_engines.h

@@ -89,14 +89,15 @@ void bind_engine(py::module& m, string name)
              ;
 
     // specialization for subclasses
-    if constexpr (std::is_same_v<T,Cls>)
+    if constexpr (std::is_same_v<T,Cls> || std::is_same_v<T,MCls>)
     {
+        using ProbType = std::conditional_t<std::is_same_v<T,MCls>, ArrayXXf, ArrayXf>;
         engine.def("predict_proba",
-                static_cast<ArrayXf (T::*)(const Dataset &d)>(&T::predict_proba),
+                static_cast<ProbType (T::*)(const Dataset &d)>(&T::predict_proba),
                 "predict from Dataset object")
            .def("predict_proba",
-                static_cast<ArrayXf (T::*)(const Ref<const ArrayXXf> &X)>(&T::predict_proba),
+                static_cast<ProbType (T::*)(const Ref<const ArrayXXf> &X)>(&T::predict_proba),
                 "predict from X data")
             ;
     }
-}
+}

src/bindings/bind_individuals.h

@@ -79,15 +79,16 @@ void bind_individual(py::module& m, string name)
        )
        ;
 
-    if constexpr (std::is_same_v<Class,Cls>)
+    if constexpr (std::is_same_v<Class,Cls> || std::is_same_v<Class,MCls>)
     {
+        using ProbType = typename br::Program<PT>::TreeType;
         ind.def("predict_proba",
-                static_cast<ArrayXf (Class::*)(const Dataset &d)>(&Class::predict_proba),
+                static_cast<ProbType (Class::*)(const Dataset &d)>(&Class::predict_proba),
                 "predict from Dataset object")
            .def("predict_proba",
-                static_cast<ArrayXf (Class::*)(const Ref<const ArrayXXf> &X)>(&Class::predict_proba),
+                static_cast<ProbType (Class::*)(const Ref<const ArrayXXf> &X)>(&Class::predict_proba),
                 "predict from X data")
             ;
     }
 
-}
+}

src/bindings/bind_programs.h

@@ -85,15 +85,16 @@ void bind_program(py::module& m, string name)
             )
         )
         ;
-    if constexpr (std::is_same_v<T,Cls>)
+    if constexpr (std::is_same_v<T,Cls> || std::is_same_v<T,MCls>)
     {
+        using ProbType = typename T::TreeType;
         prog.def("predict_proba",
-                static_cast<ArrayXf (T::*)(const Dataset &d)>(&T::predict_proba),
+                static_cast<ProbType (T::*)(const Dataset &d)>(&T::predict_proba),
                 "predict from Dataset object")
            .def("predict_proba",
-                static_cast<ArrayXf (T::*)(const Ref<const ArrayXXf> &X)>(&T::predict_proba),
+                static_cast<ProbType (T::*)(const Ref<const ArrayXXf> &X)>(&T::predict_proba),
                 "predict from X data")
             ;
     }
 
-}
+}

src/eval/evaluation.cpp

@@ -42,7 +42,7 @@ void Evaluation<T>::update_fitness(Population<T>& pop,
             // assign weights to individual
             if (fit && ind.get_is_fitted() == false)
             {
-                ind.program.fit(data.get_training_data());
+                ind.program.fit(data.get_training_data(), params.class_weights);
             }
 
             assign_fit(ind, data, params, validation);
@@ -126,4 +126,4 @@ void Evaluation<T>::assign_fit(Individual<T>& ind, const Dataset& data,
 template class Brush::Eval::Evaluation<Brush::ProgramType::Regressor>;
 template class Brush::Eval::Evaluation<Brush::ProgramType::BinaryClassifier>;
 template class Brush::Eval::Evaluation<Brush::ProgramType::MulticlassClassifier>;
-template class Brush::Eval::Evaluation<Brush::ProgramType::Representer>;
+template class Brush::Eval::Evaluation<Brush::ProgramType::Representer>;