diff --git a/pybrush/BrushEstimator.py b/pybrush/BrushEstimator.py index 6ed3321a9..b3c93a1e6 100644 --- a/pybrush/BrushEstimator.py +++ b/pybrush/BrushEstimator.py @@ -18,6 +18,7 @@ from sklearn.metrics import average_precision_score, mean_squared_error from pybrush import Parameters, Dataset, SearchSpace, brush_rng, individual +from pybrush._brush import set_random_state as set_brush_random_state from pybrush.EstimatorInterface import EstimatorInterface from pybrush import RegressorEngine, ClassifierEngine, MultiClassifierEngine @@ -63,6 +64,12 @@ def fit(self, X, y): 1-d array of (boolean) target values. """ + # Dataset construction can optionally shuffle a split, so seed the + # native generator before constructing any Brush object, not only when + # Engine::init runs later in this method. + if isinstance(self.random_state, int): + set_brush_random_state(self.random_state) + self.feature_names_ = [] self.feature_types_ = [] if isinstance(X, pd.DataFrame): diff --git a/pybrush/EstimatorInterface.py b/pybrush/EstimatorInterface.py index afbca056c..d5e67ebe2 100644 --- a/pybrush/EstimatorInterface.py +++ b/pybrush/EstimatorInterface.py @@ -174,12 +174,9 @@ class EstimatorInterface(): If specified, spits statistics into a logfile. "" means don't log. random_state: int or None, default None If int, then the value is used to seed the c++ random generator; if None, - then a seed will be generated using a non-deterministic generator. It is - important to notice that, even if the random state is fixed, it is - unlikely that running brush using multiple threads will have the same - results. This happens because the Operating System's scheduler is - responsible to choose which thread will run at any given time, thus - reproductibility is not guaranteed. + then a seed will be generated using a non-deterministic generator. A + fixed integer makes repeated runs reproducible for the same Brush build, + data, and parameter configuration, including multi-island runs. """ def __init__(self, diff --git a/src/engine.cpp b/src/engine.cpp index 4424b2079..f4e945017 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -516,11 +516,15 @@ void Engine::run(Dataset &data) // heavily inspired in https://github.com/heal-research/operon/blob/main/source/algorithms/nsga2.cpp auto [init, cond, body, back, done] = taskflow.emplace( [&](tf::Subflow& subflow) { - auto fit_init_pop = subflow.for_each_index(0, this->params.num_islands, 1, [&](int island) { - // Evaluate the individuals at least once - // Set validation loss before calling update best - - evaluator.update_fitness(this->pop, island, data, params, true, true); + // Fitting an island mutates program weights. Keep that phase in + // a stable order as well, so the first selection sees identical + // fitness values for repeated seeded runs. + auto fit_init_pop = subflow.emplace([&]() { + for (int island = 0; island < this->params.num_islands; ++island) { + // Evaluate the individuals at least once. Set validation + // loss before calling update_best. + evaluator.update_fitness(this->pop, island, data, params, true, true); + } }); auto find_init_best = subflow.emplace([&]() { // Make sure we initialize it. We do this update here because we need to @@ -541,7 +545,13 @@ void Engine::run(Dataset &data) batch = data.get_batch(); // will return the original dataset if it is set to dont use batch }).name("prepare generation");// set generation in params, get batch - auto run_generation = subflow.for_each_index(0, this->params.num_islands, 1, [&](int island) { + // Selection consumes the process-wide random stream. Keep island + // selection ordered: Taskflow workers are not OpenMP workers, so + // using their scheduling order would otherwise make a fixed seed + // nondeterministic. Initial fitness evaluation is ordered too, + // so the complete seeded evolutionary path is scheduler-neutral. + auto run_generation = subflow.emplace([&]() { + for (int island = 0; island < this->params.num_islands; ++island) { evaluator.update_fitness(this->pop, island, data, params, false, false); // fit the weights with all training data @@ -556,8 +566,8 @@ void Engine::run(Dataset &data) } this->pop.add_offspring_indexes(island); - - }).name("runs one generation at each island in parallel"); + } + }).name("runs one generation at each island in deterministic order"); auto update_pop = subflow.emplace([&]() { // sync point // Variation is not thread safe. @@ -692,4 +702,4 @@ void Engine::run(Dataset &data) template class Brush::Engine; template class Brush::Engine; template class Brush::Engine; -template class Brush::Engine; \ No newline at end of file +template class Brush::Engine; diff --git a/src/program/node.cpp b/src/program/node.cpp index 99c94966f..dd53e42e2 100644 --- a/src/program/node.cpp +++ b/src/program/node.cpp @@ -121,7 +121,8 @@ void to_json(json& j, const Node& p) { j = json{ {"name", p.name}, - {"center_op", p.center_op}, + // center_op is always set to false --> not implemented yet + {"center_op", false}, {"node_is_fixed", p.node_is_fixed}, {"weight_is_fixed", p.weight_is_fixed}, {"prob_change", p.prob_change}, @@ -319,9 +320,6 @@ void from_json(const json &j, Node& p) else p.name = NodeTypeName[p.node_type]; - if (j.contains("center_op")) - j.at("center_op").get_to(p.center_op); - // used in split nodes if (j.contains("feature")) { @@ -361,6 +359,8 @@ void from_json(const json &j, Node& p) // after this point we set attributes that are modified in init p.init(); + // center_op is intentionally ignored + p.center_op = false; // these below needs to be set after init(), since `init` sets these values if (j.contains("node_is_fixed")) diff --git a/src/program/node.h b/src/program/node.h index 21ec22331..62b362772 100644 --- a/src/program/node.h +++ b/src/program/node.h @@ -122,7 +122,9 @@ struct Node { float W; /// whether to center the operator in pretty printing - bool center_op; // TODO: use center_op in printing + // Retained for backwards-compatible program JSON. This has not being implemented yet + // While doing some random_state tests, i realized that this feature was being set to true sometimes, undesirable. So we make sure to initialize it as false always + bool center_op = false; // /// @brief a node hash / unique ID for the node, except weights // size_t node_hash; @@ -171,6 +173,8 @@ struct Node { } void init(){ + center_op = false; + // starting weights with neutral element of the operation. offsetsum // is the only node that does not multiply the weight --- instead, it adds it, // so we need to handle it differently diff --git a/src/util/rnd.cpp b/src/util/rnd.cpp index 760e3343b..6272b9777 100644 --- a/src/util/rnd.cpp +++ b/src/util/rnd.cpp @@ -63,6 +63,9 @@ namespace Brush { namespace Util{ for (size_t i = 0; i < rg.size(); ++i) { rg[i].seed(seeds[i]); } + + has_spare_normal = false; + spare_normal = 0.0f; } int Rnd::rnd_int( int lowerLimit, int upperLimit ) @@ -119,28 +122,33 @@ namespace Brush { namespace Util{ float Rnd::gasdev() //Returns a normally distributed deviate with zero mean and unit variance { - float ran = rnd_flt(-1,1); - static int iset=0; - static float gset; - float fac,rsq,v1,v2; - if (iset == 0) {// We don't have an extra deviate handy, so - do{ - v1=float(2.0*rnd_flt(-1,1)-1.0); //pick two uniform numbers in the square ex - v2=float(2.0*rnd_flt(-1,1)-1.0); //tending from -1 to +1 in each direction, - rsq=v1*v1+v2*v2; //see if they are in the unit circle, - } while (rsq >= 1.0 || rsq == 0.0); //and if they are not, try again. - fac=float(sqrt(-2.0*log(rsq)/rsq)); + // Box–Muller / polar implementation + // Returns a normally distributed deviate with zero mean + // and unit variance. + + if (has_spare_normal) { + has_spare_normal = false; + return spare_normal; + } + + // float ran = rnd_flt(-1,1); + + float rsq,v1,v2; + + do{ + v1=float(rnd_flt(-1.0f,1.0f)); //pick two uniform numbers in the square ex + v2=float(rnd_flt(-1.0f,1.0f)); //tending from -1 to +1 in each direction, + rsq=v1*v1+v2*v2; //see if they are in the unit circle, + } while (rsq >= 1.0 || rsq == 0.0); //and if they are not, try again. + + const float fac = float(sqrt(-2.0*log(rsq)/rsq)); + //Now make the Box-Muller transformation to get two normal deviates. Return one and //save the other for next time. - gset=v1*fac; - iset=1; //Set flag. + spare_normal=v1*fac; + has_spare_normal=true; + return v2*fac; - } - else - { //We have an extra deviate handy, - iset=0; //so unset the flag, - return gset; //and return it. - } } /// returns a shuffled index vector of length n diff --git a/src/util/rnd.h b/src/util/rnd.h index 6154b7460..933757f3e 100644 --- a/src/util/rnd.h +++ b/src/util/rnd.h @@ -163,6 +163,15 @@ namespace Brush { namespace Util{ // Vector of pseudo-random number generators, one for each thread vector rg; + + // Cached half of the Box-Muller transform used by gasdev(). This + // state is part of the generator and must be reset with its seed. + + // Gaussian random numbers are generated with box-muller --> that + // process generates two independent distributed random normal numbers. + // The idea here is to cache the second one and use it, instead of discarding. + bool has_spare_normal = false; + float spare_normal = 0.0f; // private static attribute used by every instance of the class. // All threads share common static members of the class diff --git a/tests/cpp/gtest.cpp b/tests/cpp/gtest.cpp index 6eef53758..adb5a18e5 100644 --- a/tests/cpp/gtest.cpp +++ b/tests/cpp/gtest.cpp @@ -1,6 +1,7 @@ #include "testsHeader.h" int main(int argc, char **argv) { + Brush::Util::r.set_seed(42); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/cpp/test_brush.cpp b/tests/cpp/test_brush.cpp index 765f1b655..49b7ab05a 100644 --- a/tests/cpp/test_brush.cpp +++ b/tests/cpp/test_brush.cpp @@ -38,6 +38,7 @@ TEST(Engine, EngineWorks) SearchSpace ss(data); Parameters params; + params.set_random_state(42); params.set_pop_size(100); params.set_max_gens(10); params.set_mig_prob(0.0); @@ -124,6 +125,38 @@ TEST(Engine, EngineWorks) // TODO: validation loss } +TEST(Engine, FixedSeedProducesIdenticalRuns) +{ + MatrixXf X(12, 2); + ArrayXf y(12); + X << 0.0f, 1.0f, 1.0f, 0.0f, 2.0f, 1.0f, 3.0f, 0.0f, + 4.0f, 1.0f, 5.0f, 0.0f, 6.0f, 1.0f, 7.0f, 0.0f, + 8.0f, 1.0f, 9.0f, 0.0f, 10.0f, 1.0f, 11.0f, 0.0f; + y << 1.0f, 2.0f, 2.5f, 3.5f, 4.0f, 5.0f, 5.5f, 6.5f, + 7.0f, 8.0f, 8.5f, 9.5f; + + Dataset data(X, y); + SearchSpace first_space(data); + SearchSpace second_space(data); + Parameters params; + params.set_random_state(42); + params.set_pop_size(12); + params.set_max_gens(4); + params.set_num_islands(2); + params.set_n_jobs(2); + params.set_mig_prob(0.0f); + params.set_verbosity(0); + + Brush::RegressorEngine first(params, first_space); + first.run(data); + Brush::RegressorEngine second(params, second_space); + second.run(data); + + ASSERT_EQ(first.best_ind.program.get_model(), second.best_ind.program.get_model()); + ASSERT_EQ(first.best_ind.fitness.get_wvalues(), second.best_ind.fitness.get_wvalues()); + ASSERT_EQ(first.get_population_as_json(), second.get_population_as_json()); +} + #include #include @@ -139,6 +172,7 @@ TEST_P(EngineTest, ClassificationEngineWorks) ASSERT_TRUE(data.classification); Parameters params; + params.set_random_state(42); params.set_pop_size(10); // TODO: this set_class_weights is not working properly. check that @@ -212,6 +246,7 @@ TEST(Engine, SavingLoadingFixedNodes) SearchSpace ss(data); Parameters params; + params.set_random_state(42); params.set_verbosity(2); params.set_scorer("log"); params.set_cx_prob(0.0); @@ -239,6 +274,7 @@ TEST(Engine, SavingLoadingFixedNodes) // TODO: why if I set cx_prob to 0.0 it does not work? (maybe because Im using the same params object for the two engines? do i need to remove save_pop file first?) Parameters params2; + params2.set_random_state(42); params2.set_verbosity(2); params2.set_scorer("log"); params2.set_load_population("./tests/cpp/__pop_clf.json"); @@ -281,6 +317,7 @@ TEST(Engine, MaxStall) SearchSpace ss(data); Parameters params; + params.set_random_state(42); params.set_pop_size(100); params.set_max_gens(10000000); params.set_mig_prob(0.0); @@ -316,6 +353,7 @@ TEST(Engine, engine_save_load_pop_works) }; Parameters params_save; + params_save.set_random_state(42); params_save.set_functions(f); params_save.set_pop_size(200); params_save.set_max_gens(10); @@ -324,6 +362,7 @@ TEST(Engine, engine_save_load_pop_works) params_save.set_save_population("./tests/cpp/__pop_analcatdata_aids.json"); Parameters params_load; + params_load.set_random_state(42); params_load.set_functions(f); params_load.set_pop_size(200); params_load.set_max_gens(10); @@ -355,6 +394,7 @@ TEST(Engine, DEnc) std::cout << "Running bandit: " << bandit << std::endl; for (int run = 0; run < 2; ++run) { Parameters params; + params.set_random_state(42); params.set_pop_size(100); params.set_max_gens(50); params.set_max_stall(100); // avoid early stopping diff --git a/tests/cpp/test_data.cpp b/tests/cpp/test_data.cpp index f5f154eb1..1589efcd6 100644 --- a/tests/cpp/test_data.cpp +++ b/tests/cpp/test_data.cpp @@ -26,6 +26,7 @@ TEST(Data, ErrorHandling) TEST(Data, MixedVariableTypes) { Parameters params; + params.set_random_state(42); MatrixXf X(5,3); X << 0 , 1, 0 , // binary with integer values diff --git a/tests/cpp/test_evaluation.cpp b/tests/cpp/test_evaluation.cpp index 62ec48a8f..7d748613f 100644 --- a/tests/cpp/test_evaluation.cpp +++ b/tests/cpp/test_evaluation.cpp @@ -131,6 +131,7 @@ TEST(Evaluation, MulticlassSoftmaxHasOneOutputPerClass) Dataset data(X, y, {}, {}, {}, true); SearchSpace search_space(data); Parameters params; + params.set_random_state(42); params.classification = true; params.set_n_classes(y); params.max_depth = 3; diff --git a/tests/cpp/test_individuals.cpp b/tests/cpp/test_individuals.cpp index c00dcb442..8e01cc1f8 100644 --- a/tests/cpp/test_individuals.cpp +++ b/tests/cpp/test_individuals.cpp @@ -24,6 +24,7 @@ TEST(Individual, FitAndPredictRegression) // We must have a SearchSpace reference, so the operator ret-type checks dont // fail even when feature names look right --- node metadata is consistent. Parameters params; + params.set_random_state(42); RegressorProgram prg = ss.make_regressor(0, 0, params); Individual ind(prg); @@ -52,6 +53,7 @@ TEST(Individual, PredictProbaBinaryClassifier) SearchSpace ss(data); Parameters params; + params.set_random_state(42); params.set_n_classes(data.y); params.set_sample_weights(data.y); @@ -79,4 +81,4 @@ TEST(Individual, ParentIdAndId) ASSERT_EQ(child.parent_id.size(), 2u); ASSERT_EQ(child.parent_id.at(0), 3u); ASSERT_EQ(child.parent_id.at(1), 7u); -} \ No newline at end of file +} diff --git a/tests/cpp/test_params.cpp b/tests/cpp/test_params.cpp index 61265e6dc..0ff9b0abc 100644 --- a/tests/cpp/test_params.cpp +++ b/tests/cpp/test_params.cpp @@ -17,6 +17,7 @@ TEST(Params, ParamsTests) std::cout << "unsigned long long: min=" << std::numeric_limits::min() << " max=" << std::numeric_limits::max() << "\n"; Parameters params; + params.set_random_state(42); params.set_max_size(12); ASSERT_EQ(params.max_size, 12); diff --git a/tests/cpp/test_population.cpp b/tests/cpp/test_population.cpp index 8994e9273..fb5e2bdb4 100644 --- a/tests/cpp/test_population.cpp +++ b/tests/cpp/test_population.cpp @@ -32,6 +32,7 @@ TEST(Population, PopulationTests) SS.init(data); Parameters params; + params.set_random_state(42); params.pop_size = 20; // small pop just for tests Population pop = Population(); @@ -147,4 +148,4 @@ TEST(Population, PopulationTests) // testing that we can save and load the population pop.save("./tests/cpp/__pop_save_100_gen.json"); pop.load("./tests/cpp/__pop_save_100_gen.json"); -} \ No newline at end of file +} diff --git a/tests/cpp/test_program.cpp b/tests/cpp/test_program.cpp index 37fa16919..51d009cfb 100644 --- a/tests/cpp/test_program.cpp +++ b/tests/cpp/test_program.cpp @@ -14,6 +14,7 @@ TEST(Program, MakeRegressor) SearchSpace SS; SS.init(data); Parameters params; + params.set_random_state(42); // Program DXtree; for (int d = 1; d < 10; ++d) @@ -64,6 +65,7 @@ TEST(Program, MakeRegressor) TEST(Program, FitRegressor) { Parameters params; + params.set_random_state(42); Dataset data = Data::read_csv("docs/examples/datasets/d_enc.csv","label"); @@ -110,6 +112,7 @@ TEST(Program, FitRegressor) TEST(Program, PredictWithWeights) { Parameters params; + params.set_random_state(42); Dataset data = Data::read_csv("docs/examples/datasets/d_enc.csv","label"); @@ -164,6 +167,7 @@ TEST(Program, PredictWithWeights) TEST(Program, FitClassifier) { Parameters params; + params.set_random_state(42); Dataset data = Data::read_csv("docs/examples/datasets/d_analcatdata_aids.csv", "target"); @@ -219,6 +223,7 @@ TEST(Program, FitClassifier) TEST(Program, Serialization) { Parameters params; + params.set_random_state(42); // test mutation // TODO: set random seed @@ -290,6 +295,7 @@ TEST(Operators, ProgramSizeAndDepthPARAMS) Dataset data(X,y); Parameters params; + params.set_random_state(42); SearchSpace SS; SS.init(data); @@ -339,6 +345,7 @@ TEST(Operators, ProgramSizeAndDepthPARAMS) TEST(Program, ComparisonAndBooleanOperators) { Parameters params; + params.set_random_state(42); // dataset with float and integer features MatrixXf X(10,6); @@ -397,4 +404,4 @@ TEST(Program, ComparisonAndBooleanOperators) ASSERT_GT(PRG.depth(), 0); ASSERT_GT(PRG.size(), 0); } -} \ No newline at end of file +} diff --git a/tests/cpp/test_simplification.cpp b/tests/cpp/test_simplification.cpp index f3a0b875f..91d1708da 100644 --- a/tests/cpp/test_simplification.cpp +++ b/tests/cpp/test_simplification.cpp @@ -8,6 +8,7 @@ using namespace Brush::Simpl; TEST(Simplification, ConstantsSimplification) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -57,6 +58,7 @@ TEST(Simplification, ConstantsSimplification) TEST(Simplification, InexactSimplification) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -116,4 +118,4 @@ TEST(Simplification, InexactSimplification) "Inexact-simplified Model: {}\n", PRG2.get_model("compact", true) ); -} \ No newline at end of file +} diff --git a/tests/cpp/test_variation.cpp b/tests/cpp/test_variation.cpp index 6122e2cdc..d34cbbf34 100644 --- a/tests/cpp/test_variation.cpp +++ b/tests/cpp/test_variation.cpp @@ -4,6 +4,7 @@ TEST(Variation, FixedRootDoesntChange) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -111,6 +112,7 @@ TEST(Variation, InsertMutationWorks) // To understand design implementation of this test, check Mutation test Parameters params; + params.set_random_state(42); params.mutation_probs = { {"point", 0.0}, {"insert", 1.0}, @@ -226,6 +228,7 @@ TEST(Variation, InsertMutationWorks) TEST(Variation, Mutation) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -319,6 +322,7 @@ TEST(Variation, Mutation) TEST(Variation, MutationSizeAndDepthLimit) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -424,6 +428,7 @@ TEST(Variation, MutationSizeAndDepthLimit) TEST(Variation, Crossover) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -517,6 +522,7 @@ TEST(Variation, Crossover) TEST(Variation, CrossoverSizeAndDepthLimit) { Parameters params; + params.set_random_state(42); MatrixXf X(10,2); ArrayXf y(10); @@ -603,4 +609,4 @@ TEST(Variation, CrossoverSizeAndDepthLimit) } } ASSERT_TRUE(successes > 0); -} \ No newline at end of file +} diff --git a/tests/python/test_deap_api.py b/tests/python/test_deap_api.py index 318f13f81..993e259a8 100644 --- a/tests/python/test_deap_api.py +++ b/tests/python/test_deap_api.py @@ -153,12 +153,12 @@ def test_predict_proba(setup, brush_args, request): def test_deap_multiclass_probabilities_and_labels(): X, y = make_classification( n_samples=36, n_features=4, n_informative=3, n_redundant=0, - n_classes=3, n_clusters_per_class=1, random_state=12) + n_classes=3, n_clusters_per_class=1, random_state=42) labels = np.array(["red", "green", "blue"])[y] est = pybrush.deap_api.DeapClassifier( max_gens=2, pop_size=8, max_size=30, max_depth=4, - num_islands=1, validation_size=0.0, random_state=12) + num_islands=1, validation_size=0.0, random_state=42) est.fit(X, labels) probabilities = est.predict_proba(X) @@ -254,17 +254,4 @@ def test_fixed_nodes(setup, fixed_node, brush_args, request): for p, p_original_model in zip(pop, pop_models): assert p.program.get_model() == p_original_model, \ "Variation operator changed the original model." - - - -# TODO: make this work (i need to make each island (thread) use its own random generator) -# def test_random_state(): -# test_y = np.array( [1. , 0. , 1.4, 1. , 0. , 1. , 1. , 0. , 0. , 0. ]) -# test_X = np.array([[1.1, 2.0, 3.0, 4.0, 5.0, 6.5, 7.0, 8.0, 9.0, 10.0], -# [2.0, 1.2, 6.0, 4.0, 5.0, 8.0, 7.0, 5.0, 9.0, 10.0]]).T - -# est1 = pybrush.BrushRegressor(random_state=42).fit(test_X, test_y) -# est2 = pybrush.BrushRegressor(random_state=42).fit(test_X, test_y) - -# assert est1.best_estimator_.program.get_model() == est2.best_estimator_.program.get_model(), \ -# "random state failed to generate same results" + \ No newline at end of file diff --git a/tests/python/test_final_model_selection.py b/tests/python/test_final_model_selection.py index 546d1a2ae..963a272de 100644 --- a/tests/python/test_final_model_selection.py +++ b/tests/python/test_final_model_selection.py @@ -11,8 +11,8 @@ def test_smallest_complexity_selection_regression(): - X, y = make_regression(n_samples=80, n_features=6, noise=0.1, random_state=1) - model = BrushRegressor(max_gens=5, pop_size=12, final_model_selection="smallest_complexity") + X, y = make_regression(n_samples=80, n_features=6, noise=0.1, random_state=42) + model = BrushRegressor(max_gens=5, pop_size=12, final_model_selection="smallest_complexity", random_state=42) model.fit(X, y) chosen = model.best_estimator_ @@ -25,9 +25,9 @@ def test_smallest_complexity_selection_regression(): def test_best_validation_ci_selection_classification(): - X, y = make_classification(n_samples=100, n_features=8, n_informative=5, random_state=2) + X, y = make_classification(n_samples=100, n_features=8, n_informative=5, random_state=42) - model = BrushClassifier(max_gens=5, pop_size=15, final_model_selection="best_validation_ci") + model = BrushClassifier(max_gens=5, pop_size=15, final_model_selection="best_validation_ci", random_state=42) model.fit(X, y) # best estimator can sometimes remain unchanged with this method @@ -40,16 +40,16 @@ def test_callable_selection(): def pick_first(pop, archive): return archive[0] - X, y = make_classification(n_samples=60, n_features=5, random_state=3) - model = BrushClassifier(max_gens=5, pop_size=10, final_model_selection=pick_first) + X, y = make_classification(n_samples=60, n_features=5, random_state=42) + model = BrushClassifier(max_gens=5, pop_size=10, final_model_selection=pick_first, random_state=42) model.fit(X, y) assert model.best_estimator_ == model.archive_[0] def test_invalid_selection_raises(): - X, y = make_regression(n_samples=50, n_features=5, random_state=4) - model = BrushRegressor(max_gens=5, pop_size=10, final_model_selection="not_a_method") + X, y = make_regression(n_samples=50, n_features=5, random_state=42) + model = BrushRegressor(max_gens=5, pop_size=10, final_model_selection="not_a_method", random_state=42) with pytest.raises(ValueError): model.fit(X, y) @@ -58,8 +58,8 @@ def test_callable_failure_raises(): def bad_selector(pop, archive): raise Exception("boom") - X, y = make_classification(n_samples=60, n_features=5, random_state=5) - model = BrushClassifier(max_gens=5, pop_size=10, final_model_selection=bad_selector) + X, y = make_classification(n_samples=60, n_features=5, random_state=42) + model = BrushClassifier(max_gens=5, pop_size=10, final_model_selection=bad_selector, random_state=42) with pytest.raises(RuntimeError): model.fit(X, y) @@ -69,7 +69,7 @@ def test_regression_model_selection(): X, y = make_regression(n_samples=100, n_features=10, noise=0.1, random_state=42) # Default selection (final model will not necessarily be in the pop or archive) - model = BrushRegressor(max_gens=10, pop_size=10, final_model_selection="").fit(X, y) + model = BrushRegressor(max_gens=10, pop_size=10, final_model_selection="", random_state=42).fit(X, y) idx = np.argmin([p.fitness.linear_complexity for p in model.archive_]) @@ -80,7 +80,7 @@ def test_classification_selection(): ) # Default selection (final model will not necessarily be in the pop or archive) - model = BrushClassifier(max_gens=10, pop_size=10, final_model_selection="").fit(X, y) + model = BrushClassifier(max_gens=10, pop_size=10, final_model_selection="", random_state=42).fit(X, y) idx = np.argmin([p.fitness.linear_complexity for p in model.archive_]) @@ -103,6 +103,7 @@ def test_final_model_selection_best_validation_ci_replicated(scorer, class_weigh functions=['Add', 'Sub', 'SplitBest'], class_weights=class_weights, validation_size=0.3, + random_state=42, verbosity=0, ) est.fit(X, y) @@ -206,7 +207,7 @@ def eval_with_sklearn(individual, sample=None, log=False): print("original loss_v", est.best_estimator_.fitness.loss_v) print("recalculated loss", eval_with_sklearn(est.best_estimator_, log=True)) - np.random.seed(0) + np.random.seed(42) val_samples = [eval_with_sklearn(est.best_estimator_, np.random.randint(len(y), size=len(y))) for _ in range(100)] @@ -251,13 +252,14 @@ def test_pickle_unpickle_with_different_final_model_selection(final_model_select # previous test is using a classification problem. this one focuses on regression X, y = make_regression( - n_samples=80, n_features=6, noise=0.1, random_state=1 + n_samples=80, n_features=6, noise=0.1, random_state=42 ) model = BrushRegressor( max_gens=5, pop_size=12, final_model_selection=final_model_selection, + random_state=42 ) model.fit(X, y) @@ -282,13 +284,14 @@ def pick_last(pop, archive): def test_pickle_unpickle_with_callable_final_model_selection(): X, y = make_classification( - n_samples=60, n_features=5, random_state=7 + n_samples=60, n_features=5, random_state=42 ) model = BrushClassifier( max_gens=5, pop_size=10, final_model_selection=pick_last, + random_state=42 ) model.fit(X, y) @@ -318,12 +321,13 @@ def test_pickle_unpickle_with_different_final_model_selection(final_model_select # previous test is using a classification problem. this one focuses on regression X, y = make_regression( - n_samples=80, n_features=6, noise=0.1, random_state=1 + n_samples=80, n_features=6, noise=0.1, random_state=42 ) model = BrushRegressor( max_gens=5, pop_size=12, + random_state=42, final_model_selection=final_model_selection, ) model.fit(X, y) diff --git a/tests/python/test_params.py b/tests/python/test_params.py index 8eee29e4b..1f79ddfa1 100644 --- a/tests/python/test_params.py +++ b/tests/python/test_params.py @@ -13,9 +13,9 @@ def test_param_random_state(): test_X = np.array([[1.1, 2.0, 3.0, 4.0, 5.0, 6.5, 7.0, 8.0, 9.0, 10.0], [2.0, 1.2, 6.0, 4.0, 5.0, 8.0, 7.0, 5.0, 9.0, 10.0]]).T - # First run with random_state=123 + # First run with random_state=42 reg1 = BrushRegressor( - random_state=123, + random_state=42, max_gens=50, pop_size=10, num_islands=4, @@ -28,9 +28,9 @@ def test_param_random_state(): assert len(first_run_models) > 0, "First run produced no individuals" - # Second run with same random_state=123 + # Second run with same random_state=42 reg2 = BrushRegressor( - random_state=123, + random_state=42, max_gens=50, pop_size=10, num_islands=4, diff --git a/tests/python/test_population_archive.py b/tests/python/test_population_archive.py index 842c5711d..73f8fee59 100644 --- a/tests/python/test_population_archive.py +++ b/tests/python/test_population_archive.py @@ -13,7 +13,7 @@ def test_archive_not_empty(): - X, y = make_classification(n_samples=40, n_features=5, n_classes=2, random_state=7) + X, y = make_classification(n_samples=40, n_features=5, n_classes=2, random_state=42) est = BrushClassifier(max_gens=5, pop_size=10, verbosity=0) est.fit(X, y) @@ -22,7 +22,7 @@ def test_archive_not_empty(): def test_archive_individual_get_model_and_predict(): - X, y = make_classification(n_samples=40, n_features=5, n_classes=2, random_state=7) + X, y = make_classification(n_samples=40, n_features=5, n_classes=2, random_state=42) est = BrushClassifier(max_gens=5, pop_size=10, verbosity=0) est.fit(X, y) @@ -40,7 +40,7 @@ def test_archive_individual_get_model_and_predict(): def test_population_individual_predict(): - X, y = make_classification(n_samples=50, n_features=6, n_classes=2, random_state=13) + X, y = make_classification(n_samples=50, n_features=6, n_classes=2, random_state=42) est = BrushClassifier(max_gens=5, pop_size=12, verbosity=0) est.fit(X, y) @@ -89,4 +89,4 @@ def test_brush_pickle_and_predict(): # Predict with an individual from population if est_loaded.population_: y_pred_pop = est_loaded.population_[0].predict(data) - assert len(y_pred_pop) == len(y) \ No newline at end of file + assert len(y_pred_pop) == len(y) diff --git a/tests/python/test_sklearn_interface.py b/tests/python/test_sklearn_interface.py index 6965b63a4..419248fed 100644 --- a/tests/python/test_sklearn_interface.py +++ b/tests/python/test_sklearn_interface.py @@ -55,6 +55,30 @@ def test_brush_classifier(): assert acc > 0.5 +def test_fixed_seed_produces_identical_brush_runs(): + X, y = make_classification( + n_samples=60, n_features=6, n_informative=4, n_redundant=0, + random_state=42, + ) + config = dict( + max_gens=4, + pop_size=12, + num_islands=2, + n_jobs=2, + random_state=42, + verbosity=0, + ) + + first = BrushClassifier(**config).fit(X, y) + second = BrushClassifier(**config).fit(X, y) + + assert first.best_estimator_.get_model() == second.best_estimator_.get_model() + assert all(np.isclose(first.best_estimator_.fitness.values, second.best_estimator_.fitness.values, atol=1e-3)) + assert [ind.get_model() for ind in first.population_] == [ + ind.get_model() for ind in second.population_ + ] + + @pytest.mark.parametrize("verbosity", [0, 1, 2]) def test_brush_classifier_verbosity_levels(verbosity): X, y = make_classification(n_samples=60, n_features=8, n_classes=2, random_state=42) @@ -138,7 +162,7 @@ def test_brush_classifier_population_reuse(tmp_path): def test_brush_classifier_checkpoint_training(tmp_path): # Small synthetic dataset - X, y = make_classification(n_samples=80, n_features=8, n_classes=2, random_state=123) + X, y = make_classification(n_samples=80, n_features=8, n_classes=2, random_state=42) checkpoint = tmp_path / "brush_checkpoint.json" @@ -173,7 +197,7 @@ def test_brush_classifier_checkpoint_training(tmp_path): def test_brush_lock_nodes_and_leaves(): # Small synthetic dataset - X, y = make_classification(n_samples=60, n_features=6, n_classes=2, random_state=99) + X, y = make_classification(n_samples=60, n_features=6, n_classes=2, random_state=42) est = BrushClassifier( functions=['Add','Mul','Sin','Cos'], @@ -207,12 +231,12 @@ def test_brush_lock_nodes_and_leaves(): def test_brush_multiclass_probabilities_and_labels(): X, y = make_classification( n_samples=45, n_features=5, n_informative=4, n_redundant=0, - n_classes=3, n_clusters_per_class=1, random_state=11) + n_classes=3, n_clusters_per_class=1, random_state=42) labels = np.array(["class-a", "class-b", "class-c"])[y] est = BrushClassifier( max_gens=2, pop_size=8, max_size=30, max_depth=4, - num_islands=1, validation_size=0.0, random_state=11) + num_islands=1, validation_size=0.0, random_state=42) est.fit(X, labels) probabilities = est.predict_proba(X)