diff --git a/fork.yaml b/fork.yaml index 376a8bc9db..1de0197cd4 100644 --- a/fork.yaml +++ b/fork.yaml @@ -353,6 +353,23 @@ def: - "mlx/backend/cpu/simd/base_simd.h" - "mlx/backend/cpu/simd/accelerate_simd.h" - "mlx/backend/cpu/binary.cpp" + - title: "Seeded Python tests" + description: | + Upstream Python tests draw their random input from unseeded `mx.random` and + `np.random`, so each run tests other input. Some tests compare with a reference under + a fixed tolerance that some inputs exceed: `test_double.test_gemm` (float32 8x8 + matmul against float64, `allclose` rtol 1e-5, atol 1e-8) fails when an element of the + product is near zero, because the float32 rounding of partial sums of order 1 is + about 1e-7 and the tolerance there is about 1e-8 + 1e-5 * |y|. It failed for 152 of + 2000 seeds on the CPU. `test_ops.test_scans` (bfloat16 cumsum against a tri-matrix + sum, rtol 0.02) failed for 4 of 100 seeds on the GPU and 2 of 100 on the CPU. The + CPU float32 and float64 gemm paths both clear C (section "CPU BLAS gemm clears C when + beta is 0"), so the gemm failures are tolerance, not a stale output. `MLXTestCase.setUp` + now seeds both generators with 0 before each test, so each test draws the same input + in every run; seed 0 passes both tests. Tests that seed their own generators still + do. The seed search ran in PR #46 (first commit). + globs: + - "python/tests/mlx_tests.py" - title: "Fork tooling" description: | The fork deletes the upstream workflows that cannot run here: the PyPI release build, diff --git a/python/tests/mlx_tests.py b/python/tests/mlx_tests.py index 10a7e5c35f..8552ac5a1a 100644 --- a/python/tests/mlx_tests.py +++ b/python/tests/mlx_tests.py @@ -54,6 +54,12 @@ def setUp(self): if device is not None: device = getattr(mx, device) mx.set_default_device(device) + # Each test draws the same random input in every run. Many tests + # compare with a reference under a fixed tolerance, and some random + # inputs fall outside it (for example a near-zero element under a + # relative tolerance), so unseeded input made them fail now and then. + mx.random.seed(0) + np.random.seed(0) def tearDown(self): mx.set_default_device(self.default)