From 955bea9aa4e8bd5f8c216bfcb30508e2d9a95d89 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 28 Sep 2026 11:28:38 +0100 Subject: [PATCH 1/4] fix: raise on quantize() of an already-quantized model Signed-off-by: Arthit Suriyawongkul --- .../fasttext/tests/test_quantize_twice.py | 42 +++++++++++++++++++ src/fasttext.cc | 8 ++++ 2 files changed, 50 insertions(+) create mode 100644 python/fasttext_module/fasttext/tests/test_quantize_twice.py diff --git a/python/fasttext_module/fasttext/tests/test_quantize_twice.py b/python/fasttext_module/fasttext/tests/test_quantize_twice.py new file mode 100644 index 0000000..d466821 --- /dev/null +++ b/python/fasttext_module/fasttext/tests/test_quantize_twice.py @@ -0,0 +1,42 @@ +# SPDX-FileContributor: Arthit Suriyawongkul +# SPDX-FileCopyrightText: 2026-present, fasttext-community +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: MIT + +"""quantize() must raise on a quantized model, and only on one.""" + +import pytest + +from .helpers import build_supervised_model, get_random_data + + +def _model(): + # thread=12: thread <= 10 leaves the input matrix partly uninitialized. + data = get_random_data(3000, max_vocab_size=600) + return build_supervised_model(data, {"thread": 12, "dim": 16, "verbose": 0}) + + +def test_quantize_twice_raises(): + model = _model() + model.quantize() + with pytest.raises(ValueError, match="already quantized"): + model.quantize() + + +def test_load_model_clears_quantized(tmp_path): + model = _model() + path = str(tmp_path / "model.bin") + model.save_model(path) + model.quantize() + model.f.loadModel(path) + assert not model.is_quantized() + model.quantize() + + +def test_set_matrices_clears_quantized(): + model = _model() + matrices = model.get_input_matrix(), model.get_output_matrix() + model.quantize() + model.set_matrices(*matrices) + assert not model.is_quantized() + model.quantize() diff --git a/src/fasttext.cc b/src/fasttext.cc index 6852aa4..ee088e0 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -86,6 +86,7 @@ namespace fasttext input_ = std::dynamic_pointer_cast(inputMatrix); output_ = std::dynamic_pointer_cast(outputMatrix); + quant_ = false; wordVectors_.reset(); args_->dim = input_->size(1); @@ -285,6 +286,7 @@ namespace fasttext void FastText::loadModel(std::istream &in) { + quant_ = false; args_ = std::make_shared(); input_ = std::make_shared(); output_ = std::make_shared(); @@ -378,6 +380,12 @@ namespace fasttext void FastText::quantize(const Args &qargs, const TrainCallback &callback) { + if (quant_) + { + throw std::invalid_argument( + "Model is already quantized. " + "Quantize the original (non-quantized) model instead."); + } if (args_->model != model_name::sup) { throw std::invalid_argument( From 7e1eb7c047f5b0ac59d9b76c2b33388d19f90f9c Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 4 Oct 2026 15:32:06 +0100 Subject: [PATCH 2/4] set quant_ from the loaded flag in loadModel Signed-off-by: Arthit Suriyawongkul --- src/fasttext.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fasttext.cc b/src/fasttext.cc index ba44a9d..f54fdde 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -294,7 +294,6 @@ namespace fasttext void FastText::loadModel(std::istream &in) { - quant_ = false; args_ = std::make_shared(); input_ = std::make_shared(); output_ = std::make_shared(); @@ -308,9 +307,9 @@ namespace fasttext bool quant_input; in.read((char *)&quant_input, sizeof(bool)); + quant_ = quant_input; if (quant_input) { - quant_ = true; input_ = std::make_shared(); } input_->load(in); From 91a6187767e98e6a331fbf54a90a7c41971c0e56 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 6 Oct 2026 15:04:37 +0100 Subject: [PATCH 3/4] fix: check sizes before quantize changes the model Signed-off-by: Arthit Suriyawongkul --- .../fasttext/tests/test_quantize_twice.py | 18 ++++++++++++++++++ src/fasttext.cc | 14 ++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/python/fasttext_module/fasttext/tests/test_quantize_twice.py b/python/fasttext_module/fasttext/tests/test_quantize_twice.py index d466821..605f206 100644 --- a/python/fasttext_module/fasttext/tests/test_quantize_twice.py +++ b/python/fasttext_module/fasttext/tests/test_quantize_twice.py @@ -7,6 +7,8 @@ import pytest +import fasttext + from .helpers import build_supervised_model, get_random_data @@ -33,6 +35,22 @@ def test_load_model_clears_quantized(tmp_path): model.quantize() +@pytest.mark.parametrize("kwargs", [{"qout": True}, {"cutoff": 100}]) +def test_failed_quantize_leaves_model_unchanged(tmp_path, kwargs): + """Used to prune the dict, or quantize input only (retry crashed).""" + data = get_random_data(3000, max_vocab_size=600) + path = tmp_path / "train.txt" + # 3 labels: too few output rows for qout. + path.write_text("".join(f"__label__{i % 3} {x}\n" for i, x in enumerate(data))) + model = fasttext.train_supervised(str(path), thread=12, dim=16, verbose=0) + nwords = len(model.get_words()) + with pytest.raises(ValueError, match="too small"): + model.quantize(**kwargs) + assert not model.is_quantized() + assert len(model.get_words()) == nwords + model.quantize() + + def test_set_matrices_clears_quantized(): model = _model() matrices = model.get_input_matrix(), model.get_output_matrix() diff --git a/src/fasttext.cc b/src/fasttext.cc index f54fdde..7b35c4c 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -398,6 +398,20 @@ namespace fasttext throw std::invalid_argument( "For now we only support quantization of supervised models"); } + // Check sizes before changing anything: failing later leaves a + // half-quantized model (pruned dict, or quantized input only). + const int64_t minRows = 256; // ProductQuantizer::ksub_ + int64_t inputRows = input_->size(0); + if (qargs.cutoff > 0 && qargs.cutoff < static_cast(inputRows)) + { + inputRows = qargs.cutoff; + } + if (inputRows < minRows || (qargs.qout && output_->size(0) < minRows)) + { + throw std::invalid_argument( + "Matrix too small for quantization, must have at least " + + std::to_string(minRows) + " rows"); + } args_->input = qargs.input; args_->qout = qargs.qout; args_->output = qargs.output; From a6c28bc13fca63bcc00ee14d17a2347e968a2c88 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 6 Oct 2026 15:11:14 +0100 Subject: [PATCH 4/4] Update comment Signed-off-by: Arthit Suriyawongkul --- python/fasttext_module/fasttext/tests/test_quantize_twice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/fasttext_module/fasttext/tests/test_quantize_twice.py b/python/fasttext_module/fasttext/tests/test_quantize_twice.py index 605f206..38520b2 100644 --- a/python/fasttext_module/fasttext/tests/test_quantize_twice.py +++ b/python/fasttext_module/fasttext/tests/test_quantize_twice.py @@ -3,7 +3,8 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: MIT -"""quantize() must raise on a quantized model, and only on one.""" +"""quantize() must raise on a quantized model, and leave the model unchanged +when it fails.""" import pytest