fix: don't count a discarded sample as generated text

src/common/AutoModel/automodel.cpp

@@ -391,7 +391,8 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens,
     }
 }
 
-bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector<int>& tokens, std::function<bool()> is_cancelled, void* payload, int first_len_run, std::optional<int> requested_max_new_tokens) {
+bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector<int>& tokens, std::function<bool()> is_cancelled, void* payload, int first_len_run, std::optional<int> requested_max_new_tokens,
+                        bool record_sampled_token) {
     this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens);
 
     this->_begin_turn(meta_info, tokens);
@@ -459,7 +460,7 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector<int>& to
         header_print("WARNING", "Max length reached, stopping prefilling...");
     }
     this->profiler_list[SAMPLING_TIME].start();
-    this->last_token = this->_sample(y, meta_info);
+    this->last_token = this->_sample(y, meta_info, record_sampled_token);
     meta_info.insert_sample_pending = true;
     this->profiler_list[SAMPLING_TIME].stop(1);
     return true;
@@ -537,12 +538,12 @@ void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector<int>&
     }
 }
 
-int AutoModel::_sample(buffer<bf16>& logits, chat_meta_info_t& meta_info) {
+int AutoModel::_sample(buffer<bf16>& logits, chat_meta_info_t& meta_info, bool record_in_history) {
     decode_constraint* constraint = meta_info.constraint.get();
     const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_);
     // The mask goes to the sampler, which applies it to its own copy: the logits
     // live in the engine's device-mapped buffer and are never written here.
-    int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr);
+    int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr, record_in_history);
     auto allowed = [this](size_t i) {
         return i / 32 < this->constraint_mask_.size() && ((this->constraint_mask_[i / 32] >> (i % 32)) & 1u);
     };

src/common/AutoModel/modeling_qwen3_5vl.cpp

@@ -290,9 +290,15 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s
     size_t n = tokens.size();
     tokens.resize(n - (this->enable_think ? 2 : 4));
 
+    // The draw below is thrown away: generate() force-feeds the think block and
+    // samples again. It must not enter the repetition and frequency windows --
+    // it was never generated text, and for a model that answers without a
+    // thinking block the token predicted here IS the answer letter. Counting it
+    // pushes that letter down by count * freq_penalty + pre_penalty at the position
+    // where it matters, so the model's preferred answer arrives penalised.
     bool success = has_images
-        ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index)
-        : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr);
+        ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false)
+        : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false);
 
     checkpoint_his = token_history;
     int checkpoint_idx = this->lm_engine->checkpoint();

src/common/AutoModel/modeling_qwen3_6_moe.cpp

@@ -299,9 +299,15 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
     size_t n = tokens.size();
     tokens.resize(n - (this->enable_think ? 2 : 4));
 
+    // The draw below is thrown away: generate() force-feeds the think block and
+    // samples again. It must not enter the repetition and frequency windows --
+    // it was never generated text, and for a model that answers without a
+    // thinking block the token predicted here IS the answer letter. Counting it
+    // pushes that letter down by count * freq_penalty + pre_penalty at the position
+    // where it matters, so the model's preferred answer arrives penalised.
     bool success = has_images
-        ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index)
-        : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr);
+        ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false)
+        : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false);
 
     checkpoint_his = token_history;
     int checkpoint_idx = this->lm_engine->checkpoint();

src/common/modules/sampler.cpp

@@ -357,7 +357,7 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) {
 /// \brief Sample the token
 /// \param x the input buffer
 /// \return the sampled token
-int Sampler::sample_greedy(buffer<bf16>& x, const std::vector<uint32_t>* allowed) {
+int Sampler::sample_greedy(buffer<bf16>& x, const std::vector<uint32_t>* allowed, bool record_in_history) {
     const bool penalties_active =
         (this->repeat_last_n != 0) &&
         (this->rep_penalty != 1.0f || this->freq_penalty != 0.0f || this->pre_penalty != 0.0f);
@@ -447,7 +447,9 @@ int Sampler::sample_greedy(buffer<bf16>& x, const std::vector<uint32_t>* allowed
 
     // callers that inspect the surviving candidates still get a consistent view
     this->top_k_logits.assign(1, logits_t{ best, best_id, 1.0f });
-    ring_buffer_update_sparse(best_id);
+    if (record_in_history) {
+        ring_buffer_update_sparse(best_id);
+    }
     return best_id;
 }
 
@@ -471,10 +473,10 @@ void Sampler::sampler_allowed_apply(const std::vector<uint32_t>* allowed) {
     }
 }
 
-int Sampler::sample(buffer<bf16>& x, const std::vector<uint32_t>* allowed) {
+int Sampler::sample(buffer<bf16>& x, const std::vector<uint32_t>* allowed, bool record_in_history) {
     // PRNG is seeded once at construction, no per-call seeding needed
     if (this->top_k == 1) {
-        return sample_greedy(x, allowed);
+        return sample_greedy(x, allowed, record_in_history);
     }
 
     // COPY FROM `x` → `this->logits[]`
@@ -515,7 +517,9 @@ int Sampler::sample(buffer<bf16>& x, const std::vector<uint32_t>* allowed) {
     }
 
     int sampled_index = sample_from_probs();
-    ring_buffer_update_sparse(sampled_index);
+    if (record_in_history) {
+        ring_buffer_update_sparse(sampled_index);
+    }
 
     return sampled_index;
 }

src/include/AutoModel/automodel.hpp

@@ -268,7 +268,8 @@ class AutoModel {
 	/// \param tokens the tokens to insert
 	/// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed
 	/// \return true if the tokens were inserted successfully, false otherwise
-	bool _shared_insert(chat_meta_info_t& meta_info, std::vector<int>& tokens, std::function<bool()> is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional<int> requested_max_new_tokens = std::nullopt);
+	bool _shared_insert(chat_meta_info_t& meta_info, std::vector<int>& tokens, std::function<bool()> is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional<int> requested_max_new_tokens = std::nullopt,
+                         bool record_sampled_token = true);
 
 	/// \brief Reject a request that cannot fit in the backend's decode limit
 	/// \param rendered_tokens the length of the rendered prompt
@@ -290,8 +291,10 @@ class AutoModel {
 	/// \brief Sample the next token under the request's constraint
 	/// \param logits the logits of the token about to be sampled, masked in place
 	/// \param meta_info the request, carrying the constraint
+	/// \param record_in_history whether the token joins the sampler's repetition
+	///        and frequency windows; false for a draw the caller throws away
 	/// \return the sampled token, already reported to the constraint
-	int _sample(buffer<bf16>& logits, chat_meta_info_t& meta_info);
+	int _sample(buffer<bf16>& logits, chat_meta_info_t& meta_info, bool record_in_history = true);
 
 	/// \brief Report a token the model is made to emit rather than samples
 	/// \note For the think preamble some generate()s feed after insert(); the

src/include/modules/sampler.hpp

@@ -102,7 +102,13 @@ class Sampler{
     ///       Going through the general path would allocate and partial_sort a
     ///       vocab-sized array of triples and run three full-vocab softmaxes for
     ///       a result that never depends on any of it.
-    int sample_greedy(buffer<bf16>& x, const std::vector<uint32_t>* allowed = nullptr);
+    /// \param record_in_history whether the drawn token joins the repetition
+    ///        and frequency windows
+    /// \note Pass false for a draw the caller throws away. Such a token was
+    ///       never generated text, and counting it pushes the tokens that
+    ///       follow it down -- by count * freq_penalty + pre_penalty each time
+    ///       it occurs.
+    int sample_greedy(buffer<bf16>& x, const std::vector<uint32_t>* allowed = nullptr, bool record_in_history = true);
 
     void sampler_topk_apply(int k);
     void sampler_topp_apply(float p);
@@ -117,7 +123,9 @@ class Sampler{
     /// \param allowed optional bitmask, bit i of word i/32 set when token i may be
     ///        sampled; applied to the sampler's own copy of the logits
     /// \return the sampled token
-    int sample(buffer<bf16>& x, const std::vector<uint32_t>* allowed = nullptr);
+    /// \param record_in_history whether the drawn token joins the repetition
+    ///        and frequency windows; see sample_greedy
+    int sample(buffer<bf16>& x, const std::vector<uint32_t>* allowed = nullptr, bool record_in_history = true);
 
     /// \brief Push every token the bitmask does not allow to -inf in this->logits
     void sampler_allowed_apply(const std::vector<uint32_t>* allowed);