diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index e5cc893f6..7ba78335c 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -391,7 +391,8 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, } } -bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens) { +bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens, + bool record_sampled_token) { this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens); this->_begin_turn(meta_info, tokens); @@ -459,7 +460,7 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); - this->last_token = this->_sample(y, meta_info); + this->last_token = this->_sample(y, meta_info, record_sampled_token); meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); return true; @@ -537,12 +538,12 @@ void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& } } -int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info) { +int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history) { decode_constraint* constraint = meta_info.constraint.get(); const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); // The mask goes to the sampler, which applies it to its own copy: the logits // live in the engine's device-mapped buffer and are never written here. - int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr); + int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr, record_in_history); auto allowed = [this](size_t i) { return i / 32 < this->constraint_mask_.size() && ((this->constraint_mask_[i / 32] >> (i % 32)) & 1u); }; diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index ab3b31dba..c3f5347da 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -290,9 +290,15 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s size_t n = tokens.size(); tokens.resize(n - (this->enable_think ? 2 : 4)); + // The draw below is thrown away: generate() force-feeds the think block and + // samples again. It must not enter the repetition and frequency windows -- + // it was never generated text, and for a model that answers without a + // thinking block the token predicted here IS the answer letter. Counting it + // pushes that letter down by count * freq_penalty + pre_penalty at the position + // where it matters, so the model's preferred answer arrives penalised. bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 9a6aaabf5..6fa07cbc6 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -299,9 +299,15 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, size_t n = tokens.size(); tokens.resize(n - (this->enable_think ? 2 : 4)); + // The draw below is thrown away: generate() force-feeds the think block and + // samples again. It must not enter the repetition and frequency windows -- + // it was never generated text, and for a model that answers without a + // thinking block the token predicted here IS the answer letter. Counting it + // pushes that letter down by count * freq_penalty + pre_penalty at the position + // where it matters, so the model's preferred answer arrives penalised. bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/common/modules/sampler.cpp b/src/common/modules/sampler.cpp index 59c01cb8e..0075f53d0 100644 --- a/src/common/modules/sampler.cpp +++ b/src/common/modules/sampler.cpp @@ -357,7 +357,7 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) { /// \brief Sample the token /// \param x the input buffer /// \return the sampled token -int Sampler::sample_greedy(buffer& x, const std::vector* allowed) { +int Sampler::sample_greedy(buffer& x, const std::vector* allowed, bool record_in_history) { const bool penalties_active = (this->repeat_last_n != 0) && (this->rep_penalty != 1.0f || this->freq_penalty != 0.0f || this->pre_penalty != 0.0f); @@ -447,7 +447,9 @@ int Sampler::sample_greedy(buffer& x, const std::vector* allowed // callers that inspect the surviving candidates still get a consistent view this->top_k_logits.assign(1, logits_t{ best, best_id, 1.0f }); - ring_buffer_update_sparse(best_id); + if (record_in_history) { + ring_buffer_update_sparse(best_id); + } return best_id; } @@ -471,10 +473,10 @@ void Sampler::sampler_allowed_apply(const std::vector* allowed) { } } -int Sampler::sample(buffer& x, const std::vector* allowed) { +int Sampler::sample(buffer& x, const std::vector* allowed, bool record_in_history) { // PRNG is seeded once at construction, no per-call seeding needed if (this->top_k == 1) { - return sample_greedy(x, allowed); + return sample_greedy(x, allowed, record_in_history); } // COPY FROM `x` → `this->logits[]` @@ -515,7 +517,9 @@ int Sampler::sample(buffer& x, const std::vector* allowed) { } int sampled_index = sample_from_probs(); - ring_buffer_update_sparse(sampled_index); + if (record_in_history) { + ring_buffer_update_sparse(sampled_index); + } return sampled_index; } \ No newline at end of file diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index a2921a192..9115f5690 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -268,7 +268,8 @@ class AutoModel { /// \param tokens the tokens to insert /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise - bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt); + bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt, + bool record_sampled_token = true); /// \brief Reject a request that cannot fit in the backend's decode limit /// \param rendered_tokens the length of the rendered prompt @@ -290,8 +291,10 @@ class AutoModel { /// \brief Sample the next token under the request's constraint /// \param logits the logits of the token about to be sampled, masked in place /// \param meta_info the request, carrying the constraint + /// \param record_in_history whether the token joins the sampler's repetition + /// and frequency windows; false for a draw the caller throws away /// \return the sampled token, already reported to the constraint - int _sample(buffer& logits, chat_meta_info_t& meta_info); + int _sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history = true); /// \brief Report a token the model is made to emit rather than samples /// \note For the think preamble some generate()s feed after insert(); the diff --git a/src/include/modules/sampler.hpp b/src/include/modules/sampler.hpp index df7997345..4a174a87e 100644 --- a/src/include/modules/sampler.hpp +++ b/src/include/modules/sampler.hpp @@ -102,7 +102,13 @@ class Sampler{ /// Going through the general path would allocate and partial_sort a /// vocab-sized array of triples and run three full-vocab softmaxes for /// a result that never depends on any of it. - int sample_greedy(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows + /// \note Pass false for a draw the caller throws away. Such a token was + /// never generated text, and counting it pushes the tokens that + /// follow it down -- by count * freq_penalty + pre_penalty each time + /// it occurs. + int sample_greedy(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); void sampler_topk_apply(int k); void sampler_topp_apply(float p); @@ -117,7 +123,9 @@ class Sampler{ /// \param allowed optional bitmask, bit i of word i/32 set when token i may be /// sampled; applied to the sampler's own copy of the logits /// \return the sampled token - int sample(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows; see sample_greedy + int sample(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); /// \brief Push every token the bitmask does not allow to -inf in this->logits void sampler_allowed_apply(const std::vector* allowed);