From 2e7e47c409bb38b38ae1d76c0ae4eb5454c8718f Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 10:19:34 +0200 Subject: [PATCH 1/6] fix: don't count a discarded sample as generated text Qwen3_5VL::insert() and Qwen3_6_MoE::insert() call _shared_insert(), which prefills and draws one token into `last_token`. Their generate() then force-feeds the think block and samples again; the drawn token is never used, and the only reference to it is an assert. sample() records every drawn token in the repetition and frequency windows first. So a token that was thrown away counts as generated text. For a model that answers without a thinking block, the token predicted after `assistant` IS the answer letter -- it is pushed down by count * freq_penalty + pre_penalty at exactly the position where it is needed, which for a single occurrence is 1.0 + 1.5 = 2.5 logits (Qwen3.5-VL's defaults; its rep_penalty is 1.0 and does nothing). Measured on Qwen3.5-2B, temperature 0, default penalties, before and after, nothing else changed: prompt Q before B after A prompt R before B after A A is the argmax of the unpenalised distribution (0.339 against 0.299 for B), so the answer is no longer the penalised one. Sampler::sample() and sample_greedy() take a record_in_history flag, defaulting to true, so no other model changes behaviour. sample() forwards it to sample_greedy() on the top_k == 1 path, and _shared_insert() passes it through -- both paths have to honour it, or the same prompt answers differently depending on a sampling knob: top_k=1 B (greedy path) top_k=40 A (sampled path) before both flags were in place Co-Authored-By: DeepSeek V4 Flash (Default Thinking) --- src/common/AutoModel/automodel.cpp | 9 +++++---- src/common/AutoModel/modeling_qwen3_5vl.cpp | 10 ++++++++-- src/common/AutoModel/modeling_qwen3_6_moe.cpp | 10 ++++++++-- src/common/modules/sampler.cpp | 14 +++++++++----- src/include/AutoModel/automodel.hpp | 7 +++++-- src/include/modules/sampler.hpp | 12 ++++++++++-- 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index e5cc893f6..7ba78335c 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -391,7 +391,8 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, } } -bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens) { +bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens, + bool record_sampled_token) { this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens); this->_begin_turn(meta_info, tokens); @@ -459,7 +460,7 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); - this->last_token = this->_sample(y, meta_info); + this->last_token = this->_sample(y, meta_info, record_sampled_token); meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); return true; @@ -537,12 +538,12 @@ void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& } } -int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info) { +int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history) { decode_constraint* constraint = meta_info.constraint.get(); const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); // The mask goes to the sampler, which applies it to its own copy: the logits // live in the engine's device-mapped buffer and are never written here. - int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr); + int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr, record_in_history); auto allowed = [this](size_t i) { return i / 32 < this->constraint_mask_.size() && ((this->constraint_mask_[i / 32] >> (i % 32)) & 1u); }; diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index ab3b31dba..c3f5347da 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -290,9 +290,15 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s size_t n = tokens.size(); tokens.resize(n - (this->enable_think ? 2 : 4)); + // The draw below is thrown away: generate() force-feeds the think block and + // samples again. It must not enter the repetition and frequency windows -- + // it was never generated text, and for a model that answers without a + // thinking block the token predicted here IS the answer letter. Counting it + // pushes that letter down by count * freq_penalty + pre_penalty at the position + // where it matters, so the model's preferred answer arrives penalised. bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 9a6aaabf5..6fa07cbc6 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -299,9 +299,15 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, size_t n = tokens.size(); tokens.resize(n - (this->enable_think ? 2 : 4)); + // The draw below is thrown away: generate() force-feeds the think block and + // samples again. It must not enter the repetition and frequency windows -- + // it was never generated text, and for a model that answers without a + // thinking block the token predicted here IS the answer letter. Counting it + // pushes that letter down by count * freq_penalty + pre_penalty at the position + // where it matters, so the model's preferred answer arrives penalised. bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/common/modules/sampler.cpp b/src/common/modules/sampler.cpp index 59c01cb8e..0075f53d0 100644 --- a/src/common/modules/sampler.cpp +++ b/src/common/modules/sampler.cpp @@ -357,7 +357,7 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) { /// \brief Sample the token /// \param x the input buffer /// \return the sampled token -int Sampler::sample_greedy(buffer& x, const std::vector* allowed) { +int Sampler::sample_greedy(buffer& x, const std::vector* allowed, bool record_in_history) { const bool penalties_active = (this->repeat_last_n != 0) && (this->rep_penalty != 1.0f || this->freq_penalty != 0.0f || this->pre_penalty != 0.0f); @@ -447,7 +447,9 @@ int Sampler::sample_greedy(buffer& x, const std::vector* allowed // callers that inspect the surviving candidates still get a consistent view this->top_k_logits.assign(1, logits_t{ best, best_id, 1.0f }); - ring_buffer_update_sparse(best_id); + if (record_in_history) { + ring_buffer_update_sparse(best_id); + } return best_id; } @@ -471,10 +473,10 @@ void Sampler::sampler_allowed_apply(const std::vector* allowed) { } } -int Sampler::sample(buffer& x, const std::vector* allowed) { +int Sampler::sample(buffer& x, const std::vector* allowed, bool record_in_history) { // PRNG is seeded once at construction, no per-call seeding needed if (this->top_k == 1) { - return sample_greedy(x, allowed); + return sample_greedy(x, allowed, record_in_history); } // COPY FROM `x` → `this->logits[]` @@ -515,7 +517,9 @@ int Sampler::sample(buffer& x, const std::vector* allowed) { } int sampled_index = sample_from_probs(); - ring_buffer_update_sparse(sampled_index); + if (record_in_history) { + ring_buffer_update_sparse(sampled_index); + } return sampled_index; } \ No newline at end of file diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index a2921a192..9115f5690 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -268,7 +268,8 @@ class AutoModel { /// \param tokens the tokens to insert /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise - bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt); + bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt, + bool record_sampled_token = true); /// \brief Reject a request that cannot fit in the backend's decode limit /// \param rendered_tokens the length of the rendered prompt @@ -290,8 +291,10 @@ class AutoModel { /// \brief Sample the next token under the request's constraint /// \param logits the logits of the token about to be sampled, masked in place /// \param meta_info the request, carrying the constraint + /// \param record_in_history whether the token joins the sampler's repetition + /// and frequency windows; false for a draw the caller throws away /// \return the sampled token, already reported to the constraint - int _sample(buffer& logits, chat_meta_info_t& meta_info); + int _sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history = true); /// \brief Report a token the model is made to emit rather than samples /// \note For the think preamble some generate()s feed after insert(); the diff --git a/src/include/modules/sampler.hpp b/src/include/modules/sampler.hpp index df7997345..4a174a87e 100644 --- a/src/include/modules/sampler.hpp +++ b/src/include/modules/sampler.hpp @@ -102,7 +102,13 @@ class Sampler{ /// Going through the general path would allocate and partial_sort a /// vocab-sized array of triples and run three full-vocab softmaxes for /// a result that never depends on any of it. - int sample_greedy(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows + /// \note Pass false for a draw the caller throws away. Such a token was + /// never generated text, and counting it pushes the tokens that + /// follow it down -- by count * freq_penalty + pre_penalty each time + /// it occurs. + int sample_greedy(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); void sampler_topk_apply(int k); void sampler_topp_apply(float p); @@ -117,7 +123,9 @@ class Sampler{ /// \param allowed optional bitmask, bit i of word i/32 set when token i may be /// sampled; applied to the sampler's own copy of the logits /// \return the sampled token - int sample(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows; see sample_greedy + int sample(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); /// \brief Push every token the bitmask does not allow to -inf in this->logits void sampler_allowed_apply(const std::vector* allowed); From 969a1b6329c8844468b20892cc921c7815b0a1df Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 11:05:50 +0200 Subject: [PATCH 2/6] feat(server): report logprobs when the request asks for them /v1/chat/completions and /v1/completions wrote a hard-coded `{"logprobs", nullptr}` into their responses. This fills the field from the sampler, in OpenAI's shape, and only when the request asks for it. For a model that answers with a single token -- a classifier, a router, a decision model -- the letter alone says nothing about how sure the model was. sampler.hpp/.cpp capture_if_requested(), log_sum_exp_all() and a table of exp() for every bf16 value; the sampling paths capture once per draw while a request has the sampler armed automodel.hpp/.cpp top_logprob_t, logprobs_entry_t, _request_logprobs_capture() and _record_top_logprobs() modeling_qwen3_5vl records where a token reaches the caller's text, not where it was drawn rest_handler.cpp reads the request, builds both response shapes One entry per token of message.content, in the same order, so a client that pairs entries with tokens stays aligned. Recording happens where the token is written to the response, which is what keeps the two in step: a token the engine filters out gets no entry, and neither does EOS. A 17-token answer comes back with 16 entries. An answer that carries a tool call gets null, because message.content is empty and there are no tokens to describe. logprob is a real log-probability -- logit minus logsumexp over the WHOLE vocabulary, after the penalties and before the temperature. It is NOT renormalised over the entries listed, so those do not sum to 1; the rest of the vocabulary lies outside the list. The logsumexp is exact and cheap: the logits arrive as bf16 from the engine, so there are only 65536 possible values, and a table of exp() for each of them (512 KB, fits in L2) replaces a quarter of a million exp() calls with as many reads. Values the penalties moved off the bf16 grid take the exact call. The result differs from the shifted computation by about one float ulp, which is rounding, not approximation. For the alternatives, N <= top_k needs no sort at all: top_k_logits is already sorted by logit with the same tie-break, so its beginning is the top-n. Cost, wall clock on Qwen3.5-2B, 300 tokens, nothing else running, platform_profile=balanced. The runs drift by about 2 % on their own, so this is an upper bound rather than a figure: without logprobs 10.55 10.46 10.35 s with logprobs 10.57 s at most about 1 % The sampler is disarmed at the start of every request, before anything that can return early; it is armed only in Qwen3.5-VL's generate(), and only when the request asks for logprobs. The two request shapes differ, as they do upstream: chat takes `logprobs` as a boolean plus `top_logprobs` (0-20), completions takes `logprobs` as an integer 0-5 and has no `top_logprobs`. For completions 0 is a valid request: it asks for the drawn token's own logprob with no alternatives. With a decoding constraint (response_format, tool_choice) the capture sees the masked logits, the ones the draw used, so logprob is normalised over the tokens the constraint allows. Alternatives the constraint forbids sit at -inf and are not reported, since -inf has no JSON number. Limitations, unchanged by this patch: - only Qwen3.5-VL reports; every other model still returns null - streaming responses carry no logprobs - with think:true the answer gets null: the tokens of the reasoning block land in reasoning_content, not in content Seen while working on this, not touched here: the AVX2 argmax in sample_greedy() breaks ties within a lane by the lowest index but merges lanes by the lowest lane, so an exact tie at the top can pick a higher id than the scalar path would. Co-Authored-By: DeepSeek V4 Flash (Default Thinking) --- src/common/AutoModel/automodel.cpp | 84 +++++++++++++ src/common/AutoModel/modeling_qwen3_5vl.cpp | 13 ++ src/common/modules/sampler.cpp | 133 ++++++++++++++++++++ src/include/AutoModel/automodel.hpp | 48 +++++++ src/include/modules/sampler.hpp | 32 +++++ src/server/rest_handler.cpp | 127 ++++++++++++++++++- 6 files changed, 435 insertions(+), 2 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 7ba78335c..ab4161946 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -393,6 +393,13 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens, bool record_sampled_token) { + // Disarmed here, before anything that can return early: every request goes + // through this function, and a request that does not ask for logprobs must + // not inherit the arming of an earlier one. Arming happens where the + // recording happens, which for Qwen3.5-VL is its own generate(). + this->sampler->capture_top_logprobs = false; + this->sampler->captured = false; + this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens); this->_begin_turn(meta_info, tokens); @@ -634,6 +641,83 @@ std::shared_ptr AutoModel::make_constraint(const chat_constra return chat_make_constraint(this->chat_format, effective, vocab, engine); } +/// True when nlohmann::json can serialise this string. +/// +/// A token can be the start of a multi-byte sequence: decoded on its own it is +/// not valid text, and json::dump() throws on it. Asking the same library that +/// would throw is exact, where a hand-written UTF-8 check is only an +/// approximation of what it accepts. +static bool is_serialisable(const std::string& s) { + try { + (void)json(s).dump(); + return true; + } catch (const std::exception&) { + return false; + } +} + +void AutoModel::_request_logprobs_capture(chat_meta_info_t& meta_info) { + // Once per request: the sampler stays armed for all of it, because every + // token that reaches the caller's text needs its own distribution. + if (!meta_info.want_logprobs || meta_info.logprobs_armed) { + return; + } + meta_info.logprobs_armed = true; + this->sampler->captured = false; + this->sampler->capture_count = meta_info.top_logprobs_count; + this->sampler->capture_top_logprobs = true; +} + +void AutoModel::_record_top_logprobs(chat_meta_info_t& meta_info, int sampled_token) { + if (!meta_info.want_logprobs || !this->sampler || !this->sampler->captured) { + return; + } + + // The sampler's logits are not touched by the sampling -- only + // top_k_logits is -- so the value here and the captured logsumexp come + // from the same moment. + const float lse = this->sampler->captured_logsumexp; + auto entry_for = [&](int id) { + top_logprob_t e; + e.token_id = id; + // True log-probability: logit - logsumexp over the WHOLE vocabulary, + // not one renormalised over the entries shown. The two differ whenever + // top_k is smaller than the vocabulary, which is always. + e.logprob = this->sampler->logits[id] - lse; + const std::string raw = this->tokenizer + ? this->tokenizer->run_time_decoder(id) + : std::string(); + // Decode here, not in the REST handler: that one has no tokenizer. + // As unsigned: `char` is signed on x86-64, so a byte above 0x7F would + // land here as a negative number, and OpenAI defines `bytes` as + // integers 0-255. A client rebuilding the text from them would fail on + // every umlaut, every CJK character, every emoji. + for (unsigned char c : raw) { + e.bytes.push_back(c); + } + e.token = is_serialisable(raw) ? raw : std::string(); + return e; + }; + + logprobs_entry_t entry; + if (sampled_token >= 0 && sampled_token < static_cast(this->sampler->logits.size())) { + entry.chosen = entry_for(sampled_token); + } + const auto& src = this->sampler->captured_top_k; + const size_t limit = std::min(meta_info.top_logprobs_count, src.size()); + entry.alternatives.reserve(limit); + for (size_t i = 0; i < limit; i++) { + // A decoding constraint pushes the tokens it forbids to -inf. They + // were never drawable, and -inf has no JSON number; the list is sorted, + // so the first one ends it. + if (!std::isfinite(src[i].logits)) { + break; + } + entry.alternatives.push_back(entry_for(src[i].token_id)); + } + meta_info.logprobs_entries.push_back(std::move(entry)); +} + std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { std::vector sampled_tokens; std::string result; diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index c3f5347da..dbea47718 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -341,6 +341,11 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s token_str = this->tokenizer->run_time_decoder(198); result += token_str; // First token of the reasoning block. + // No capture here: this draw is the first token of the reasoning + // block, and that lands in reasoning_content, not in content. OpenAI's + // logprobs.content describes the tokens of message.content, so a + // report from here would name a token the caller cannot find in the + // text it was given. Requests with think:true get logprobs: null. sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; } @@ -369,6 +374,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s buffer y = this->lm_engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); + this->_request_logprobs_capture(meta_info); sampled_token = this->_sample(y, meta_info); } this->total_tokens++; @@ -377,6 +383,12 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; + // Recorded where the token reaches the caller's text, not where it was + // drawn, and under the same filter the loop below uses: a token the engine + // would have dropped must not get an entry either. + if (this->is_normal_token(last_sampled_token)) { + this->_record_top_logprobs(meta_info, last_sampled_token); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); @@ -403,6 +415,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s std::string token_str = this->tokenizer->run_time_decoder(sampled_token); os << token_str << std::flush; result += token_str; + this->_record_top_logprobs(meta_info, sampled_token); } this->profiler_list[TKOEN_DECODE_TIME].stop(1); this->token_history.push_back(sampled_token); diff --git a/src/common/modules/sampler.cpp b/src/common/modules/sampler.cpp index 0075f53d0..dacc36f48 100644 --- a/src/common/modules/sampler.cpp +++ b/src/common/modules/sampler.cpp @@ -8,6 +8,9 @@ #include "modules/sampler.hpp" +#include +#include +#include #include #include // for std::sort #include @@ -354,6 +357,108 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) { this->total_tokens++; } +namespace { +/// exp() for every value a bf16 can hold. +/// +/// The logits arrive as bf16 from the engine and the widening to fp32 is +/// exact, so this->logits can only take 65536 distinct values. Looking the +/// exponential up is the same number std::exp would return -- precomputed, +/// not approximated -- and it turns a quarter of a million exp() calls into as +/// many reads. 512 KB, which fits in L2. +const std::array& bf16_exp_table() { + static const std::array table = [] { + std::array t{}; + for (uint32_t b = 0; b < 65536; b++) { + const uint32_t bits = b << 16; + float f; + std::memcpy(&f, &bits, sizeof(f)); + t[b] = std::exp(static_cast(f)); + } + return t; + }(); + return table; +} +} // namespace + +float Sampler::log_sum_exp_all() const { + if (this->logits.empty()) { + return 0.0f; + } + const auto& table = bf16_exp_table(); + double sum = 0.0; + for (int i = 0; i < this->in_features; i++) { + uint32_t bits; + std::memcpy(&bits, &this->logits[i], sizeof(bits)); + // The low 16 bits are non-zero for anything the penalties moved off + // the bf16 grid; those take the exact call. The branch predicts well + // and costs nothing measurable. + sum += (bits & 0xFFFFu) != 0u + ? std::exp(static_cast(this->logits[i])) + : table[bits >> 16]; + } + if (std::isfinite(sum) && sum > 0.0) { + return static_cast(std::log(sum)); + } + // Nothing on the bf16 grid produced a usable sum -- reachable only for a + // logit beyond about +-700, where exp overflows a double. Fall back to the + // shifted form. Note what it does NOT cover: +-inf and NaN are not + // meaningful here either way, and a logit of +inf would come back as 0. + float max_logit = -std::numeric_limits::infinity(); + for (float v : this->logits) { + if (v > max_logit) { + max_logit = v; + } + } + if (!std::isfinite(max_logit)) { + return 0.0f; + } + double shifted = 0.0; + for (float v : this->logits) { + shifted += std::exp(static_cast(v) - max_logit); + } + return max_logit + static_cast(std::log(shifted)); +} + +void Sampler::capture_if_requested() { + if (!this->capture_top_logprobs) { + return; + } + // Own top-N by logit, taken from the full logits rather than from + // top_k_logits: the report should not depend on the sampler's top_k, and + // the greedy path keeps only one entry there. + const int n = std::min(this->capture_count, this->in_features); + this->captured_top_k.clear(); + this->captured_logsumexp = this->log_sum_exp_all(); + if (n > 0 && static_cast(n) <= this->top_k_logits.size()) { + // top_k_logits is already sorted by logit with the same tie-break, so + // its beginning IS the top-n. No second sort, no vocab-sized index + // vector -- which matters because this runs once per token. + this->captured_top_k.assign(this->top_k_logits.begin(), + this->top_k_logits.begin() + n); + } else if (n > 0) { + std::vector idx(this->in_features); + std::iota(idx.begin(), idx.end(), 0); + // Same order as sampler_topk_apply: bf16 logits over 100k+ entries + // are often equal, partial_sort is not stable, and a tie broken + // differently would put another token at the top than the one the + // draw would have taken. + std::partial_sort(idx.begin(), idx.begin() + n, idx.end(), + [this](int a, int b) { + if (this->logits[a] != this->logits[b]) { + return this->logits[a] > this->logits[b]; + } + return a < b; + }); + this->captured_top_k.reserve(n); + for (int i = 0; i < n; i++) { + this->captured_top_k.push_back(logits_t{ this->logits[idx[i]], idx[i], 0.0f }); + } + } + // Stays armed: every token that reaches the caller's text needs its own + // distribution. `captured` marks that THIS draw has one. + this->captured = true; +} + /// \brief Sample the token /// \param x the input buffer /// \return the sampled token @@ -447,6 +552,32 @@ int Sampler::sample_greedy(buffer& x, const std::vector* allowed // callers that inspect the surviving candidates still get a consistent view this->top_k_logits.assign(1, logits_t{ best, best_id, 1.0f }); + if (this->capture_top_logprobs) { + // log_sum_exp_all() reads this->logits, which the branch above only + // fills when penalties or a constraint mask are active -- and then + // penalised and masked, as the draw saw them, so it must not be + // overwritten. Otherwise fill it here the same way sample() does: the + // logits come back in device-mapped memory where element-at-a-time + // reads are punishing, so this pulls eight at a time. + if (!penalties_active && allowed == nullptr) { + #if USEAVX2 + const int simd_width = 8; + int i = 0; + for (; i <= in_features - simd_width; i += simd_width) { + __m128i bf16_vals_x = _mm_loadu_si128((__m128i*)&x[i]); + _mm256_storeu_ps(&this->logits[i], bf16o_fp32(bf16_vals_x)); + } + for (; i < in_features; i++) { + this->logits[i] = x[i]; + } + #else + for (int i = 0; i < in_features; i++) { + this->logits[i] = x[i]; + } + #endif + } + this->capture_if_requested(); + } if (record_in_history) { ring_buffer_update_sparse(best_id); } @@ -502,6 +633,8 @@ int Sampler::sample(buffer& x, const std::vector* allowed, bool sampler_penalty_apply_sparse(); sampler_allowed_apply(allowed); sampler_topk_apply(this->top_k); + + this->capture_if_requested(); if (this->use_optimized_sampling) { softmax_with_topp_minp(this->top_p, this->min_p); sampler_temp_apply(this->temperature); diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 9115f5690..e04d84da1 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -85,8 +85,42 @@ inline std::string stop_reason_to_string(stop_reason_t reason){ } } +/// One entry of the reported distribution. +/// +/// A struct of its own rather than Sampler::logits_t: this header should not +/// have to know about the sampler. +struct top_logprob_t { + int token_id = -1; + std::string token; ///< decoded; empty when the bytes are not valid UTF-8 + std::vector bytes; ///< the token's raw bytes, as OpenAI reports them + float logprob = 0.0f; ///< logit - logsumexp over the whole vocabulary +}; + +/// One token of the answer, with the distribution it was drawn from. +/// +/// The chosen token has its own entry, independent of how many alternatives +/// were asked for: a draw at rank 3 must still be named when the caller asked +/// for one alternative. +struct logprobs_entry_t { + top_logprob_t chosen; + std::vector alternatives; +}; + struct chat_meta_info_t { int max_prefill_len; + + /// Whether and how many alternatives the caller asked for. Every token + /// that reaches message.content is described, in that order. + bool want_logprobs = false; + int top_logprobs_count = 0; + + /// One entry per token that reached the caller's text, in that order -- + /// the same order and the same count as the tokens of message.content. + /// Each carries the token that was drawn plus its alternatives. + std::vector logprobs_entries; + + /// Guards the once-per-request rule for arming the sampler. + bool logprobs_armed = false; int prompt_tokens; // whole prompt, cached prefix included int cached_prompt_tokens; // subset of prompt_tokens served from the KV cache int generated_tokens; @@ -263,6 +297,20 @@ class AutoModel { return backend_cap == 0 ? MAX_L : std::min(MAX_L, backend_cap); } + /// \brief Ask the sampler to capture a distribution for this request + /// \param meta_info carries the request's wish + /// \note Call before a draw that will also be counted. A draw whose token + /// is thrown away must not be captured -- the numbers would describe + /// a position the answer does not come from. + void _request_logprobs_capture(chat_meta_info_t& meta_info); + + /// \brief Append one entry for a token that reached the caller's text + /// \param meta_info carries it to the REST handler + /// \param sampled_token the token the sampler just chose + /// \note Call where the token is written to the response, not where it was + /// drawn: a token the engine filters out, and EOS, get no entry. + void _record_top_logprobs(chat_meta_info_t& meta_info, int sampled_token); + /// \brief Insert tokens into the model /// \param meta_info the meta information of the chat /// \param tokens the tokens to insert diff --git a/src/include/modules/sampler.hpp b/src/include/modules/sampler.hpp index 4a174a87e..6f71934c1 100644 --- a/src/include/modules/sampler.hpp +++ b/src/include/modules/sampler.hpp @@ -51,6 +51,29 @@ class Sampler{ int in_features; std::vector counters; logits_list_t top_k_logits; + + /// Set by the model for a request that asked for logprobs, and cleared at + /// the start of every request whether it asked or not. It stays set for + /// the whole request: every token that reaches the caller's text needs its + /// own distribution. A request that does not ask never arms it, so it pays + /// nothing. + bool capture_top_logprobs = false; + /// How many entries to keep. Bounded by the caller, OpenAI allows 20. + int capture_count = 0; + + /// The top entries by logit at the moment of the capture, and the + /// log-sum-exp of the WHOLE vocabulary at the same moment. + /// + /// Sorted by logit and cut to the count the caller asked for, so it is NOT + /// the list the draw used and the drawn token need not be in it: with + /// top_logprobs: 1 and a draw at rank 3 it is not. That is what `chosen` + /// is for. Entries behind the top-p and min-p cut are listed although they + /// had no chance of being drawn. + logits_list_t captured_top_k; + /// Set by the capture, cleared when a request arms it. Guards against + /// reading values that belong to an earlier draw or request. + bool captured = false; + float captured_logsumexp = 0.0f; float rep_penalty; float freq_penalty; float pre_penalty; @@ -110,6 +133,15 @@ class Sampler{ /// it occurs. int sample_greedy(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); + /// \brief Capture the current top-k if a request asked for it + /// \note Called by the sampling paths right after the top-k is settled and + /// before the temperature. Stays armed until the next request + /// disarms it; `captured` marks that this draw has values. + void capture_if_requested(); + /// \brief log-sum-exp over the whole vocabulary + /// \note Reads this->logits, which the sampling paths have filled by then + float log_sum_exp_all() const; + void sampler_topk_apply(int k); void sampler_topp_apply(float p); void sampler_minp_apply(float p); diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index b37acca08..afd206eb5 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -612,6 +612,124 @@ void RestHandler::configure_chat_engine_parameters(const json& options, const js } } +/// Read the caller's logprobs wish into meta_info, for the chat shape. +/// +/// OpenAI: `logprobs` is a boolean, and `top_logprobs` (0-20) says how many +/// alternatives to list. Without `top_logprobs`, only the chosen token's own +/// logprob is reported and the alternatives stay empty. +static void read_logprobs_request(const json& request, chat_meta_info_t& meta_info) { + if (!request.contains("logprobs") || !request["logprobs"].is_boolean()) { + return; + } + meta_info.want_logprobs = request["logprobs"].get(); + if (!meta_info.want_logprobs) { + return; + } + if (request.contains("top_logprobs") && request["top_logprobs"].is_number_integer()) { + meta_info.top_logprobs_count = std::clamp(request["top_logprobs"].get(), 0, 20); + } +} + +/// The same for the legacy completions shape, where `logprobs` is an INTEGER: +/// N asks for the N most likely tokens, 0-5, and there is no `top_logprobs`. +/// Zero is a valid request -- it asks for the drawn token's own logprob with +/// no alternatives -- and the drawn token is always reported. +/// +/// A boolean is not accepted here: the SDKs send the integer, and a generous +/// dialect would become a contract that clients come to rely on. +static void read_legacy_logprobs_request(const json& request, chat_meta_info_t& meta_info) { + if (!request.contains("logprobs") || !request["logprobs"].is_number_integer()) { + return; + } + const int n = request["logprobs"].get(); + meta_info.want_logprobs = n >= 0; + meta_info.top_logprobs_count = std::clamp(n, 0, 5); +} + +static json top_logprob_to_json(const top_logprob_t& e) { + return json{ + {"token", e.token}, + {"token_id", e.token_id}, + {"bytes", e.bytes}, + {"logprob", e.logprob}, + }; +} + +/// The OpenAI logprobs object for the answer. +/// +/// One entry per token of message.content, in the same order, so a client that +/// pairs entries with tokens stays aligned. +/// +/// `logprob` is the natural logarithm of the token's probability over the +/// WHOLE vocabulary, after the repetition, frequency and presence penalties and +/// before the temperature, top-k, top-p and min-p. The entries listed +/// therefore do NOT sum to 1 -- the rest of the vocabulary lies outside the +/// list. Comparing two entries gives the same answer whether you use logprob +/// or the raw logit, because both differ by the same constant. +/// +/// A request with think:true gets null: the tokens of the reasoning block land +/// in reasoning_content, not in content, and this field describes content. +static json logprobs_from_meta(const chat_meta_info_t& meta_info) { + if (!meta_info.want_logprobs || meta_info.logprobs_entries.empty()) { + return nullptr; + } + json content = json::array(); + for (const auto& entry : meta_info.logprobs_entries) { + json alternatives = json::array(); + for (const auto& e : entry.alternatives) { + alternatives.push_back(top_logprob_to_json(e)); + } + json chosen = top_logprob_to_json(entry.chosen); + chosen["top_logprobs"] = alternatives; + content.push_back(std::move(chosen)); + } + return json{{"content", content}}; +} + +/// The same information in the legacy completions shape, which pairs one entry +/// per token rather than nesting them. +static json legacy_logprobs_from_meta(const chat_meta_info_t& meta_info) { + if (!meta_info.want_logprobs || meta_info.logprobs_entries.empty()) { + return nullptr; + } + json tokens = json::array(); + json token_logprobs = json::array(); + json top_logprobs = json::array(); + json text_offset = json::array(); + int offset = 0; + for (const auto& entry : meta_info.logprobs_entries) { + tokens.push_back(entry.chosen.token); + token_logprobs.push_back(entry.chosen.logprob); + json per_token = json::object(); + for (const auto& e : entry.alternatives) { + // This shape keys by the token string and has no `bytes`, so a + // token that is not valid UTF-8 on its own would be keyed by "" + // and overwrite the previous one. Those are left out. + if (e.token.empty()) { + continue; + } + per_token[e.token] = e.logprob; + } + top_logprobs.push_back(std::move(per_token)); + text_offset.push_back(offset); + // OpenAI counts characters here. Counting the bytes that are not + // continuation bytes gives exactly that, and it stays right when one + // character is split across two tokens: it is counted once, at the + // token carrying its first byte. + for (int b : entry.chosen.bytes) { + if ((b & 0xC0) != 0x80) { + offset++; + } + } + } + return json{ + {"tokens", tokens}, + {"token_logprobs", token_logprobs}, + {"top_logprobs", top_logprobs}, + {"text_offset", text_offset}, + }; +} + json RestHandler::build_nstream_response(const std::string& response_text, chat_meta_info_t& meta_info, const chat_parse_options& parse_options) { const chat_format_spec& spec = auto_chat_engine->get_chat_format(); @@ -659,7 +777,10 @@ json RestHandler::build_nstream_response(const std::string& response_text, chat_ { {"index", 0}, {"message", message}, - {"logprobs", nullptr}, + // A tool call puts nothing in message.content, so there are no + // tokens for a logprobs object to describe. Same for a mixed + // answer that carries both text and a call. + {"logprobs", is_tool_call ? json(nullptr) : logprobs_from_meta(meta_info)}, {"finish_reason", stop_reason_to_string(meta_info.stop_reason)} } }); @@ -1208,6 +1329,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, // see if we can use prompt cache chat_meta_info_t meta_info; + read_logprobs_request(request, meta_info); bool can_use_prompt_cache = false; if (auto_chat_engine->single_turn) { // reject multi-turn requests: any assistant message in the history @@ -1599,6 +1721,7 @@ void RestHandler::handle_openai_completion(const json& request, configure_chat_engine_parameters(options, request); chat_meta_info_t meta_info; + read_legacy_logprobs_request(request, meta_info); meta_info.max_prefill_len = this->prefill_chunk_len; lm_uniform_input_t uniformed_input; uniformed_input.requested_max_new_tokens = @@ -1692,7 +1815,7 @@ void RestHandler::handle_openai_completion(const json& request, { {"text", response_text}, {"index", 0}, - {"logprobs", nullptr}, + {"logprobs", legacy_logprobs_from_meta(meta_info)}, {"finish_reason", stop_reason_to_string(meta_info.stop_reason)} } })}, From a326a3131a17e1945cdfdb110148b2ccfa39f543 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 12:52:56 +0200 Subject: [PATCH 3/6] fix(qwen3.5): stop at max_tokens instead of one token past it Qwen3_5VL::generate() draws the first token before its decode loop and checked the limit only at the bottom of the loop, after the next draw. So every request that ran into max_tokens got one token more than it asked for, and paid one more forward() for it: max_tokens=1 returned "A\n" with completion_tokens 2. The loop now checks the limit first, before it draws. Its shape follows _shared_generate(): the context limit is checked inside the loop, and reaching max_tokens still wins over it, so finish_reason is unchanged. Measured on Qwen3.5-2B (NPU2), 20 independent one-token prompts, three alternating rounds against main: median wall clock 604 -> 574 ms per request, completion_tokens 2 -> 1. The first token of every answer and two 60-token generations are identical to main. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/modeling_qwen3_5vl.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index dbea47718..9f49daf57 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -395,7 +395,17 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s reason = MAX_LENGTH_REACHED; return result; } - while (this->total_tokens < this->MAX_L){ + while (true){ + // The first token is drawn before the loop, so the limit can already be + // reached here -- with max_tokens=1 it always is. Checked only after a + // draw, the loop would draw and emit one token past the limit. + if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + break; + } + if (this->total_tokens >= this->MAX_L){ + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -425,10 +435,6 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s break; } meta_info.generated_tokens++; - if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ - reason = MAX_LENGTH_REACHED; - break; - } } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; From 2e3171a47e97debfbe1152eba3623be34d283ca3 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 12:54:51 +0200 Subject: [PATCH 4/6] perf(qwen3.5): prefill the empty think block for single-turn models Without thinking, the chat template ends with an empty think block. Qwen3_5VL::insert() cut those four tokens off before the prefill, so the checkpoint lands where the next turn of a conversation diverges, and generate() then fed them in with one forward() each. Only after those four decode steps came the draw for the answer's first token. A single-turn model never returns to that checkpoint. For such a model the block is now prefilled with the prompt, and the token _shared_insert() draws from the prefill logits is the answer's first token. Multi-turn models keep the old path, because there the checkpoint pays off. So does thinking, where the model writes the block itself. Under a decoding constraint the draw is kept as well. The constraint starts from the prompt tail, which now ends behind the closed block, and it has already accepted the token; nothing is rolled back. The catalog's "single-turn" label now also sets AutoModel::single_turn. Until now only three classes set the flag themselves, all with the label, so nothing changes for them. It is only ever switched on, never off. Measured on Qwen3.5-2B (NPU2) with the label set, 20 independent one-token prompts, three alternating rounds against main: median wall clock 604 -> 486 ms per request. Prefill time is unchanged (430 vs 431 ms); the four tokens it gains cost nothing measurable. The numbers are not bit-identical. The template's tokens are exactly the four ids generate() used to feed ([248068, 271, 248069, 271]), but they now go through the prefill kernels instead of four decode steps. Against main with the discarded-sample fix (#776) 1 of 20 answers differs, and that was an exact tie (A and C both at logprob -1.939). One of two 60-token generations diverges after 90 characters. With logprobs applied on top, the log-odds between options move by 0.25 nats median, 0.5 at most. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/automodel.cpp | 10 +++++++- src/common/AutoModel/modeling_qwen3_5vl.cpp | 26 +++++++++++++++++--- src/include/AutoModel/modeling_qwen3_5vl.hpp | 4 +++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index ab4161946..108dc8f15 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -232,7 +232,15 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int } void AutoModel::_shared_initialize_model_state( - std::string model_path, json, int context_length) { + std::string model_path, json model_info, int context_length) { + // The catalog marks a model single-turn with a label, and some classes set + // the flag themselves. Only ever switched on here, so an entry without the + // label does not override a class that knows it is single-turn. + if (model_info.contains("label") && + std::find(model_info["label"].begin(), model_info["label"].end(), + "single-turn") != model_info["label"].end()) { + this->single_turn = true; + } this->model_path = std::move(model_path); header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index 9f49daf57..eb0b9f4ef 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -287,8 +287,19 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic } - size_t n = tokens.size(); - tokens.resize(n - (this->enable_think ? 2 : 4)); + // The template ends with the opening of the assistant turn and, without + // thinking, an empty think block. Normally the prefill stops before that + // block, so the checkpoint below sits where the next turn of the + // conversation will diverge, and generate() feeds the block one forward() + // at a time. A single-turn model never comes back to that checkpoint, so + // the four extra decode steps buy nothing: prefill the block with the + // prompt, and the token drawn from the prefill logits is already the first + // token of the answer. + this->think_block_prefilled = this->single_turn && !this->enable_think; + if (!this->think_block_prefilled) { + size_t n = tokens.size(); + tokens.resize(n - (this->enable_think ? 2 : 4)); + } // The draw below is thrown away: generate() force-feeds the think block and // samples again. It must not enter the repetition and frequency windows -- @@ -322,7 +333,16 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[TKOEN_DECODE_TIME].reset(); std::string token_str; int sampled_token; - if(this->enable_think) { + if (this->think_block_prefilled) { + // insert() prefilled the empty think block; _shared_insert drew the + // answer's first token from those logits, under the constraint. That + // draw is kept, so nothing is rolled back: the constraint started + // behind the closed block (the prompt tail ends with it) and has + // already accepted the token. + sampled_token = this->last_token; + meta_info.insert_sample_pending = false; + } + else if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); this->_force_token(meta_info, think_start_id); diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index c7d55af10..17b00e5f0 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -28,6 +28,10 @@ class Qwen3_5VL : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; + /// Set by insert() when it prefilled the empty think block together with + /// the prompt; generate() then takes the first token from that draw + /// instead of feeding the block one forward() at a time. + bool think_block_prefilled = false; void setup_tokenizer(std::string model_path); From 109ed15cabc470237ba78b0e0380fc3195173cce Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 15:09:47 +0200 Subject: [PATCH 5/6] fix: prefill again when a request matches the whole history _shared_insert() skips the prefix a request shares with token_history. If the request matches the history completely, nothing is left to prefill, _chunked_insert() returns an empty buffer, and Sampler::sample() reads it: a segfault at address 0. With the empty think block prefilled, a single-turn Qwen3.5 request puts exactly its own tokens into token_history. The first generated token is never added there. On its own this change cannot hit the bug: with max_tokens=1 the decode loop still draws one more token and appends it. Together with the max_tokens fix from the companion PR the loop no longer runs, and the same request sent twice crashed the server on the second one. Reproduced under gdb: the crashing request logs no "Prefill chunk" line. A complete match is now treated like a divergence: clear and prefill everything. Stepping back by one token is not an option, because the linear attention layers carry recurrent state that cannot be rewound. With both changes applied, eight identical requests in a row pass, and answers and confidences are unchanged. The extra prefill is not measurable, because a single-turn model cleared its context on these requests before anyway. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/automodel.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 108dc8f15..dfe0655ac 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -436,7 +436,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to break; } } - if (skip_count != idx) { + // A request that matches the whole history leaves nothing to prefill, and + // then there are no logits to draw from: the sampler would read an empty + // buffer. It happens when a request repeats the previous one exactly and + // that answer added nothing to the history -- a single-turn Qwen3.5 model + // with the think block prefilled and max_tokens=1. Recurrent state cannot + // be stepped back by one token, so treat it like a divergence. + if (skip_count != idx || skip_count == tokens.size()) { if (is_server_mode) { header_print("FLM", "Conversation context diverged from cache, clearing context..."); } From 1d2232603b69825eb8657073748b2928a37f6297 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 14:52:32 +0200 Subject: [PATCH 6/6] fix(qwen3.5): capture logprobs on the prefill draw when the think block is prefilled Builds on #777 (logprobs, which contains #776) and on the single-turn prefill change. With the empty think block prefilled, the token _shared_insert() draws is the answer's first token. It must be counted in the repetition window, and a logprobs request must capture it. Until now generate() armed the capture only on the old path, before the draw that follows the forced block, so a single-turn model returned no logprobs at all. _shared_insert() takes capture_logprobs and arms the sampler right before its own draw, after the disarming at its top. Qwen3_5VL::insert() passes it, together with record_sampled_token, exactly when the block was prefilled. Measured on Qwen3.5-2B (NPU2), the same binary with and without the "single-turn" label, 20 one-token prompts: 572 -> 455 ms median per request. Both report one logprobs entry per answer token, the chosen entry matches the content, and both return null without logprobs and with think:true. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/automodel.cpp | 7 ++++++- src/common/AutoModel/modeling_qwen3_5vl.cpp | 19 +++++++++++-------- src/include/AutoModel/automodel.hpp | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index dfe0655ac..5849faa39 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -400,7 +400,7 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, } bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens, - bool record_sampled_token) { + bool record_sampled_token, bool capture_logprobs) { // Disarmed here, before anything that can return early: every request goes // through this function, and a request that does not ask for logprobs must // not inherit the arming of an earlier one. Arming happens where the @@ -481,6 +481,11 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); + // A caller whose first token of the answer is this draw arms the capture + // here, after the disarming above -- a thrown-away draw never is. + if (capture_logprobs && record_sampled_token) { + this->_request_logprobs_capture(meta_info); + } this->last_token = this->_sample(y, meta_info, record_sampled_token); meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index eb0b9f4ef..285477c40 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -301,15 +301,18 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s tokens.resize(n - (this->enable_think ? 2 : 4)); } - // The draw below is thrown away: generate() force-feeds the think block and - // samples again. It must not enter the repetition and frequency windows -- - // it was never generated text, and for a model that answers without a - // thinking block the token predicted here IS the answer letter. Counting it - // pushes that letter down by count * freq_penalty + pre_penalty at the position - // where it matters, so the model's preferred answer arrives penalised. + // Unless the think block was prefilled, the draw below is thrown away: + // generate() force-feeds the block and samples again. Such a draw must not + // enter the repetition and frequency windows -- it was never generated + // text, and for a model that answers without a thinking block the token + // predicted here IS the answer letter. Counting it pushes that letter down + // by count * freq_penalty + pre_penalty at the position where it matters. + // With the block prefilled the draw is the answer's first token, so it is + // counted, and it is the draw a logprobs request has to capture. + const bool real_draw = this->think_block_prefilled; bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, real_draw, real_draw) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, real_draw, real_draw); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index e04d84da1..79c7b5324 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -317,7 +317,7 @@ class AutoModel { /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt, - bool record_sampled_token = true); + bool record_sampled_token = true, bool capture_logprobs = false); /// \brief Reject a request that cannot fit in the backend's decode limit /// \param rendered_tokens the length of the rendered prompt