diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index e5cc893f6..5849faa39 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -232,7 +232,15 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int } void AutoModel::_shared_initialize_model_state( - std::string model_path, json, int context_length) { + std::string model_path, json model_info, int context_length) { + // The catalog marks a model single-turn with a label, and some classes set + // the flag themselves. Only ever switched on here, so an entry without the + // label does not override a class that knows it is single-turn. + if (model_info.contains("label") && + std::find(model_info["label"].begin(), model_info["label"].end(), + "single-turn") != model_info["label"].end()) { + this->single_turn = true; + } this->model_path = std::move(model_path); header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); @@ -391,7 +399,15 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, } } -bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens) { +bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens, + bool record_sampled_token, bool capture_logprobs) { + // Disarmed here, before anything that can return early: every request goes + // through this function, and a request that does not ask for logprobs must + // not inherit the arming of an earlier one. Arming happens where the + // recording happens, which for Qwen3.5-VL is its own generate(). + this->sampler->capture_top_logprobs = false; + this->sampler->captured = false; + this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens); this->_begin_turn(meta_info, tokens); @@ -420,7 +436,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to break; } } - if (skip_count != idx) { + // A request that matches the whole history leaves nothing to prefill, and + // then there are no logits to draw from: the sampler would read an empty + // buffer. It happens when a request repeats the previous one exactly and + // that answer added nothing to the history -- a single-turn Qwen3.5 model + // with the think block prefilled and max_tokens=1. Recurrent state cannot + // be stepped back by one token, so treat it like a divergence. + if (skip_count != idx || skip_count == tokens.size()) { if (is_server_mode) { header_print("FLM", "Conversation context diverged from cache, clearing context..."); } @@ -459,7 +481,12 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); - this->last_token = this->_sample(y, meta_info); + // A caller whose first token of the answer is this draw arms the capture + // here, after the disarming above -- a thrown-away draw never is. + if (capture_logprobs && record_sampled_token) { + this->_request_logprobs_capture(meta_info); + } + this->last_token = this->_sample(y, meta_info, record_sampled_token); meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); return true; @@ -537,12 +564,12 @@ void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& } } -int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info) { +int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history) { decode_constraint* constraint = meta_info.constraint.get(); const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); // The mask goes to the sampler, which applies it to its own copy: the logits // live in the engine's device-mapped buffer and are never written here. - int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr); + int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr, record_in_history); auto allowed = [this](size_t i) { return i / 32 < this->constraint_mask_.size() && ((this->constraint_mask_[i / 32] >> (i % 32)) & 1u); }; @@ -633,6 +660,83 @@ std::shared_ptr AutoModel::make_constraint(const chat_constra return chat_make_constraint(this->chat_format, effective, vocab, engine); } +/// True when nlohmann::json can serialise this string. +/// +/// A token can be the start of a multi-byte sequence: decoded on its own it is +/// not valid text, and json::dump() throws on it. Asking the same library that +/// would throw is exact, where a hand-written UTF-8 check is only an +/// approximation of what it accepts. +static bool is_serialisable(const std::string& s) { + try { + (void)json(s).dump(); + return true; + } catch (const std::exception&) { + return false; + } +} + +void AutoModel::_request_logprobs_capture(chat_meta_info_t& meta_info) { + // Once per request: the sampler stays armed for all of it, because every + // token that reaches the caller's text needs its own distribution. + if (!meta_info.want_logprobs || meta_info.logprobs_armed) { + return; + } + meta_info.logprobs_armed = true; + this->sampler->captured = false; + this->sampler->capture_count = meta_info.top_logprobs_count; + this->sampler->capture_top_logprobs = true; +} + +void AutoModel::_record_top_logprobs(chat_meta_info_t& meta_info, int sampled_token) { + if (!meta_info.want_logprobs || !this->sampler || !this->sampler->captured) { + return; + } + + // The sampler's logits are not touched by the sampling -- only + // top_k_logits is -- so the value here and the captured logsumexp come + // from the same moment. + const float lse = this->sampler->captured_logsumexp; + auto entry_for = [&](int id) { + top_logprob_t e; + e.token_id = id; + // True log-probability: logit - logsumexp over the WHOLE vocabulary, + // not one renormalised over the entries shown. The two differ whenever + // top_k is smaller than the vocabulary, which is always. + e.logprob = this->sampler->logits[id] - lse; + const std::string raw = this->tokenizer + ? this->tokenizer->run_time_decoder(id) + : std::string(); + // Decode here, not in the REST handler: that one has no tokenizer. + // As unsigned: `char` is signed on x86-64, so a byte above 0x7F would + // land here as a negative number, and OpenAI defines `bytes` as + // integers 0-255. A client rebuilding the text from them would fail on + // every umlaut, every CJK character, every emoji. + for (unsigned char c : raw) { + e.bytes.push_back(c); + } + e.token = is_serialisable(raw) ? raw : std::string(); + return e; + }; + + logprobs_entry_t entry; + if (sampled_token >= 0 && sampled_token < static_cast(this->sampler->logits.size())) { + entry.chosen = entry_for(sampled_token); + } + const auto& src = this->sampler->captured_top_k; + const size_t limit = std::min(meta_info.top_logprobs_count, src.size()); + entry.alternatives.reserve(limit); + for (size_t i = 0; i < limit; i++) { + // A decoding constraint pushes the tokens it forbids to -inf. They + // were never drawable, and -inf has no JSON number; the list is sorted, + // so the first one ends it. + if (!std::isfinite(src[i].logits)) { + break; + } + entry.alternatives.push_back(entry_for(src[i].token_id)); + } + meta_info.logprobs_entries.push_back(std::move(entry)); +} + std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { std::vector sampled_tokens; std::string result; diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index ab3b31dba..285477c40 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -287,12 +287,32 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic } - size_t n = tokens.size(); - tokens.resize(n - (this->enable_think ? 2 : 4)); + // The template ends with the opening of the assistant turn and, without + // thinking, an empty think block. Normally the prefill stops before that + // block, so the checkpoint below sits where the next turn of the + // conversation will diverge, and generate() feeds the block one forward() + // at a time. A single-turn model never comes back to that checkpoint, so + // the four extra decode steps buy nothing: prefill the block with the + // prompt, and the token drawn from the prefill logits is already the first + // token of the answer. + this->think_block_prefilled = this->single_turn && !this->enable_think; + if (!this->think_block_prefilled) { + size_t n = tokens.size(); + tokens.resize(n - (this->enable_think ? 2 : 4)); + } + // Unless the think block was prefilled, the draw below is thrown away: + // generate() force-feeds the block and samples again. Such a draw must not + // enter the repetition and frequency windows -- it was never generated + // text, and for a model that answers without a thinking block the token + // predicted here IS the answer letter. Counting it pushes that letter down + // by count * freq_penalty + pre_penalty at the position where it matters. + // With the block prefilled the draw is the answer's first token, so it is + // counted, and it is the draw a logprobs request has to capture. + const bool real_draw = this->think_block_prefilled; bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, real_draw, real_draw) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, real_draw, real_draw); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); @@ -316,7 +336,16 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[TKOEN_DECODE_TIME].reset(); std::string token_str; int sampled_token; - if(this->enable_think) { + if (this->think_block_prefilled) { + // insert() prefilled the empty think block; _shared_insert drew the + // answer's first token from those logits, under the constraint. That + // draw is kept, so nothing is rolled back: the constraint started + // behind the closed block (the prompt tail ends with it) and has + // already accepted the token. + sampled_token = this->last_token; + meta_info.insert_sample_pending = false; + } + else if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); this->_force_token(meta_info, think_start_id); @@ -335,6 +364,11 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s token_str = this->tokenizer->run_time_decoder(198); result += token_str; // First token of the reasoning block. + // No capture here: this draw is the first token of the reasoning + // block, and that lands in reasoning_content, not in content. OpenAI's + // logprobs.content describes the tokens of message.content, so a + // report from here would name a token the caller cannot find in the + // text it was given. Requests with think:true get logprobs: null. sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; } @@ -363,6 +397,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s buffer y = this->lm_engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); + this->_request_logprobs_capture(meta_info); sampled_token = this->_sample(y, meta_info); } this->total_tokens++; @@ -371,13 +406,29 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; + // Recorded where the token reaches the caller's text, not where it was + // drawn, and under the same filter the loop below uses: a token the engine + // would have dropped must not get an entry either. + if (this->is_normal_token(last_sampled_token)) { + this->_record_top_logprobs(meta_info, last_sampled_token); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; return result; } - while (this->total_tokens < this->MAX_L){ + while (true){ + // The first token is drawn before the loop, so the limit can already be + // reached here -- with max_tokens=1 it always is. Checked only after a + // draw, the loop would draw and emit one token past the limit. + if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + break; + } + if (this->total_tokens >= this->MAX_L){ + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -397,6 +448,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s std::string token_str = this->tokenizer->run_time_decoder(sampled_token); os << token_str << std::flush; result += token_str; + this->_record_top_logprobs(meta_info, sampled_token); } this->profiler_list[TKOEN_DECODE_TIME].stop(1); this->token_history.push_back(sampled_token); @@ -406,10 +458,6 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s break; } meta_info.generated_tokens++; - if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ - reason = MAX_LENGTH_REACHED; - break; - } } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 9a6aaabf5..6fa07cbc6 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -299,9 +299,15 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, size_t n = tokens.size(); tokens.resize(n - (this->enable_think ? 2 : 4)); + // The draw below is thrown away: generate() force-feeds the think block and + // samples again. It must not enter the repetition and frequency windows -- + // it was never generated text, and for a model that answers without a + // thinking block the token predicted here IS the answer letter. Counting it + // pushes that letter down by count * freq_penalty + pre_penalty at the position + // where it matters, so the model's preferred answer arrives penalised. bool success = has_images - ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) - : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index, std::nullopt, false) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, std::nullopt, false); checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); diff --git a/src/common/modules/sampler.cpp b/src/common/modules/sampler.cpp index 59c01cb8e..dacc36f48 100644 --- a/src/common/modules/sampler.cpp +++ b/src/common/modules/sampler.cpp @@ -8,6 +8,9 @@ #include "modules/sampler.hpp" +#include +#include +#include #include #include // for std::sort #include @@ -354,10 +357,112 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) { this->total_tokens++; } +namespace { +/// exp() for every value a bf16 can hold. +/// +/// The logits arrive as bf16 from the engine and the widening to fp32 is +/// exact, so this->logits can only take 65536 distinct values. Looking the +/// exponential up is the same number std::exp would return -- precomputed, +/// not approximated -- and it turns a quarter of a million exp() calls into as +/// many reads. 512 KB, which fits in L2. +const std::array& bf16_exp_table() { + static const std::array table = [] { + std::array t{}; + for (uint32_t b = 0; b < 65536; b++) { + const uint32_t bits = b << 16; + float f; + std::memcpy(&f, &bits, sizeof(f)); + t[b] = std::exp(static_cast(f)); + } + return t; + }(); + return table; +} +} // namespace + +float Sampler::log_sum_exp_all() const { + if (this->logits.empty()) { + return 0.0f; + } + const auto& table = bf16_exp_table(); + double sum = 0.0; + for (int i = 0; i < this->in_features; i++) { + uint32_t bits; + std::memcpy(&bits, &this->logits[i], sizeof(bits)); + // The low 16 bits are non-zero for anything the penalties moved off + // the bf16 grid; those take the exact call. The branch predicts well + // and costs nothing measurable. + sum += (bits & 0xFFFFu) != 0u + ? std::exp(static_cast(this->logits[i])) + : table[bits >> 16]; + } + if (std::isfinite(sum) && sum > 0.0) { + return static_cast(std::log(sum)); + } + // Nothing on the bf16 grid produced a usable sum -- reachable only for a + // logit beyond about +-700, where exp overflows a double. Fall back to the + // shifted form. Note what it does NOT cover: +-inf and NaN are not + // meaningful here either way, and a logit of +inf would come back as 0. + float max_logit = -std::numeric_limits::infinity(); + for (float v : this->logits) { + if (v > max_logit) { + max_logit = v; + } + } + if (!std::isfinite(max_logit)) { + return 0.0f; + } + double shifted = 0.0; + for (float v : this->logits) { + shifted += std::exp(static_cast(v) - max_logit); + } + return max_logit + static_cast(std::log(shifted)); +} + +void Sampler::capture_if_requested() { + if (!this->capture_top_logprobs) { + return; + } + // Own top-N by logit, taken from the full logits rather than from + // top_k_logits: the report should not depend on the sampler's top_k, and + // the greedy path keeps only one entry there. + const int n = std::min(this->capture_count, this->in_features); + this->captured_top_k.clear(); + this->captured_logsumexp = this->log_sum_exp_all(); + if (n > 0 && static_cast(n) <= this->top_k_logits.size()) { + // top_k_logits is already sorted by logit with the same tie-break, so + // its beginning IS the top-n. No second sort, no vocab-sized index + // vector -- which matters because this runs once per token. + this->captured_top_k.assign(this->top_k_logits.begin(), + this->top_k_logits.begin() + n); + } else if (n > 0) { + std::vector idx(this->in_features); + std::iota(idx.begin(), idx.end(), 0); + // Same order as sampler_topk_apply: bf16 logits over 100k+ entries + // are often equal, partial_sort is not stable, and a tie broken + // differently would put another token at the top than the one the + // draw would have taken. + std::partial_sort(idx.begin(), idx.begin() + n, idx.end(), + [this](int a, int b) { + if (this->logits[a] != this->logits[b]) { + return this->logits[a] > this->logits[b]; + } + return a < b; + }); + this->captured_top_k.reserve(n); + for (int i = 0; i < n; i++) { + this->captured_top_k.push_back(logits_t{ this->logits[idx[i]], idx[i], 0.0f }); + } + } + // Stays armed: every token that reaches the caller's text needs its own + // distribution. `captured` marks that THIS draw has one. + this->captured = true; +} + /// \brief Sample the token /// \param x the input buffer /// \return the sampled token -int Sampler::sample_greedy(buffer& x, const std::vector* allowed) { +int Sampler::sample_greedy(buffer& x, const std::vector* allowed, bool record_in_history) { const bool penalties_active = (this->repeat_last_n != 0) && (this->rep_penalty != 1.0f || this->freq_penalty != 0.0f || this->pre_penalty != 0.0f); @@ -447,7 +552,35 @@ int Sampler::sample_greedy(buffer& x, const std::vector* allowed // callers that inspect the surviving candidates still get a consistent view this->top_k_logits.assign(1, logits_t{ best, best_id, 1.0f }); - ring_buffer_update_sparse(best_id); + if (this->capture_top_logprobs) { + // log_sum_exp_all() reads this->logits, which the branch above only + // fills when penalties or a constraint mask are active -- and then + // penalised and masked, as the draw saw them, so it must not be + // overwritten. Otherwise fill it here the same way sample() does: the + // logits come back in device-mapped memory where element-at-a-time + // reads are punishing, so this pulls eight at a time. + if (!penalties_active && allowed == nullptr) { + #if USEAVX2 + const int simd_width = 8; + int i = 0; + for (; i <= in_features - simd_width; i += simd_width) { + __m128i bf16_vals_x = _mm_loadu_si128((__m128i*)&x[i]); + _mm256_storeu_ps(&this->logits[i], bf16o_fp32(bf16_vals_x)); + } + for (; i < in_features; i++) { + this->logits[i] = x[i]; + } + #else + for (int i = 0; i < in_features; i++) { + this->logits[i] = x[i]; + } + #endif + } + this->capture_if_requested(); + } + if (record_in_history) { + ring_buffer_update_sparse(best_id); + } return best_id; } @@ -471,10 +604,10 @@ void Sampler::sampler_allowed_apply(const std::vector* allowed) { } } -int Sampler::sample(buffer& x, const std::vector* allowed) { +int Sampler::sample(buffer& x, const std::vector* allowed, bool record_in_history) { // PRNG is seeded once at construction, no per-call seeding needed if (this->top_k == 1) { - return sample_greedy(x, allowed); + return sample_greedy(x, allowed, record_in_history); } // COPY FROM `x` → `this->logits[]` @@ -500,6 +633,8 @@ int Sampler::sample(buffer& x, const std::vector* allowed) { sampler_penalty_apply_sparse(); sampler_allowed_apply(allowed); sampler_topk_apply(this->top_k); + + this->capture_if_requested(); if (this->use_optimized_sampling) { softmax_with_topp_minp(this->top_p, this->min_p); sampler_temp_apply(this->temperature); @@ -515,7 +650,9 @@ int Sampler::sample(buffer& x, const std::vector* allowed) { } int sampled_index = sample_from_probs(); - ring_buffer_update_sparse(sampled_index); + if (record_in_history) { + ring_buffer_update_sparse(sampled_index); + } return sampled_index; } \ No newline at end of file diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index a2921a192..79c7b5324 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -85,8 +85,42 @@ inline std::string stop_reason_to_string(stop_reason_t reason){ } } +/// One entry of the reported distribution. +/// +/// A struct of its own rather than Sampler::logits_t: this header should not +/// have to know about the sampler. +struct top_logprob_t { + int token_id = -1; + std::string token; ///< decoded; empty when the bytes are not valid UTF-8 + std::vector bytes; ///< the token's raw bytes, as OpenAI reports them + float logprob = 0.0f; ///< logit - logsumexp over the whole vocabulary +}; + +/// One token of the answer, with the distribution it was drawn from. +/// +/// The chosen token has its own entry, independent of how many alternatives +/// were asked for: a draw at rank 3 must still be named when the caller asked +/// for one alternative. +struct logprobs_entry_t { + top_logprob_t chosen; + std::vector alternatives; +}; + struct chat_meta_info_t { int max_prefill_len; + + /// Whether and how many alternatives the caller asked for. Every token + /// that reaches message.content is described, in that order. + bool want_logprobs = false; + int top_logprobs_count = 0; + + /// One entry per token that reached the caller's text, in that order -- + /// the same order and the same count as the tokens of message.content. + /// Each carries the token that was drawn plus its alternatives. + std::vector logprobs_entries; + + /// Guards the once-per-request rule for arming the sampler. + bool logprobs_armed = false; int prompt_tokens; // whole prompt, cached prefix included int cached_prompt_tokens; // subset of prompt_tokens served from the KV cache int generated_tokens; @@ -263,12 +297,27 @@ class AutoModel { return backend_cap == 0 ? MAX_L : std::min(MAX_L, backend_cap); } + /// \brief Ask the sampler to capture a distribution for this request + /// \param meta_info carries the request's wish + /// \note Call before a draw that will also be counted. A draw whose token + /// is thrown away must not be captured -- the numbers would describe + /// a position the answer does not come from. + void _request_logprobs_capture(chat_meta_info_t& meta_info); + + /// \brief Append one entry for a token that reached the caller's text + /// \param meta_info carries it to the REST handler + /// \param sampled_token the token the sampler just chose + /// \note Call where the token is written to the response, not where it was + /// drawn: a token the engine filters out, and EOS, get no entry. + void _record_top_logprobs(chat_meta_info_t& meta_info, int sampled_token); + /// \brief Insert tokens into the model /// \param meta_info the meta information of the chat /// \param tokens the tokens to insert /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise - bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt); + bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0, std::optional requested_max_new_tokens = std::nullopt, + bool record_sampled_token = true, bool capture_logprobs = false); /// \brief Reject a request that cannot fit in the backend's decode limit /// \param rendered_tokens the length of the rendered prompt @@ -290,8 +339,10 @@ class AutoModel { /// \brief Sample the next token under the request's constraint /// \param logits the logits of the token about to be sampled, masked in place /// \param meta_info the request, carrying the constraint + /// \param record_in_history whether the token joins the sampler's repetition + /// and frequency windows; false for a draw the caller throws away /// \return the sampled token, already reported to the constraint - int _sample(buffer& logits, chat_meta_info_t& meta_info); + int _sample(buffer& logits, chat_meta_info_t& meta_info, bool record_in_history = true); /// \brief Report a token the model is made to emit rather than samples /// \note For the think preamble some generate()s feed after insert(); the diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index c7d55af10..17b00e5f0 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -28,6 +28,10 @@ class Qwen3_5VL : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; + /// Set by insert() when it prefilled the empty think block together with + /// the prompt; generate() then takes the first token from that draw + /// instead of feeding the block one forward() at a time. + bool think_block_prefilled = false; void setup_tokenizer(std::string model_path); diff --git a/src/include/modules/sampler.hpp b/src/include/modules/sampler.hpp index df7997345..6f71934c1 100644 --- a/src/include/modules/sampler.hpp +++ b/src/include/modules/sampler.hpp @@ -51,6 +51,29 @@ class Sampler{ int in_features; std::vector counters; logits_list_t top_k_logits; + + /// Set by the model for a request that asked for logprobs, and cleared at + /// the start of every request whether it asked or not. It stays set for + /// the whole request: every token that reaches the caller's text needs its + /// own distribution. A request that does not ask never arms it, so it pays + /// nothing. + bool capture_top_logprobs = false; + /// How many entries to keep. Bounded by the caller, OpenAI allows 20. + int capture_count = 0; + + /// The top entries by logit at the moment of the capture, and the + /// log-sum-exp of the WHOLE vocabulary at the same moment. + /// + /// Sorted by logit and cut to the count the caller asked for, so it is NOT + /// the list the draw used and the drawn token need not be in it: with + /// top_logprobs: 1 and a draw at rank 3 it is not. That is what `chosen` + /// is for. Entries behind the top-p and min-p cut are listed although they + /// had no chance of being drawn. + logits_list_t captured_top_k; + /// Set by the capture, cleared when a request arms it. Guards against + /// reading values that belong to an earlier draw or request. + bool captured = false; + float captured_logsumexp = 0.0f; float rep_penalty; float freq_penalty; float pre_penalty; @@ -102,7 +125,22 @@ class Sampler{ /// Going through the general path would allocate and partial_sort a /// vocab-sized array of triples and run three full-vocab softmaxes for /// a result that never depends on any of it. - int sample_greedy(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows + /// \note Pass false for a draw the caller throws away. Such a token was + /// never generated text, and counting it pushes the tokens that + /// follow it down -- by count * freq_penalty + pre_penalty each time + /// it occurs. + int sample_greedy(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); + + /// \brief Capture the current top-k if a request asked for it + /// \note Called by the sampling paths right after the top-k is settled and + /// before the temperature. Stays armed until the next request + /// disarms it; `captured` marks that this draw has values. + void capture_if_requested(); + /// \brief log-sum-exp over the whole vocabulary + /// \note Reads this->logits, which the sampling paths have filled by then + float log_sum_exp_all() const; void sampler_topk_apply(int k); void sampler_topp_apply(float p); @@ -117,7 +155,9 @@ class Sampler{ /// \param allowed optional bitmask, bit i of word i/32 set when token i may be /// sampled; applied to the sampler's own copy of the logits /// \return the sampled token - int sample(buffer& x, const std::vector* allowed = nullptr); + /// \param record_in_history whether the drawn token joins the repetition + /// and frequency windows; see sample_greedy + int sample(buffer& x, const std::vector* allowed = nullptr, bool record_in_history = true); /// \brief Push every token the bitmask does not allow to -inf in this->logits void sampler_allowed_apply(const std::vector* allowed); diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index b37acca08..afd206eb5 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -612,6 +612,124 @@ void RestHandler::configure_chat_engine_parameters(const json& options, const js } } +/// Read the caller's logprobs wish into meta_info, for the chat shape. +/// +/// OpenAI: `logprobs` is a boolean, and `top_logprobs` (0-20) says how many +/// alternatives to list. Without `top_logprobs`, only the chosen token's own +/// logprob is reported and the alternatives stay empty. +static void read_logprobs_request(const json& request, chat_meta_info_t& meta_info) { + if (!request.contains("logprobs") || !request["logprobs"].is_boolean()) { + return; + } + meta_info.want_logprobs = request["logprobs"].get(); + if (!meta_info.want_logprobs) { + return; + } + if (request.contains("top_logprobs") && request["top_logprobs"].is_number_integer()) { + meta_info.top_logprobs_count = std::clamp(request["top_logprobs"].get(), 0, 20); + } +} + +/// The same for the legacy completions shape, where `logprobs` is an INTEGER: +/// N asks for the N most likely tokens, 0-5, and there is no `top_logprobs`. +/// Zero is a valid request -- it asks for the drawn token's own logprob with +/// no alternatives -- and the drawn token is always reported. +/// +/// A boolean is not accepted here: the SDKs send the integer, and a generous +/// dialect would become a contract that clients come to rely on. +static void read_legacy_logprobs_request(const json& request, chat_meta_info_t& meta_info) { + if (!request.contains("logprobs") || !request["logprobs"].is_number_integer()) { + return; + } + const int n = request["logprobs"].get(); + meta_info.want_logprobs = n >= 0; + meta_info.top_logprobs_count = std::clamp(n, 0, 5); +} + +static json top_logprob_to_json(const top_logprob_t& e) { + return json{ + {"token", e.token}, + {"token_id", e.token_id}, + {"bytes", e.bytes}, + {"logprob", e.logprob}, + }; +} + +/// The OpenAI logprobs object for the answer. +/// +/// One entry per token of message.content, in the same order, so a client that +/// pairs entries with tokens stays aligned. +/// +/// `logprob` is the natural logarithm of the token's probability over the +/// WHOLE vocabulary, after the repetition, frequency and presence penalties and +/// before the temperature, top-k, top-p and min-p. The entries listed +/// therefore do NOT sum to 1 -- the rest of the vocabulary lies outside the +/// list. Comparing two entries gives the same answer whether you use logprob +/// or the raw logit, because both differ by the same constant. +/// +/// A request with think:true gets null: the tokens of the reasoning block land +/// in reasoning_content, not in content, and this field describes content. +static json logprobs_from_meta(const chat_meta_info_t& meta_info) { + if (!meta_info.want_logprobs || meta_info.logprobs_entries.empty()) { + return nullptr; + } + json content = json::array(); + for (const auto& entry : meta_info.logprobs_entries) { + json alternatives = json::array(); + for (const auto& e : entry.alternatives) { + alternatives.push_back(top_logprob_to_json(e)); + } + json chosen = top_logprob_to_json(entry.chosen); + chosen["top_logprobs"] = alternatives; + content.push_back(std::move(chosen)); + } + return json{{"content", content}}; +} + +/// The same information in the legacy completions shape, which pairs one entry +/// per token rather than nesting them. +static json legacy_logprobs_from_meta(const chat_meta_info_t& meta_info) { + if (!meta_info.want_logprobs || meta_info.logprobs_entries.empty()) { + return nullptr; + } + json tokens = json::array(); + json token_logprobs = json::array(); + json top_logprobs = json::array(); + json text_offset = json::array(); + int offset = 0; + for (const auto& entry : meta_info.logprobs_entries) { + tokens.push_back(entry.chosen.token); + token_logprobs.push_back(entry.chosen.logprob); + json per_token = json::object(); + for (const auto& e : entry.alternatives) { + // This shape keys by the token string and has no `bytes`, so a + // token that is not valid UTF-8 on its own would be keyed by "" + // and overwrite the previous one. Those are left out. + if (e.token.empty()) { + continue; + } + per_token[e.token] = e.logprob; + } + top_logprobs.push_back(std::move(per_token)); + text_offset.push_back(offset); + // OpenAI counts characters here. Counting the bytes that are not + // continuation bytes gives exactly that, and it stays right when one + // character is split across two tokens: it is counted once, at the + // token carrying its first byte. + for (int b : entry.chosen.bytes) { + if ((b & 0xC0) != 0x80) { + offset++; + } + } + } + return json{ + {"tokens", tokens}, + {"token_logprobs", token_logprobs}, + {"top_logprobs", top_logprobs}, + {"text_offset", text_offset}, + }; +} + json RestHandler::build_nstream_response(const std::string& response_text, chat_meta_info_t& meta_info, const chat_parse_options& parse_options) { const chat_format_spec& spec = auto_chat_engine->get_chat_format(); @@ -659,7 +777,10 @@ json RestHandler::build_nstream_response(const std::string& response_text, chat_ { {"index", 0}, {"message", message}, - {"logprobs", nullptr}, + // A tool call puts nothing in message.content, so there are no + // tokens for a logprobs object to describe. Same for a mixed + // answer that carries both text and a call. + {"logprobs", is_tool_call ? json(nullptr) : logprobs_from_meta(meta_info)}, {"finish_reason", stop_reason_to_string(meta_info.stop_reason)} } }); @@ -1208,6 +1329,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, // see if we can use prompt cache chat_meta_info_t meta_info; + read_logprobs_request(request, meta_info); bool can_use_prompt_cache = false; if (auto_chat_engine->single_turn) { // reject multi-turn requests: any assistant message in the history @@ -1599,6 +1721,7 @@ void RestHandler::handle_openai_completion(const json& request, configure_chat_engine_parameters(options, request); chat_meta_info_t meta_info; + read_legacy_logprobs_request(request, meta_info); meta_info.max_prefill_len = this->prefill_chunk_len; lm_uniform_input_t uniformed_input; uniformed_input.requested_max_new_tokens = @@ -1692,7 +1815,7 @@ void RestHandler::handle_openai_completion(const json& request, { {"text", response_text}, {"index", 0}, - {"logprobs", nullptr}, + {"logprobs", legacy_logprobs_from_meta(meta_info)}, {"finish_reason", stop_reason_to_string(meta_info.stop_reason)} } })},