diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index e5cc893f6..7a1c7b49d 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -232,7 +232,15 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int } void AutoModel::_shared_initialize_model_state( - std::string model_path, json, int context_length) { + std::string model_path, json model_info, int context_length) { + // The catalog marks a model single-turn with a label, and some classes set + // the flag themselves. Only ever switched on here, so an entry without the + // label does not override a class that knows it is single-turn. + if (model_info.contains("label") && + std::find(model_info["label"].begin(), model_info["label"].end(), + "single-turn") != model_info["label"].end()) { + this->single_turn = true; + } this->model_path = std::move(model_path); header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); @@ -420,7 +428,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to break; } } - if (skip_count != idx) { + // A request that matches the whole history leaves nothing to prefill, and + // then there are no logits to draw from: the sampler would read an empty + // buffer. It happens when a request repeats the previous one exactly and + // that answer added nothing to the history -- a single-turn Qwen3.5 model + // with the think block prefilled and max_tokens=1. Recurrent state cannot + // be stepped back by one token, so treat it like a divergence. + if (skip_count != idx || skip_count == tokens.size()) { if (is_server_mode) { header_print("FLM", "Conversation context diverged from cache, clearing context..."); } diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index ab3b31dba..9ac6851cd 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -287,8 +287,19 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic } - size_t n = tokens.size(); - tokens.resize(n - (this->enable_think ? 2 : 4)); + // The template ends with the opening of the assistant turn and, without + // thinking, an empty think block. Normally the prefill stops before that + // block, so the checkpoint below sits where the next turn of the + // conversation will diverge, and generate() feeds the block one forward() + // at a time. A single-turn model never comes back to that checkpoint, so + // the four extra decode steps buy nothing: prefill the block with the + // prompt, and the token drawn from the prefill logits is already the first + // token of the answer. + this->think_block_prefilled = this->single_turn && !this->enable_think; + if (!this->think_block_prefilled) { + size_t n = tokens.size(); + tokens.resize(n - (this->enable_think ? 2 : 4)); + } bool success = has_images ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) @@ -316,7 +327,16 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[TKOEN_DECODE_TIME].reset(); std::string token_str; int sampled_token; - if(this->enable_think) { + if (this->think_block_prefilled) { + // insert() prefilled the empty think block; _shared_insert drew the + // answer's first token from those logits, under the constraint. That + // draw is kept, so nothing is rolled back: the constraint started + // behind the closed block (the prompt tail ends with it) and has + // already accepted the token. + sampled_token = this->last_token; + meta_info.insert_sample_pending = false; + } + else if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); this->_force_token(meta_info, think_start_id); diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index c7d55af10..17b00e5f0 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -28,6 +28,10 @@ class Qwen3_5VL : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; + /// Set by insert() when it prefilled the empty think block together with + /// the prompt; generate() then takes the first token from that draw + /// instead of feeding the block one forward() at a time. + bool think_block_prefilled = false; void setup_tokenizer(std::string model_path);