From b52493a658b7639dfefc2fe45239fb1880ef6b40 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 12:54:51 +0200 Subject: [PATCH 1/2] perf(qwen3.5): prefill the empty think block for single-turn models Without thinking, the chat template ends with an empty think block. Qwen3_5VL::insert() cut those four tokens off before the prefill, so the checkpoint lands where the next turn of a conversation diverges, and generate() then fed them in with one forward() each. Only after those four decode steps came the draw for the answer's first token. A single-turn model never returns to that checkpoint. For such a model the block is now prefilled with the prompt, and the token _shared_insert() draws from the prefill logits is the answer's first token. Multi-turn models keep the old path, because there the checkpoint pays off. So does thinking, where the model writes the block itself. Under a decoding constraint the draw is kept as well. The constraint starts from the prompt tail, which now ends behind the closed block, and it has already accepted the token; nothing is rolled back. The catalog's "single-turn" label now also sets AutoModel::single_turn. Until now only three classes set the flag themselves, all with the label, so nothing changes for them. It is only ever switched on, never off. Measured on Qwen3.5-2B (NPU2) with the label set, 20 independent one-token prompts, three alternating rounds against main: median wall clock 604 -> 486 ms per request. Prefill time is unchanged (430 vs 431 ms); the four tokens it gains cost nothing measurable. The numbers are not bit-identical. The template's tokens are exactly the four ids generate() used to feed ([248068, 271, 248069, 271]), but they now go through the prefill kernels instead of four decode steps. Against main with the discarded-sample fix (#776) 1 of 20 answers differs, and that was an exact tie (A and C both at logprob -1.939). One of two 60-token generations diverges after 90 characters. With logprobs applied on top, the log-odds between options move by 0.25 nats median, 0.5 at most. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/automodel.cpp | 10 +++++++- src/common/AutoModel/modeling_qwen3_5vl.cpp | 26 +++++++++++++++++--- src/include/AutoModel/modeling_qwen3_5vl.hpp | 4 +++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index e5cc893f6..a4d9b3299 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -232,7 +232,15 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int } void AutoModel::_shared_initialize_model_state( - std::string model_path, json, int context_length) { + std::string model_path, json model_info, int context_length) { + // The catalog marks a model single-turn with a label, and some classes set + // the flag themselves. Only ever switched on here, so an entry without the + // label does not override a class that knows it is single-turn. + if (model_info.contains("label") && + std::find(model_info["label"].begin(), model_info["label"].end(), + "single-turn") != model_info["label"].end()) { + this->single_turn = true; + } this->model_path = std::move(model_path); header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index ab3b31dba..9ac6851cd 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -287,8 +287,19 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic } - size_t n = tokens.size(); - tokens.resize(n - (this->enable_think ? 2 : 4)); + // The template ends with the opening of the assistant turn and, without + // thinking, an empty think block. Normally the prefill stops before that + // block, so the checkpoint below sits where the next turn of the + // conversation will diverge, and generate() feeds the block one forward() + // at a time. A single-turn model never comes back to that checkpoint, so + // the four extra decode steps buy nothing: prefill the block with the + // prompt, and the token drawn from the prefill logits is already the first + // token of the answer. + this->think_block_prefilled = this->single_turn && !this->enable_think; + if (!this->think_block_prefilled) { + size_t n = tokens.size(); + tokens.resize(n - (this->enable_think ? 2 : 4)); + } bool success = has_images ? this->_shared_insert(meta_info, tokens, is_cancelled, &image_payload, last_image_token_index) @@ -316,7 +327,16 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[TKOEN_DECODE_TIME].reset(); std::string token_str; int sampled_token; - if(this->enable_think) { + if (this->think_block_prefilled) { + // insert() prefilled the empty think block; _shared_insert drew the + // answer's first token from those logits, under the constraint. That + // draw is kept, so nothing is rolled back: the constraint started + // behind the closed block (the prompt tail ends with it) and has + // already accepted the token. + sampled_token = this->last_token; + meta_info.insert_sample_pending = false; + } + else if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); this->_force_token(meta_info, think_start_id); diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index c7d55af10..17b00e5f0 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -28,6 +28,10 @@ class Qwen3_5VL : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; + /// Set by insert() when it prefilled the empty think block together with + /// the prompt; generate() then takes the first token from that draw + /// instead of feeding the block one forward() at a time. + bool think_block_prefilled = false; void setup_tokenizer(std::string model_path); From f3b4bf9123243bd1112b86144fdcd2daf8242835 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 15:09:47 +0200 Subject: [PATCH 2/2] fix: prefill again when a request matches the whole history _shared_insert() skips the prefix a request shares with token_history. If the request matches the history completely, nothing is left to prefill, _chunked_insert() returns an empty buffer, and Sampler::sample() reads it: a segfault at address 0. With the empty think block prefilled, a single-turn Qwen3.5 request puts exactly its own tokens into token_history. The first generated token is never added there. On its own this change cannot hit the bug: with max_tokens=1 the decode loop still draws one more token and appends it. Together with the max_tokens fix from the companion PR the loop no longer runs, and the same request sent twice crashed the server on the second one. Reproduced under gdb: the crashing request logs no "Prefill chunk" line. A complete match is now treated like a divergence: clear and prefill everything. Stepping back by one token is not an option, because the linear attention layers carry recurrent state that cannot be rewound. With both changes applied, eight identical requests in a row pass, and answers and confidences are unchanged. The extra prefill is not measurable, because a single-turn model cleared its context on these requests before anyway. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/automodel.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index a4d9b3299..7a1c7b49d 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -428,7 +428,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to break; } } - if (skip_count != idx) { + // A request that matches the whole history leaves nothing to prefill, and + // then there are no logits to draw from: the sampler would read an empty + // buffer. It happens when a request repeats the previous one exactly and + // that answer added nothing to the history -- a single-turn Qwen3.5 model + // with the think block prefilled and max_tokens=1. Recurrent state cannot + // be stepped back by one token, so treat it like a divergence. + if (skip_count != idx || skip_count == tokens.size()) { if (is_server_mode) { header_print("FLM", "Conversation context diverged from cache, clearing context..."); }