From 5706f064252a89982210e52ecadba34e6c759106 Mon Sep 17 00:00:00 2001 From: Manuel Domke Date: Mon, 12 Oct 2026 12:52:56 +0200 Subject: [PATCH] fix(qwen3.5): stop at max_tokens instead of one token past it Qwen3_5VL::generate() draws the first token before its decode loop and checked the limit only at the bottom of the loop, after the next draw. So every request that ran into max_tokens got one token more than it asked for, and paid one more forward() for it: max_tokens=1 returned "A\n" with completion_tokens 2. The loop now checks the limit first, before it draws. Its shape follows _shared_generate(): the context limit is checked inside the loop, and reaching max_tokens still wins over it, so finish_reason is unchanged. Measured on Qwen3.5-2B (NPU2), 20 independent one-token prompts, three alternating rounds against main: median wall clock 604 -> 574 ms per request, completion_tokens 2 -> 1. The first token of every answer and two 60-token generations are identical to main. Co-Authored-By: Claude Opus 5.5 --- src/common/AutoModel/modeling_qwen3_5vl.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index a09f09731..3efef3751 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -377,7 +377,17 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s reason = MAX_LENGTH_REACHED; return result; } - while (this->total_tokens < this->MAX_L){ + while (true){ + // The first token is drawn before the loop, so the limit can already be + // reached here -- with max_tokens=1 it always is. Checked only after a + // draw, the loop would draw and emit one token past the limit. + if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + break; + } + if (this->total_tokens >= this->MAX_L){ + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; // reset stream content @@ -414,10 +424,6 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s break; } meta_info.generated_tokens++; - if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ - reason = MAX_LENGTH_REACHED; - break; - } } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason;