From b73f3eb2b4901fb9208f35758cb247f838969f4f Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 23 Sep 2026 12:29:49 -0600 Subject: [PATCH 01/18] feat: openai policy --- src/include/openai_policy.hpp | 164 ++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/include/openai_policy.hpp diff --git a/src/include/openai_policy.hpp b/src/include/openai_policy.hpp new file mode 100644 index 000000000..cbb8a69a8 --- /dev/null +++ b/src/include/openai_policy.hpp @@ -0,0 +1,164 @@ +#pragma once + +#include +#include +#include +#include + +#include "AutoModel/automodel.hpp" + +// content type +enum class openai_content_type { + CONTENT, + REASONING, + TOOL +}; + +// reasoning tags +struct reasoning_tags { + std::string start_tag; + std::string end_tag; +}; + +// response struct +struct openai_content { + std::string text; + openai_content_type type; +}; + + + +class openai_policy { +public: + explicit openai_policy(chat_template_type_t template_type, bool start_in_reasoning = false); + + static reasoning_tags get_reasoning_tags(chat_template_type_t template_type); + + std::vector parse(const std::string& text); + std::vector flush(); + void reset(bool start_in_reasoning = false); + + bool in_reasoning_part() const; + +private: + static void append(std::vector& parts, + openai_content_type type, + const std::string& text); + + static std::size_t partial_tag_length(const std::string& buffer, const std::string& tag); + + openai_content_type current_type() const; + const std::string& pending_tag() const; + + chat_template_type_t template_type; + reasoning_tags tags; + std::string buffer; + bool in_reasoning; +}; + +inline openai_policy::openai_policy(chat_template_type_t template_type, bool start_in_reasoning) + : template_type(template_type), + tags(get_reasoning_tags(template_type)), + in_reasoning(start_in_reasoning) { + buffer.reserve(256); +} + +inline reasoning_tags openai_policy::get_reasoning_tags(chat_template_type_t template_type) { + switch (template_type) { + case chat_template_type_t::chat_ml: + return reasoning_tags{"", ""}; + case chat_template_type_t::harmony: + return reasoning_tags{"<|start|>assistant<|channel|>analysis<|message|>", + "<|end|><|start|>assistant<|channel|>final<|message|>"}; + case chat_template_type_t::gemma4: + return reasoning_tags{"<|channel>thought", ""}; + case chat_template_type_t::unknown_template: + return reasoning_tags{"", ""}; + } + return reasoning_tags{"", ""}; +} + +inline std::vector openai_policy::parse(const std::string& text) { + std::vector parts; + // carry over whatever was held back last call, a tag may straddle two calls + buffer += text; + + // consume every complete tag sitting in the buffer, one call can cross + // several boundaries; pending_tag() is the start tag while we are outside + // reasoning and the end tag while we are inside it + while (!pending_tag().empty()) { + const std::string& tag = pending_tag(); + std::size_t pos = buffer.find(tag); + if (pos == std::string::npos) { + // no boundary yet, the rest is handled below + break; + } + // everything before the tag belongs to the part we are leaving + append(parts, current_type(), buffer.substr(0, pos)); + // drop the tag itself, it is a marker and never reaches the client + buffer.erase(0, pos + tag.size()); + in_reasoning = !in_reasoning; + } + + // the tail may be the beginning of the next tag, hold that many bytes back + // so a split marker is not emitted as visible text + std::size_t held = partial_tag_length(buffer, pending_tag()); + if (buffer.size() > held) { + std::size_t emit_len = buffer.size() - held; + append(parts, current_type(), buffer.substr(0, emit_len)); + buffer.erase(0, emit_len); + } + + // whatever stays in buffer is a partial tag, flush() releases it at end of stream + return parts; +} + +inline std::vector openai_policy::flush() { + std::vector parts; + append(parts, current_type(), buffer); + buffer.clear(); + return parts; +} + +inline void openai_policy::reset(bool start_in_reasoning) { + buffer.clear(); + in_reasoning = start_in_reasoning; +} + +inline bool openai_policy::in_reasoning_part() const { + return in_reasoning; +} + +inline void openai_policy::append(std::vector& parts, + openai_content_type type, + const std::string& text) { + if (text.empty()) { + return; + } + if (!parts.empty() && parts.back().type == type) { + parts.back().text += text; + return; + } + parts.push_back(openai_content{text, type}); +} + +inline std::size_t openai_policy::partial_tag_length(const std::string& buffer, const std::string& tag) { + if (tag.empty() || buffer.empty()) { + return 0; + } + std::size_t max_len = std::min(buffer.size(), tag.size() - 1); + for (std::size_t len = max_len; len > 0; --len) { + if (buffer.compare(buffer.size() - len, len, tag, 0, len) == 0) { + return len; + } + } + return 0; +} + +inline openai_content_type openai_policy::current_type() const { + return in_reasoning ? openai_content_type::REASONING : openai_content_type::CONTENT; +} + +inline const std::string& openai_policy::pending_tag() const { + return in_reasoning ? tags.end_tag : tags.start_tag; +} From 1865383384b12cdbfb4f45aa7e2c5af94d177e7f Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 7 Oct 2026 14:48:22 -0600 Subject: [PATCH 02/18] Add xgrammar as a dependency for constrained decoding xgrammar (mlc-ai, Apache-2.0) compiles JSON schemas and structural tags into grammars and computes, per sampling step, which tokens keep the output valid. It will back tool_choice and response_format. The submodule is pinned at v0.2.8. Its sources are built as the static library flm_xgrammar instead of through its own CMakeLists, which would add install rules and -Werror; cpptrace stays off. Clones need `git submodule update --init --recursive` for its dlpack headers. Co-Authored-By: Claude --- .gitmodules | 3 +++ src/CMakeLists.txt | 19 +++++++++++++++++++ third_party/xgrammar | 1 + 3 files changed, 23 insertions(+) create mode 160000 third_party/xgrammar diff --git a/.gitmodules b/.gitmodules index b20292bf7..e109f63cb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "third_party/tokenizers-cpp"] path = third_party/tokenizers-cpp url = https://github.com/mlc-ai/tokenizers-cpp +[submodule "third_party/xgrammar"] + path = third_party/xgrammar + url = https://github.com/mlc-ai/xgrammar diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48910cc06..847147adf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -282,6 +282,23 @@ add_subdirectory(${CMAKE_SOURCE_DIR}/../third_party/tokenizers-cpp ${CMAKE_BINARY_DIR}/tokenizers-cpp EXCLUDE_FROM_ALL) +# ——————————————————————————————————————————————— +# xgrammar: the grammar engine behind constrained decoding (tool_choice, +# response_format). Built from its sources as a plain static library rather than +# through its own CMakeLists, which would add install rules and -Werror. +# ——————————————————————————————————————————————— +set(XGRAMMAR_DIR ${CMAKE_SOURCE_DIR}/../third_party/xgrammar) +file(GLOB_RECURSE XGRAMMAR_SOURCES ${XGRAMMAR_DIR}/cpp/*.cc) +list(FILTER XGRAMMAR_SOURCES EXCLUDE REGEX "/cpp/tvm_ffi/.*\\.cc$") +add_library(flm_xgrammar STATIC ${XGRAMMAR_SOURCES}) +target_compile_features(flm_xgrammar PUBLIC cxx_std_17) +target_include_directories(flm_xgrammar PUBLIC ${XGRAMMAR_DIR}/include) +target_include_directories(flm_xgrammar SYSTEM PUBLIC + ${XGRAMMAR_DIR}/3rdparty/picojson + ${XGRAMMAR_DIR}/3rdparty/dlpack/include) +target_compile_definitions(flm_xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) +set_target_properties(flm_xgrammar PROPERTIES POSITION_INDEPENDENT_CODE ON) + # ——————————————————————————————————————————————— # Gather your sources # ——————————————————————————————————————————————— @@ -522,6 +539,8 @@ list(APPEND FLM_ENGINE_LINK_LIBS ) # Link your custom libraries (these may still be DLLs if no static versions available) +target_link_libraries(flm PRIVATE flm_xgrammar) + target_link_libraries(flm PUBLIC ${FLM_ENGINE_LINK_LIBS} # flm's own code calls nothing in the four shared-runtime libs below -- the diff --git a/third_party/xgrammar b/third_party/xgrammar new file mode 160000 index 000000000..97787376f --- /dev/null +++ b/third_party/xgrammar @@ -0,0 +1 @@ +Subproject commit 97787376faee5ed8466cfad57c99855e4ce2f6aa From 8a842f3293899051711e91200422027b22b5d09e Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 7 Oct 2026 14:48:32 -0600 Subject: [PATCH 03/18] Add a shared chat library: format detection, parsing, constraints Every model carried its own reasoning and tool-call parser. This adds one library, src/common/chat, that all of them can share: - chat_format: detects a model's format once, from its chat template and vocabulary (hermes_json, qwen_xml, gemma4, lfm2_pythonic, harmony, content_only), and derives the prefill the template already wrote. - chat_parser / chat_args: split output into reasoning, content and tool calls; convert Gemma4's relaxed syntax and LFM2's Python calls to JSON, including the irregular forms models produce. - chat_stream / chat_msg: reparse on every flush and emit only what changed. The parser holds back anything uncertain (partial markers, unfinished names, bare values), so every field only grows and the deltas add up to the non-streamed result. - chat_constraint / chat_grammar / decode_constraint: turn tool_choice, parallel_tool_calls and response_format into a per-token decoding constraint. tool_choice none bans the tool-call opener; everything else is an xgrammar structural tag built from the same format spec the parser uses (xgrammar's qwen_3, qwen_3_5 and harmony styles; hand-written EBNF for Gemma4 and LFM2 arguments). Nothing uses the library yet. Co-Authored-By: Claude --- src/common/chat/chat_args.cpp | 516 +++++++++++++++++ src/common/chat/chat_constraint.cpp | 98 ++++ src/common/chat/chat_format.cpp | 139 +++++ src/common/chat/chat_grammar.cpp | 739 +++++++++++++++++++++++++ src/common/chat/chat_msg.cpp | 54 ++ src/common/chat/chat_parser.cpp | 639 +++++++++++++++++++++ src/common/chat/chat_stream.cpp | 37 ++ src/include/chat/chat_args.hpp | 46 ++ src/include/chat/chat_constraint.hpp | 62 +++ src/include/chat/chat_format.hpp | 68 +++ src/include/chat/chat_grammar.hpp | 45 ++ src/include/chat/chat_msg.hpp | 60 ++ src/include/chat/chat_parser.hpp | 38 ++ src/include/chat/chat_stream.hpp | 43 ++ src/include/chat/decode_constraint.hpp | 66 +++ 15 files changed, 2650 insertions(+) create mode 100644 src/common/chat/chat_args.cpp create mode 100644 src/common/chat/chat_constraint.cpp create mode 100644 src/common/chat/chat_format.cpp create mode 100644 src/common/chat/chat_grammar.cpp create mode 100644 src/common/chat/chat_msg.cpp create mode 100644 src/common/chat/chat_parser.cpp create mode 100644 src/common/chat/chat_stream.cpp create mode 100644 src/include/chat/chat_args.hpp create mode 100644 src/include/chat/chat_constraint.hpp create mode 100644 src/include/chat/chat_format.hpp create mode 100644 src/include/chat/chat_grammar.hpp create mode 100644 src/include/chat/chat_msg.hpp create mode 100644 src/include/chat/chat_parser.hpp create mode 100644 src/include/chat/chat_stream.hpp create mode 100644 src/include/chat/decode_constraint.hpp diff --git a/src/common/chat/chat_args.cpp b/src/common/chat/chat_args.cpp new file mode 100644 index 000000000..06f60ef2b --- /dev/null +++ b/src/common/chat/chat_args.cpp @@ -0,0 +1,516 @@ +/// \file chat_args.cpp +/// \brief Incremental conversion of tool call arguments to JSON + +#include "chat/chat_args.hpp" + +#include + +#include + +namespace { + +const std::string GEMMA4_QUOTE = "<|\"|>"; + +bool is_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +bool is_closer(char c) { + return c == '}' || c == ']' || c == ')'; +} + +std::string trim(const std::string& s) { + size_t b = 0; + size_t e = s.size(); + while (b < e && is_ws(s[b])) b++; + while (e > b && is_ws(s[e - 1])) e--; + return s.substr(b, e - b); +} + +void append_utf8(std::string& out, uint32_t cp) { + if (cp < 0x80) { + out += static_cast(cp); + } else if (cp < 0x800) { + out += static_cast(0xC0 | (cp >> 6)); + out += static_cast(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + out += static_cast(0xE0 | (cp >> 12)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } else { + out += static_cast(0xF0 | (cp >> 18)); + out += static_cast(0x80 | ((cp >> 12) & 0x3F)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } +} + +bool parse_hex4(const std::string& s, size_t pos, uint32_t& value) { + if (pos + 4 > s.size()) { + return false; + } + value = 0; + for (size_t i = pos; i < pos + 4; i++) { + char c = s[i]; + value <<= 4; + if (c >= '0' && c <= '9') value |= c - '0'; + else if (c >= 'a' && c <= 'f') value |= c - 'a' + 10; + else if (c >= 'A' && c <= 'F') value |= c - 'A' + 10; + else return false; + } + return true; +} + +/// \brief Walks relaxed argument text and writes the JSON it means +/// \note Every method returns true when its construct ended normally and false +/// when the text ran out first. On running out the whole call stack +/// unwinds; with input_done each level closes what it opened on the way. +class relaxed_writer { +public: + relaxed_writer(const std::string& s, size_t pos, chat_relaxed_dialect dialect, bool input_done) + : s_(s), pos_(pos), dialect_(dialect), done_(input_done) {} + + chat_relaxed_result run() { + chat_relaxed_result result; + if (dialect_ == chat_relaxed_dialect::GEMMA4) { + if (pos_ < s_.size() && s_[pos_] == '{') { + pos_++; + result.closed = members('}', ':'); + } else if (done_) { + out_ = "{}"; + } + } else { + result.closed = members(')', '='); + } + result.json = std::move(out_); + result.end = pos_; + return result; + } + +private: + enum class key_status { OK, STARVED, SKIP }; + + const std::string& s_; + size_t pos_; + chat_relaxed_dialect dialect_; + bool done_; + std::string out_; + + bool starved() const { + return pos_ >= s_.size(); + } + + void skip_ws() { + while (pos_ < s_.size() && is_ws(s_[pos_])) pos_++; + } + + bool give_up(char closer) { + if (done_) { + out_ += closer; + } + return false; + } + + /// \brief true when the rest of the text could still grow into marker + bool partial_marker_at_end(const std::string& marker) const { + size_t rem = s_.size() - pos_; + return rem < marker.size() && marker.compare(0, rem, s_, pos_, rem) == 0; + } + + /// \brief Delimiter opening a string at pos_, empty if none + std::string string_delim_at() const { + if (dialect_ == chat_relaxed_dialect::GEMMA4 && s_.compare(pos_, GEMMA4_QUOTE.size(), GEMMA4_QUOTE) == 0) { + return GEMMA4_QUOTE; + } + char c = s_[pos_]; + if (c == '"' || (dialect_ == chat_relaxed_dialect::PYTHON && c == '\'')) { + return std::string(1, c); + } + return ""; + } + + /// \brief Read string content after its opening delimiter into dst + /// \param for_json escape what is appended so dst stays JSON string content + bool scan_string(const std::string& delim, std::string& dst, bool for_json) { + const bool quote_char = delim.size() == 1; + while (true) { + if (starved()) { + return false; + } + if (s_.compare(pos_, delim.size(), delim) == 0) { + pos_ += delim.size(); + return true; + } + if (!quote_char && !done_ && partial_marker_at_end(delim)) { + return false; + } + std::string piece; + char c = s_[pos_]; + if (quote_char && c == '\\') { + if (pos_ + 1 >= s_.size()) { + return false; + } + char e = s_[pos_ + 1]; + size_t used = 2; + switch (e) { + case 'n': piece = "\n"; break; + case 't': piece = "\t"; break; + case 'r': piece = "\r"; break; + case 'b': piece = "\b"; break; + case 'f': piece = "\f"; break; + case 'u': { + uint32_t cp = 0; + if (!parse_hex4(s_, pos_ + 2, cp)) { + if (pos_ + 6 > s_.size() && !done_) { + return false; + } + piece = "u"; + break; + } + used = 6; + if (cp >= 0xD800 && cp < 0xDC00) { + uint32_t low = 0; + if (pos_ + 12 > s_.size() && !done_) { + return false; + } + if (s_.compare(pos_ + 6, 2, "\\u") == 0 && parse_hex4(s_, pos_ + 8, low) && + low >= 0xDC00 && low < 0xE000) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00); + used = 12; + } + } + append_utf8(piece, cp); + break; + } + default: piece = std::string(1, e); break; + } + pos_ += used; + } else { + piece = std::string(1, c); + pos_++; + } + if (for_json) { + chat_json_escape_append(dst, piece); + } else { + dst += piece; + } + } + } + + /// \brief Read a Gemma4 <|"|> string after its opener into dst, JSON escaped + /// \note Gemma4 strings are raw text, so a <|"|> (or the plain " or ` the + /// model sometimes writes instead) only closes the string when the + /// value ends right after it; otherwise it is part of the text. Until + /// the next non-space character shows which, the candidate is held. + bool scan_gemma4_string(std::string& dst) { + while (true) { + if (starved()) { + return false; + } + size_t len = 0; + if (s_.compare(pos_, GEMMA4_QUOTE.size(), GEMMA4_QUOTE) == 0) { + len = GEMMA4_QUOTE.size(); + } else if (s_[pos_] == '"' || s_[pos_] == '`') { + len = 1; + } else if (!done_ && partial_marker_at_end(GEMMA4_QUOTE)) { + return false; + } + if (len > 0) { + size_t next = pos_ + len; + while (next < s_.size() && is_ws(s_[next])) next++; + if (next >= s_.size() && !done_) { + return false; + } + if (next >= s_.size() || s_[next] == ',' || is_closer(s_[next])) { + pos_ += len; + return true; + } + chat_json_escape_append(dst, s_.substr(pos_, len)); + pos_ += len; + continue; + } + chat_json_escape_append(dst, std::string(1, s_[pos_])); + pos_++; + } + } + + bool string_value(const std::string& delim) { + pos_ += delim.size(); + out_ += '"'; + bool closed = delim == GEMMA4_QUOTE ? scan_gemma4_string(out_) : scan_string(delim, out_, true); + if (closed || done_) { + out_ += '"'; + } + return closed; + } + + bool scalar() { + size_t start = pos_; + int depth = 0; + while (pos_ < s_.size()) { + char c = s_[pos_]; + if (c == '{' || c == '[' || c == '(') { + depth++; + } else if (is_closer(c) || c == ',') { + if (depth == 0) break; + if (c != ',') depth--; + } + pos_++; + } + if (starved() && !done_) { + // a bare value is only known once it ends: "1" may become "12" + return false; + } + std::string token = s_.substr(start, pos_ - start); + if (dialect_ == chat_relaxed_dialect::GEMMA4) { + // a string whose <|"|> opener the model left out still has its closer + std::string trimmed = trim(token); + if (trimmed.size() >= GEMMA4_QUOTE.size() && + trimmed.compare(trimmed.size() - GEMMA4_QUOTE.size(), GEMMA4_QUOTE.size(), GEMMA4_QUOTE) == 0) { + token = trimmed.substr(0, trimmed.size() - GEMMA4_QUOTE.size()); + out_ += '"'; + chat_json_escape_append(out_, trim(token)); + out_ += '"'; + return !starved(); + } + } + out_ += chat_scalar_to_json(token); + return !starved(); + } + + bool value() { + skip_ws(); + if (starved()) { + if (done_) out_ += "null"; + return false; + } + if (dialect_ == chat_relaxed_dialect::GEMMA4 && !done_ && partial_marker_at_end(GEMMA4_QUOTE)) { + return false; + } + std::string delim = string_delim_at(); + if (!delim.empty()) { + return string_value(delim); + } + char c = s_[pos_]; + if (c == '{') { + pos_++; + return members('}', ':'); + } + if (c == '[' || (c == '(' && dialect_ == chat_relaxed_dialect::PYTHON)) { + pos_++; + return array(); + } + return scalar(); + } + + key_status read_key(std::string& key, char sep) { + if (dialect_ == chat_relaxed_dialect::GEMMA4 && !done_ && partial_marker_at_end(GEMMA4_QUOTE)) { + return key_status::STARVED; + } + std::string delim = string_delim_at(); + if (!delim.empty()) { + pos_ += delim.size(); + if (!scan_string(delim, key, false)) { + return key_status::STARVED; + } + skip_ws(); + if (starved()) { + return key_status::STARVED; + } + if (s_[pos_] == sep || s_[pos_] == ':' || s_[pos_] == '=') { + pos_++; + return key_status::OK; + } + return key_status::SKIP; + } + size_t start = pos_; + while (pos_ < s_.size() && s_[pos_] != sep) { + if (s_[pos_] == ',' || is_closer(s_[pos_])) { + // a value with no name, nothing to map it to + return key_status::SKIP; + } + pos_++; + } + if (starved()) { + return key_status::STARVED; + } + key = trim(s_.substr(start, pos_ - start)); + pos_++; + return key_status::OK; + } + + bool members(char close, char sep) { + out_ += '{'; + bool first = true; + while (true) { + skip_ws(); + if (starved()) { + return give_up('}'); + } + char c = s_[pos_]; + if (c == close || is_closer(c)) { + pos_++; + out_ += '}'; + return true; + } + if (c == ',') { + pos_++; + continue; + } + std::string key; + key_status status = read_key(key, sep); + if (status == key_status::STARVED) { + return give_up('}'); + } + if (status == key_status::SKIP) { + if (pos_ < s_.size() && s_[pos_] == ',') pos_++; + continue; + } + if (!first) { + out_ += ','; + } + first = false; + out_ += '"'; + chat_json_escape_append(out_, key); + out_ += "\":"; + if (!value()) { + return give_up('}'); + } + } + } + + bool array() { + out_ += '['; + bool first = true; + while (true) { + skip_ws(); + if (starved()) { + return give_up(']'); + } + char c = s_[pos_]; + if (is_closer(c)) { + pos_++; + out_ += ']'; + return true; + } + if (c == ',') { + pos_++; + continue; + } + // something starts here, so a value will follow and the comma is safe to send + if (!first) { + out_ += ','; + } + first = false; + if (!value()) { + return give_up(']'); + } + } + } +}; + +} // namespace + +void chat_json_escape_append(std::string& out, const std::string& s) { + static const char HEX[] = "0123456789abcdef"; + for (char ch : s) { + unsigned char c = static_cast(ch); + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (c < 0x20) { + out += "\\u00"; + out += HEX[c >> 4]; + out += HEX[c & 0xF]; + } else { + out += ch; + } + } + } +} + +size_t chat_json_value_end(const std::string& s, size_t pos) { + if (pos >= s.size()) { + return std::string::npos; + } + char c = s[pos]; + if (c == '"') { + for (size_t i = pos + 1; i < s.size(); i++) { + if (s[i] == '\\') { + i++; + } else if (s[i] == '"') { + return i + 1; + } + } + return std::string::npos; + } + if (c == '{' || c == '[') { + int depth = 0; + bool in_string = false; + for (size_t i = pos; i < s.size(); i++) { + char d = s[i]; + if (in_string) { + if (d == '\\') i++; + else if (d == '"') in_string = false; + continue; + } + if (d == '"') { + in_string = true; + } else if (d == '{' || d == '[') { + depth++; + } else if (d == '}' || d == ']') { + if (--depth == 0) { + return i + 1; + } + } + } + return std::string::npos; + } + // literal: it may still be growing until a delimiter shows up + for (size_t i = pos; i < s.size(); i++) { + char d = s[i]; + if (d == ',' || d == '}' || d == ']' || is_ws(d)) { + return i; + } + } + return std::string::npos; +} + +std::string chat_scalar_to_json(const std::string& token) { + std::string t = trim(token); + if (t.empty() || t == "None" || t == "null") { + return "null"; + } + if (t == "True" || t == "true") { + return "true"; + } + if (t == "False" || t == "false") { + return "false"; + } + char c = t[0]; + if (c == '-' || (c >= '0' && c <= '9')) { + try { + nlohmann::json j = nlohmann::json::parse(t); + if (j.is_number()) { + return t; + } + } catch (const nlohmann::json::exception&) { + } + } + std::string out = "\""; + chat_json_escape_append(out, t); + out += '"'; + return out; +} + +chat_relaxed_result chat_relaxed_args_to_json(const std::string& s, size_t pos, + chat_relaxed_dialect dialect, bool input_done) { + relaxed_writer writer(s, pos, dialect, input_done); + return writer.run(); +} diff --git a/src/common/chat/chat_constraint.cpp b/src/common/chat/chat_constraint.cpp new file mode 100644 index 000000000..514592c7a --- /dev/null +++ b/src/common/chat/chat_constraint.cpp @@ -0,0 +1,98 @@ +/// \file chat_constraint.cpp +/// \brief Build the decoding constraint a chat request asks for + +#include "chat/chat_constraint.hpp" +#include "chat/chat_grammar.hpp" + +using json = nlohmann::ordered_json; + +bool chat_constraint_request_from_openai(const json& request, const json& tools, + chat_constraint_request& out, std::string& error) { + out = chat_constraint_request(); + out.tools = &tools; + out.parallel_tool_calls = request.value("parallel_tool_calls", true); + // thinking turned off: some models still get an empty think block fed in, but + // the grammar must not offer room to reason + if (request.contains("think") && request["think"].is_boolean() && !request["think"].get()) { + out.allow_reasoning = false; + } + if (request.value("reasoning_effort", "") == "none") { + out.allow_reasoning = false; + } + + const json choice = request.value("tool_choice", json("auto")); + if (choice.is_string()) { + const std::string mode = choice.get(); + if (mode == "auto") { + out.tool_choice = chat_tool_choice::AUTO; + } else if (mode == "none") { + out.tool_choice = chat_tool_choice::NONE; + } else if (mode == "required") { + out.tool_choice = chat_tool_choice::REQUIRED; + } else { + error = "tool_choice must be \"auto\", \"none\", \"required\" or a function, got \"" + mode + "\""; + return false; + } + } else if (choice.is_object() && choice.value("type", "") == "function" && choice.contains("function") && + choice["function"].is_object() && choice["function"].contains("name")) { + out.tool_choice = chat_tool_choice::FUNCTION; + out.function_name = choice["function"]["name"].get(); + } else { + error = "tool_choice must be a string or {\"type\": \"function\", \"function\": {\"name\": ...}}"; + return false; + } + + if (out.tool_choice == chat_tool_choice::REQUIRED || out.tool_choice == chat_tool_choice::FUNCTION) { + if (!tools.is_array() || tools.empty()) { + error = "tool_choice requires a tool call but the request has no tools"; + return false; + } + } + if (out.tool_choice == chat_tool_choice::FUNCTION) { + bool found = false; + for (const auto& tool : tools) { + if (tool.contains("function") && tool["function"].value("name", "") == out.function_name) { + found = true; + } + } + if (!found) { + error = "tool_choice names function \"" + out.function_name + "\" which is not in tools"; + return false; + } + } + + if (request.contains("response_format") && request["response_format"].is_object()) { + const json& format = request["response_format"]; + const std::string type = format.value("type", "text"); + if (type == "json_object") { + out.json_object = true; + } else if (type == "json_schema") { + if (!format.contains("json_schema") || !format["json_schema"].contains("schema")) { + error = "response_format json_schema needs json_schema.schema"; + return false; + } + out.json_schema = format["json_schema"]["schema"]; + } else if (type != "text") { + error = "response_format type must be text, json_object or json_schema"; + return false; + } + } + return true; +} + +std::shared_ptr chat_make_constraint(const chat_format_spec& spec, + const chat_constraint_request& request, + const chat_vocab& vocab, + const std::function& grammar_engine) { + if (!chat_structural_tag(spec, request).is_null()) { + return grammar_engine().make(spec, request); + } + if (request.tool_choice == chat_tool_choice::NONE && !spec.tool_start.empty()) { + // a single token opens every tool call block: taking it away is enough + std::vector ids = vocab.encode(spec.tool_start); + if (ids.size() == 1) { + return std::make_shared(vocab.size, ids); + } + } + return nullptr; +} diff --git a/src/common/chat/chat_format.cpp b/src/common/chat/chat_format.cpp new file mode 100644 index 000000000..ad866424e --- /dev/null +++ b/src/common/chat/chat_format.cpp @@ -0,0 +1,139 @@ +/// \file chat_format.cpp +/// \brief Chat format detection and prefill computation + +#include "chat/chat_format.hpp" + +namespace { + +struct format_name_entry { + chat_format_kind kind; + const char* name; +}; + +const format_name_entry FORMAT_NAMES[] = { + {chat_format_kind::CONTENT_ONLY, "content_only"}, + {chat_format_kind::HERMES_JSON, "hermes_json"}, + {chat_format_kind::QWEN_XML, "qwen_xml"}, + {chat_format_kind::LFM2_PYTHONIC, "lfm2_pythonic"}, + {chat_format_kind::GEMMA4, "gemma4"}, + {chat_format_kind::HARMONY, "harmony"}, +}; + +} // namespace + +const char* chat_format_name(chat_format_kind kind) { + for (const auto& entry : FORMAT_NAMES) { + if (entry.kind == kind) { + return entry.name; + } + } + return "unknown"; +} + +bool chat_format_from_name(const std::string& name, chat_format_kind& kind) { + for (const auto& entry : FORMAT_NAMES) { + if (name == entry.name) { + kind = entry.kind; + return true; + } + } + return false; +} + +chat_format_spec chat_format_make(chat_format_kind kind, bool with_reasoning) { + chat_format_spec spec; + spec.kind = kind; + if (with_reasoning) { + spec.reasoning_start = ""; + spec.reasoning_end = ""; + } + switch (kind) { + case chat_format_kind::CONTENT_ONLY: + break; + case chat_format_kind::HERMES_JSON: + case chat_format_kind::QWEN_XML: + spec.tool_start = ""; + spec.tool_end = ""; + break; + case chat_format_kind::LFM2_PYTHONIC: + spec.tool_start = "<|tool_call_start|>"; + spec.tool_end = "<|tool_call_end|>"; + break; + case chat_format_kind::GEMMA4: + // Gemma4 always reasons in its own channel, never in + spec.reasoning_start = "<|channel>thought"; + spec.reasoning_end = ""; + spec.tool_start = "<|tool_call>"; + spec.tool_end = ""; + // the model hands the turn to the tool here; anything after is not its answer + spec.stop_marker = "<|tool_response>"; + spec.extra_stops = {"<|tool_response>"}; + break; + case chat_format_kind::HARMONY: + // channels, not tags; the harmony parser knows its own markers + spec.reasoning_start.clear(); + spec.reasoning_end.clear(); + // a call ends the turn, the tool answers next; gpt-oss only lists <|return|> as eos + spec.extra_stops = {"<|call|>"}; + break; + } + return spec; +} + +chat_format_spec chat_format_detect(const std::string& template_src, + const std::function& is_token) { + auto in_template = [&template_src](const char* marker) { + return template_src.find(marker) != std::string::npos; + }; + auto known = [&](const char* marker) { + return in_template(marker) || is_token(marker); + }; + + const bool with_reasoning = known("") && known(""); + + // most specific first: these markers are unique to one family + if (known("<|channel|>") && known("<|message|>")) { + return chat_format_make(chat_format_kind::HARMONY, false); + } + if (known("<|tool_call>") && known("<|channel>")) { + return chat_format_make(chat_format_kind::GEMMA4, false); + } + if (known("<|tool_call_start|>")) { + return chat_format_make(chat_format_kind::LFM2_PYTHONIC, with_reasoning); + } + if (in_template("")) { + return chat_format_make(chat_format_kind::HERMES_JSON, with_reasoning); + } + chat_format_spec spec = chat_format_make(chat_format_kind::CONTENT_ONLY, with_reasoning); + // DeepSeek-R1 and its distills were trained to always open with reasoning; + // after a tool result the template does not open , yet they reason anyway + spec.reasoning_forced = with_reasoning && in_template("<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>"); + return spec; +} + +std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail) { + if (spec.kind == chat_format_kind::HARMONY) { + // the generation prompt ends with an open "<|start|>assistant" header, and + // may have started a channel too; the header is parsed with the output + size_t start = prompt_tail.rfind("<|start|>"); + return start == std::string::npos ? std::string() : prompt_tail.substr(start + 9); + } + if (!spec.has_reasoning()) { + return ""; + } + // a reasoning block opened by the template and not closed yet is where the + // output starts; one closed by the template (thinking disabled) is not + size_t open = prompt_tail.rfind(spec.reasoning_start); + size_t close = prompt_tail.rfind(spec.reasoning_end); + if (open == std::string::npos || (close != std::string::npos && close > open)) { + return spec.reasoning_forced ? spec.reasoning_start : ""; + } + // only the tag: what follows it in the prompt is whitespace, and some + // decoders render that as byte tokens like <0x0A> + return spec.reasoning_start; +} diff --git a/src/common/chat/chat_grammar.cpp b/src/common/chat/chat_grammar.cpp new file mode 100644 index 000000000..4d0219ce0 --- /dev/null +++ b/src/common/chat/chat_grammar.cpp @@ -0,0 +1,739 @@ +/// \file chat_grammar.cpp +/// \brief Grammar constraints for tool calls and response_format, on xgrammar + +#include "chat/chat_grammar.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "utils/debug_utils.hpp" + +namespace { + +using json = nlohmann::ordered_json; + +// ---------------------------------------------------------------- structural tag pieces + +json any_text(const std::vector& excludes = {}) { + json j = {{"type", "any_text"}}; + if (!excludes.empty()) { + j["excludes"] = excludes; + } + return j; +} + +json const_string(const std::string& value) { + return {{"type", "const_string"}, {"value", value}}; +} + +json tag(const std::string& begin, json content, json end) { + return {{"type", "tag"}, {"begin", begin}, {"content", std::move(content)}, {"end", std::move(end)}}; +} + +json sequence(std::vector elements) { + return {{"type", "sequence"}, {"elements", std::move(elements)}}; +} + +json optional_format(json content) { + return {{"type", "optional"}, {"content", std::move(content)}}; +} + +/// \note Whitespace is bounded everywhere: a model kept from what it wants to write +/// (a ``` fence before JSON, say) otherwise tends to pad with blanks forever. +json whitespace() { + return {{"type", "regex"}, {"pattern", "[ \\t\\r\\n]{0,8}"}}; +} + +constexpr int MAX_JSON_WHITESPACE = 16; // room for indented JSON, not for endless padding + +json schema_content(const json& schema, const char* style = nullptr) { + json j = {{"type", "json_schema"}, {"json_schema", schema}, {"max_whitespace_cnt", MAX_JSON_WHITESPACE}}; + if (style != nullptr) { + j["style"] = style; + } + return j; +} + +/// \brief The arguments schema as Qwen XML spells top level values +/// \note The Qwen3.5+ template renders a parameter with Jinja's |string, so the +/// model learned Python's True/False for booleans, while xgrammar's qwen_xml +/// style only allows JSON's true/false. Strings are written raw in this +/// style, so a string enum lets both through; the parser reads either. +json qwen_xml_schema(json schema) { + if (!schema.is_object() || !schema.contains("properties") || !schema["properties"].is_object()) { + return schema; + } + for (auto& [name, property] : schema["properties"].items()) { + if (property.is_object() && property.value("type", json()) == "boolean") { + json replacement = {{"type", "string"}, {"enum", {"True", "False", "true", "false"}}}; + if (property.contains("description")) replacement["description"] = property["description"]; + property = replacement; + } + } + return schema; +} + +json grammar_content(const std::string& ebnf) { + return {{"type", "grammar"}, {"grammar", ebnf}}; +} + +// ---------------------------------------------------------------- tools of a request + +/// \brief Spell out what a schema leaves implicit where xgrammar needs it +/// \note An array without "items" means any items; the qwen_xml style rejects +/// every value of such an array unless that is written as "items": {}. +json normalize_schema(json schema) { + if (schema.is_array()) { + for (auto& item : schema) item = normalize_schema(std::move(item)); + return schema; + } + if (!schema.is_object()) { + return schema; + } + if (schema.value("type", json()) == "array" && !schema.contains("items") && !schema.contains("prefixItems")) { + schema["items"] = json::object(); + } + for (auto& [key, value] : schema.items()) { + if (value.is_object() || value.is_array()) { + value = normalize_schema(std::move(value)); + } + } + return schema; +} + +struct tool_def { + std::string name; + json parameters; // a JSON schema, or true for any arguments +}; + +std::vector request_tools(const chat_constraint_request& request) { + std::vector tools; + if (request.tools == nullptr || !request.tools->is_array()) { + return tools; + } + for (const auto& entry : *request.tools) { + if (!entry.contains("function") || !entry["function"].is_object()) { + continue; + } + const json& fn = entry["function"]; + tool_def tool; + tool.name = fn.value("name", ""); + if (tool.name.empty()) { + continue; + } + if (request.tool_choice == chat_tool_choice::FUNCTION && tool.name != request.function_name) { + continue; + } + // as xgrammar does: no schema, or a non-strict tool, still gets well formed arguments + const bool strict = fn.value("strict", true); + tool.parameters = strict && fn.contains("parameters") && fn["parameters"].is_object() + ? normalize_schema(fn["parameters"]) : json(true); + tools.push_back(std::move(tool)); + } + return tools; +} + +// ---------------------------------------------------------------- argument grammars (EBNF) + +std::string ebnf_literal(const std::string& text) { + std::string out = "\""; + for (char c : text) { + if (c == '"' || c == '\\') out += '\\'; + if (c == '\n') { out += "\\n"; continue; } + out += c; + } + return out + "\""; +} + +/// \brief The rule for one parameter's value, picked from its schema type +std::string value_rule(const json& property, const char* generic) { + if (!property.is_object()) { + return generic; + } + if (property.contains("enum") && property["enum"].is_array() && !property["enum"].empty()) { + bool all_strings = true; + for (const auto& v : property["enum"]) all_strings = all_strings && v.is_string(); + if (all_strings) { + return ""; // the caller spells the alternatives out + } + } + const std::string type = property.contains("type") && property["type"].is_string() + ? property["type"].get() : ""; + if (type == "string") return "string"; + if (type == "integer") return "integer"; + if (type == "number") return "number"; + if (type == "boolean") return "boolean"; + if (type == "array") return "array"; + if (type == "object") return "object"; + return generic; +} + +/// \brief EBNF for one property's value, from its schema type +/// \param quote_string spells a string enum value the way the dialect writes strings +/// \param array_sep EBNF between two array items +/// \note An array follows its "items" schema, so enums and types inside lists are +/// kept too; objects nested in a value are only held to the dialect's syntax. +std::string property_value(const json& property, const std::function& quote_string, + const std::string& array_sep) { + std::string value = value_rule(property, "value"); + if (value == "array" && property.contains("items") && property["items"].is_object() && + !property["items"].empty()) { + const std::string item = property_value(property["items"], quote_string, array_sep); + return "(\"[\" (" + item + " (" + array_sep + " " + item + ")*)? \"]\")"; + } + if (!value.empty()) { + return value; + } + value = "("; + bool first = true; + for (const auto& option : property["enum"]) { + value += (first ? "" : " | ") + quote_string(option.get()); + first = false; + } + return value + ")"; +} + +/// \brief Rules for the argument list of one call +/// \param prefix keeps the rule names of different tools apart in one grammar +/// \param kv_sep EBNF between a key and its value +/// \param item_sep EBNF between two arguments +/// \param array_sep EBNF between two items of an array value +/// \param sorted keys come in alphabetical order (Gemma4's template uses dictsort), +/// otherwise in declaration order +/// \return rules whose list rule is args; a schema without declared +/// properties gets any key/value list, built from the caller's "pair" rule +/// \note The declared properties come in one fixed order, each required one present +/// and each optional one free to be left out. Rule first is the list +/// starting at property i, rest the remainder after something was written. +std::string argument_list_rules(const json& schema, const std::string& prefix, const std::string& kv_sep, + const std::string& item_sep, const std::string& array_sep, + const std::function& quote_string, bool sorted) { + std::ostringstream g; + const bool declared = schema.is_object() && schema.contains("properties") && schema["properties"].is_object() && + !schema["properties"].empty() && schema.value("additionalProperties", false) == false; + if (!declared) { + g << prefix << "args ::= (pair (" << item_sep << " pair)*)?\n"; + return g.str(); + } + std::vector names; + for (const auto& [name, property] : schema["properties"].items()) { + names.push_back(name); + } + if (sorted) { + std::sort(names.begin(), names.end()); + } + std::set required; + for (const auto& name : schema.value("required", json::array())) { + if (name.is_string()) required.insert(name.get()); + } + + const size_t n = names.size(); + auto rule = [&prefix](const char* kind, size_t i) { return prefix + kind + std::to_string(i); }; + for (size_t i = 0; i < n; i++) { + g << rule("p", i) << " ::= " << ebnf_literal(names[i]) << " " << kv_sep << " " + << property_value(schema["properties"][names[i]], quote_string, array_sep) << "\n"; + } + for (size_t i = 0; i <= n; i++) { + if (i == n) { + g << rule("first", i) << " ::= \"\"\n" << rule("rest", i) << " ::= \"\"\n"; + continue; + } + const std::string with_first = rule("p", i) + " " + rule("rest", i + 1); + const std::string with_rest = item_sep + " " + rule("p", i) + " " + rule("rest", i + 1); + if (required.count(names[i])) { + g << rule("first", i) << " ::= " << with_first << "\n"; + g << rule("rest", i) << " ::= " << with_rest << "\n"; + } else { + g << rule("first", i) << " ::= (" << with_first << ") | " << rule("first", i + 1) << "\n"; + g << rule("rest", i) << " ::= (" << with_rest << ") | " << rule("rest", i + 1) << "\n"; + } + } + g << prefix << "args ::= " << rule("first", 0) << "\n"; + return g.str(); +} + +/// \brief Numbers with bounded digits: a model kept from writing "13.9 million" will +/// otherwise go on adding digits until it runs out of tokens +const char* BOUNDED_NUMBER_RULES = + "integer ::= \"-\"? (\"0\" | [1-9] [0-9]{0,15})\n" + "number ::= integer (\".\" [0-9]{1,15})? ([eE] [+-]? [0-9]{1,3})?\n"; + +/// \brief Gemma4 call arguments: {key:<|"|>text<|"|>,n:5,flag:true,list:[...]} +std::string gemma4_arguments_grammar(const json& schema) { + auto quote = [](const std::string& s) { return ebnf_literal("<|\"|>" + s + "<|\"|>"); }; + std::ostringstream g; + g << "root ::= \"{\" args \"}\"\n" + << argument_list_rules(schema, "", "\":\"", "\",\"", "\",\"", quote, true) + << "pair ::= ident \":\" value\n" + << "ident ::= [a-zA-Z_] [a-zA-Z0-9_]*\n" + << "value ::= string | number | boolean | \"null\" | object | array\n" + << "string ::= \"<|\\\"|>\" ([^<] | \"<\" [^|])* \"<|\\\"|>\"\n" + << BOUNDED_NUMBER_RULES + << "boolean ::= \"true\" | \"false\"\n" + << "object ::= \"{\" (pair (\",\" pair)*)? \"}\"\n" + << "array ::= \"[\" (value (\",\" value)*)? \"]\"\n"; + return g.str(); +} + +/// \brief The calls inside one LFM2 block: f(key="text", n=5), g(flag=True) +std::string python_calls_grammar(const std::vector& tools, bool parallel) { + auto quote = [](const std::string& s) { return ebnf_literal("\"" + s + "\""); }; + std::ostringstream g; + g << "root ::= call" << (parallel ? " (\",\" \" \"? call)*" : "") << "\n"; + g << "call ::= "; + for (size_t i = 0; i < tools.size(); i++) { + g << (i ? " | " : "") << "call_" << i; + } + g << "\n"; + for (size_t i = 0; i < tools.size(); i++) { + const std::string prefix = "t" + std::to_string(i) + "_"; + g << "call_" << i << " ::= " << ebnf_literal(tools[i].name + "(") << " " << prefix << "args \")\"\n" + << argument_list_rules(tools[i].parameters, prefix, "\"=\"", "\",\" \" \"?", "\",\" \" \"?", quote, false); + } + g << "pair ::= ident \"=\" value\n" + << "ident ::= [a-zA-Z_] [a-zA-Z0-9_]*\n" + << "value ::= string | number | boolean | \"None\" | object | array\n" + << "string ::= \"\\\"\" ([^\"\\\\] | \"\\\\\" [^\\n])* \"\\\"\" | \"'\" ([^'\\\\] | \"\\\\\" [^\\n])* \"'\"\n" + << BOUNDED_NUMBER_RULES + << "boolean ::= \"True\" | \"False\"\n" + << "object ::= \"{\" (string \":\" \" \"? value (\",\" \" \"? string \":\" \" \"? value)*)? \"}\"\n" + << "array ::= \"[\" (value (\",\" \" \"? value)*)? \"]\"\n"; + return g.str(); +} + +// ---------------------------------------------------------------- per format tool tags + +struct tool_syntax { + std::string trigger; + std::vector tags; +}; + +tool_syntax tool_tags(const chat_format_spec& spec, const std::vector& tools, bool parallel) { + tool_syntax syntax; + if (spec.kind == chat_format_kind::LFM2_PYTHONIC) { + // LFM2 lists every call of a turn in one block, so one tag holds them all + syntax.trigger = "<|tool_call_start|>"; + syntax.tags.push_back(tag("<|tool_call_start|>[", grammar_content(python_calls_grammar(tools, parallel)), + "]<|tool_call_end|>")); + return syntax; + } + for (const auto& tool : tools) { + switch (spec.kind) { + case chat_format_kind::HERMES_JSON: + // xgrammar's qwen_3 style + syntax.trigger = ""; + syntax.tags.push_back(tag("\n{\"name\": \"" + tool.name + "\", \"arguments\": ", + schema_content(tool.parameters), "}\n")); + break; + case chat_format_kind::QWEN_XML: + // xgrammar's qwen_3_5 style + syntax.trigger = "\n\n\n", + schema_content(qwen_xml_schema(tool.parameters), "qwen_xml"), + "\n\n")); + break; + case chat_format_kind::GEMMA4: + syntax.trigger = "<|tool_call>"; + syntax.tags.push_back(tag("<|tool_call>call:" + tool.name, + grammar_content(gemma4_arguments_grammar(tool.parameters)), "")); + break; + case chat_format_kind::LFM2_PYTHONIC: + case chat_format_kind::CONTENT_ONLY: + case chat_format_kind::HARMONY: + break; + } + } + return syntax; +} + +/// \brief Optional reasoning block in front of the answer, for tag formats +/// \param allow false when the request turned thinking off; models that always +/// open with reasoning keep the block, their prefill already holds the tag +json reasoning_prefix(const chat_format_spec& spec, bool allow) { + if (!spec.has_reasoning() || !(allow || spec.reasoning_forced)) { + return nullptr; + } + // the prefill is fed to the grammar too, so a block the template opened is + // already inside this optional tag when generation starts + return optional_format(sequence({ + tag(spec.reasoning_start, any_text({spec.reasoning_start, spec.reasoning_end}), spec.reasoning_end), + whitespace(), + })); +} + +json response_schema(const chat_constraint_request& request) { + if (!request.json_schema.is_null()) { + return request.json_schema; + } + return json{{"type", "object"}}; +} + +bool has_response_format(const chat_constraint_request& request) { + return request.json_object || !request.json_schema.is_null(); +} + +json harmony_format(const chat_constraint_request& request, const std::vector& tools) { + // xgrammar's harmony style + const std::string CALL_END = "<|call|>"; + const json FINAL_END = json::array({"<|end|>", "<|return|>"}); + const std::string SEPARATOR = "<|start|>assistant"; + + std::vector call_tags; + const bool want_tools = request.tool_choice == chat_tool_choice::REQUIRED || + request.tool_choice == chat_tool_choice::FUNCTION || + (request.tool_choice == chat_tool_choice::AUTO && !has_response_format(request)); + if (want_tools) { + for (const auto& tool : tools) { + json content = schema_content(tool.parameters); + // gpt-oss writes "to=functions.x <|constrain|>json"; xgrammar's style has no space + call_tags.push_back(tag("<|channel|>commentary to=functions." + tool.name + " <|constrain|>json<|message|>", content, CALL_END)); + call_tags.push_back(tag("<|channel|>commentary to=functions." + tool.name + "<|constrain|>json<|message|>", content, CALL_END)); + call_tags.push_back(tag(" to=functions." + tool.name + "<|channel|>commentary <|constrain|>json<|message|>", content, CALL_END)); + call_tags.push_back(tag(" to=functions." + tool.name + "<|channel|>commentary json<|message|>", content, CALL_END)); + } + } + std::vector message_tags; + message_tags.push_back(tag("<|channel|>analysis<|message|>", any_text(), FINAL_END)); + if (request.tool_choice == chat_tool_choice::AUTO || request.tool_choice == chat_tool_choice::NONE) { + json final_content = has_response_format(request) ? schema_content(response_schema(request)) : any_text(); + message_tags.push_back(tag("<|channel|>final<|message|>", final_content, FINAL_END)); + } + + if (call_tags.empty()) { + return {{"type", "tags_with_separator"}, {"tags", message_tags}, {"separator", SEPARATOR}, {"at_least_one", true}}; + } + const bool must_call = request.tool_choice != chat_tool_choice::AUTO; + if (request.parallel_tool_calls && !must_call) { + std::vector all = call_tags; + all.insert(all.end(), message_tags.begin(), message_tags.end()); + return {{"type", "tags_with_separator"}, {"tags", all}, {"separator", SEPARATOR}}; + } + // any number of reasoning/answer messages, then the call(s), which end the turn + const bool one_call = !request.parallel_tool_calls || request.tool_choice == chat_tool_choice::FUNCTION; + json calls = {{"type", "tags_with_separator"}, {"tags", call_tags}, {"separator", SEPARATOR}, + {"at_least_one", true}, {"stop_after_first", one_call}}; + json messages = {{"type", "tags_with_separator"}, {"tags", message_tags}, {"separator", SEPARATOR}, + {"at_least_one", true}}; + json messages_then_calls = sequence({messages, const_string(SEPARATOR), calls}); + std::vector options = {calls, messages_then_calls}; + if (!must_call) { + options.insert(options.begin(), messages); + } + return {{"type", "or"}, {"elements", options}}; +} + +/// \brief Stop tokens the grammar matches as text +/// \note xgrammar never lets a stop token stand for text, so these stay out of the +/// stop list although the model's eos list has them (gpt-oss ends a call with <|call|>) +const std::set& text_markers(const chat_format_spec& spec) { + static const std::set harmony = {"<|call|>", "<|return|>", "<|end|>"}; + static const std::set none; + return spec.kind == chat_format_kind::HARMONY ? harmony : none; +} + +/// \brief What to drop from the prefill before feeding it to the grammar +std::string prefill_skip(const chat_format_spec& spec) { + // the harmony prefill opens with the role of the "<|start|>assistant" header, + // which the grammar does not cover + return spec.kind == chat_format_kind::HARMONY ? "assistant" : ""; +} + +// ---------------------------------------------------------------- the constraint + +class xgrammar_constraint : public decode_constraint { +public: + /// \param with_reasoning when the grammar leaves reasoning out, the same grammar with + /// it: a model made to open a reasoning block switches to this one + xgrammar_constraint(const xgrammar::CompiledGrammar& grammar, std::optional with_reasoning, + std::string prefill_skip, size_t vocab_size) + : matcher_(grammar, std::nullopt, false, MAX_ROLLBACK), + with_reasoning_(std::move(with_reasoning)), + skip_(std::move(prefill_skip)), + words_((vocab_size + 31) / 32) {} + + void start(const std::string& prefill) override { + std::string text = prefill; + if (!skip_.empty() && text.compare(0, skip_.size(), skip_) == 0) { + text = text.substr(skip_.size()); + } + prefill_ = text; + if (!text.empty() && !matcher_.AcceptString(text)) { + give_up("the grammar does not accept the prompt's prefill \"" + text + "\""); + } + } + + bool fill_mask(std::vector& mask) override { + if (failed_) { + return false; + } + mask.resize(words_); + int64_t shape[1] = {static_cast(words_)}; + DLTensor tensor; + tensor.data = mask.data(); + tensor.device = DLDevice{kDLCPU, 0}; + tensor.ndim = 1; + tensor.dtype = DLDataType{kDLInt, 32, 1}; + tensor.shape = shape; + tensor.strides = nullptr; + tensor.byte_offset = 0; + return matcher_.FillNextTokenBitmask(&tensor); + } + + void accept(int token) override { + if (failed_) { + return; + } + if (!matcher_.AcceptToken(token)) { + give_up("the grammar rejected token " + std::to_string(token)); + return; + } + history_.push_back(token); + } + + void accept_forced(int token) override { + if (failed_) { + return; + } + if (matcher_.AcceptToken(token)) { + history_.push_back(token); + return; + } + if (!with_reasoning_) { + give_up("the grammar rejected forced token " + std::to_string(token)); + return; + } + // the model is made to reason although the request turned thinking off: + // carry on with the grammar that has the reasoning block, from the same point + xgrammar::GrammarMatcher reasoning(*with_reasoning_, std::nullopt, false, MAX_ROLLBACK); + with_reasoning_.reset(); + bool ok = prefill_.empty() || reasoning.AcceptString(prefill_); + for (size_t i = 0; ok && i < history_.size(); i++) { + ok = reasoning.AcceptToken(history_[i]); + } + if (!ok || !reasoning.AcceptToken(token)) { + give_up("the grammar rejected forced token " + std::to_string(token)); + return; + } + matcher_ = std::move(reasoning); + history_.push_back(token); + } + + void rollback(int n) override { + if (!failed_) { + matcher_.Rollback(n); + history_.resize(history_.size() > static_cast(n) ? history_.size() - n : 0); + } + } + +private: + static constexpr int MAX_ROLLBACK = 8; + + void give_up(const std::string& why) { + // a desynchronised grammar would only produce garbage; let the model finish free + header_print("WARNING", "Constrained decoding disabled for this request: " << why); + failed_ = true; + } + + xgrammar::GrammarMatcher matcher_; + std::optional with_reasoning_; + std::string skip_; + std::string prefill_; + std::vector history_; // accepted tokens, to replay into with_reasoning_ + size_t words_; + bool failed_ = false; +}; + +/// \brief An xgrammar error as the client should see it +/// \note xgrammar prefixes its messages with "[hh:mm:ss] :: ", +/// which tells the client nothing and leaks the build machine's paths. +static std::string client_message(std::string message) { + if (!message.empty() && message.front() == '[') { + size_t close = message.find("] "); + size_t location = close == std::string::npos ? std::string::npos : message.find(": ", close + 2); + if (location != std::string::npos) { + message = message.substr(location + 2); + } + } + while (!message.empty() && (message.back() == '\n' || message.back() == ' ')) { + message.pop_back(); + } + return message; +} + +/// \brief Token strings in id order, as tokenizer.json stores them +std::vector encoded_vocab(const nlohmann::json& tokenizer) { + std::vector vocab; + const auto& model_vocab = tokenizer["model"]["vocab"]; + if (model_vocab.is_object()) { + for (const auto& [token, id] : model_vocab.items()) { + size_t i = id.get(); + if (i >= vocab.size()) vocab.resize(i + 1); + vocab[i] = token; + } + } else if (model_vocab.is_array()) { + // unigram: [[piece, score], ...] in id order + for (const auto& entry : model_vocab) { + vocab.push_back(entry[0].get()); + } + } + for (const auto& added : tokenizer.value("added_tokens", nlohmann::json::array())) { + size_t i = added["id"].get(); + if (i >= vocab.size()) vocab.resize(i + 1); + vocab[i] = added["content"].get(); + } + return vocab; +} + +} // namespace + +json chat_structural_tag(const chat_format_spec& spec, const chat_constraint_request& request) { + const std::vector tools = request_tools(request); + // auto only constrains a format that has a call syntax; required and forced + // calls on one that has none are an error below + const bool tool_grammar = request.tool_choice == chat_tool_choice::REQUIRED || + request.tool_choice == chat_tool_choice::FUNCTION || + (request.tool_choice == chat_tool_choice::AUTO && !tools.empty() && + !has_response_format(request) && + (spec.has_tools() || spec.kind == chat_format_kind::HARMONY)); + if (!tool_grammar && !has_response_format(request)) { + return nullptr; + } + if (tool_grammar && tools.empty()) { + throw std::invalid_argument("tool_choice asks for a tool call but no usable tool is defined"); + } + + json format; + if (spec.kind == chat_format_kind::HARMONY) { + format = harmony_format(request, tools); + } else if (tool_grammar) { + if (!spec.has_tools()) { + throw std::invalid_argument(std::string("this model's chat format (") + chat_format_name(spec.kind) + + ") has no tool call syntax to constrain"); + } + // a forced function is exactly one call + const bool parallel = request.parallel_tool_calls && request.tool_choice != chat_tool_choice::FUNCTION; + tool_syntax syntax = tool_tags(spec, tools, parallel); + json suffix; + if (request.tool_choice == chat_tool_choice::FUNCTION) { + suffix = sequence({whitespace(), syntax.tags.front()}); + } else { + std::vector excludes; + if (spec.has_reasoning()) { + excludes = {spec.reasoning_start, spec.reasoning_end}; + } + suffix = {{"type", "triggered_tags"}, {"triggers", {syntax.trigger}}, {"tags", syntax.tags}, + {"stop_after_first", !request.parallel_tool_calls}}; + if (!excludes.empty()) suffix["excludes"] = excludes; + if (request.tool_choice == chat_tool_choice::REQUIRED) suffix["at_least_one"] = true; + } + json prefix = reasoning_prefix(spec, request.allow_reasoning); + format = prefix.is_null() ? suffix : sequence({prefix, suffix}); + } else { + json suffix = sequence({whitespace(), schema_content(response_schema(request))}); + json prefix = reasoning_prefix(spec, request.allow_reasoning); + format = prefix.is_null() ? suffix : sequence({prefix, suffix}); + } + return {{"type", "structural_tag"}, {"format", format}}; +} + +struct chat_grammar_engine::impl { + std::vector vocab; + size_t vocab_size = 0; + std::unique_ptr info; + std::unique_ptr compiler; +}; + +chat_grammar_engine::chat_grammar_engine(const chat_vocab& vocab, const chat_format_spec& spec) + : impl_(std::make_unique()) { + std::ifstream file(vocab.tokenizer_json, std::ios::binary); + if (!file) { + throw std::runtime_error("cannot open " + vocab.tokenizer_json + " for the grammar engine"); + } + const std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + impl_->vocab = encoded_vocab(nlohmann::json::parse(content)); + impl_->vocab_size = std::max(vocab.size, impl_->vocab.size()); + + std::vector stops; + const auto& markers = text_markers(spec); + for (int id : vocab.stop_tokens) { + if (id < 0 || static_cast(id) >= impl_->vocab.size() || !markers.count(impl_->vocab[id])) { + stops.push_back(id); + } + } + + const json metadata = json::parse(xgrammar::TokenizerInfo::DetectMetadataFromHF(content)); + impl_->info = std::make_unique( + impl_->vocab, static_cast(metadata.value("vocab_type", 0)), + static_cast(impl_->vocab_size), stops, + metadata.value("add_prefix_space", false)); + impl_->compiler = std::make_unique(*impl_->info); +} + +chat_grammar_engine::~chat_grammar_engine() = default; + +std::shared_ptr chat_grammar_engine::make(const chat_format_spec& spec, + const chat_constraint_request& request) { + const json structural_tag = chat_structural_tag(spec, request); + if (structural_tag.is_null()) { + return nullptr; + } + auto compile = [this](const json& tag) { + try { + return impl_->compiler->CompileStructuralTag(tag.dump()); + } catch (const std::exception& e) { + throw std::invalid_argument("cannot build the decoding grammar: " + client_message(e.what())); + } + }; + xgrammar::CompiledGrammar grammar = compile(structural_tag); + + // thinking turned off keeps reasoning out of the grammar, so no spelling of the + // opener can be sampled; a thinking-only model is still made to open it + std::optional with_reasoning; + if (!request.allow_reasoning && spec.has_reasoning() && !spec.reasoning_forced) { + chat_constraint_request reasoning_request = request; + reasoning_request.allow_reasoning = true; + with_reasoning = compile(chat_structural_tag(spec, reasoning_request)); + } + try { + return std::make_shared(grammar, with_reasoning, prefill_skip(spec), impl_->vocab_size); + } catch (const std::exception& e) { + throw std::invalid_argument("cannot start the decoding grammar: " + client_message(e.what())); + } +} + +bool chat_grammar_engine::accepts(const chat_format_spec& spec, const chat_constraint_request& request, + const std::string& prefill, const std::string& text, std::string& error) { + const json structural_tag = chat_structural_tag(spec, request); + if (structural_tag.is_null()) { + return true; + } + xgrammar::GrammarMatcher matcher(impl_->compiler->CompileStructuralTag(structural_tag.dump())); + std::string full = prefill; + const std::string skip = prefill_skip(spec); + if (!skip.empty() && full.compare(0, skip.size(), skip) == 0) { + full = full.substr(skip.size()); + } + full += text; + // feed in small pieces so a rejection can be located + for (size_t pos = 0; pos < full.size();) { + size_t n = std::min(16, full.size() - pos); + // do not split a UTF-8 sequence + while (pos + n < full.size() && (static_cast(full[pos + n]) & 0xC0) == 0x80) n++; + if (!matcher.AcceptString(full.substr(pos, n))) { + error = "rejected at byte " + std::to_string(pos) + ": ..." + + full.substr(pos >= 40 ? pos - 40 : 0, std::min(pos, 40)) + "[" + full.substr(pos, n) + "]"; + return false; + } + pos += n; + } + return true; +} diff --git a/src/common/chat/chat_msg.cpp b/src/common/chat/chat_msg.cpp new file mode 100644 index 000000000..1167744f8 --- /dev/null +++ b/src/common/chat/chat_msg.cpp @@ -0,0 +1,54 @@ +/// \file chat_msg.cpp +/// \brief Delta computation between two parses of the same assistant turn + +#include "chat/chat_msg.hpp" +#include "utils/debug_utils.hpp" + +namespace { + +/// \brief The part of cur that extends prev, or nothing if it does not extend it +bool suffix_after(const std::string& prev, const std::string& cur, const char* field, std::string& out) { + if (cur.size() < prev.size() || cur.compare(0, prev.size(), prev) != 0) { + header_print("WARNING", "chat parser rewrote already streamed " << field << ", dropping the change"); + return false; + } + out = cur.substr(prev.size()); + return true; +} + +} // namespace + +std::vector chat_compute_deltas(const chat_msg& prev, const chat_msg& cur) { + std::vector deltas; + + std::string text; + if (suffix_after(prev.reasoning, cur.reasoning, "reasoning", text) && !text.empty()) { + chat_msg_delta d; + d.reasoning = std::move(text); + deltas.push_back(std::move(d)); + } + if (suffix_after(prev.content, cur.content, "content", text) && !text.empty()) { + chat_msg_delta d; + d.content = std::move(text); + deltas.push_back(std::move(d)); + } + + for (size_t i = 0; i < cur.tool_calls.size(); i++) { + const chat_tool_call& call = cur.tool_calls[i]; + chat_msg_delta d; + d.tool_index = static_cast(i); + if (i >= prev.tool_calls.size()) { + // first sight of this call: id and name go out with whatever arguments exist + d.tool_id = call.id; + d.tool_name = call.name; + d.tool_arguments = call.arguments; + deltas.push_back(std::move(d)); + continue; + } + if (suffix_after(prev.tool_calls[i].arguments, call.arguments, "tool arguments", text) && !text.empty()) { + d.tool_arguments = std::move(text); + deltas.push_back(std::move(d)); + } + } + return deltas; +} diff --git a/src/common/chat/chat_parser.cpp b/src/common/chat/chat_parser.cpp new file mode 100644 index 000000000..6d664088f --- /dev/null +++ b/src/common/chat/chat_parser.cpp @@ -0,0 +1,639 @@ +/// \file chat_parser.cpp +/// \brief Output parser for every chat format + +#include "chat/chat_parser.hpp" + +#include +#include +#include + +#include "chat/chat_args.hpp" + +namespace { + +using json = nlohmann::ordered_json; + +constexpr size_t npos = std::string::npos; + +bool is_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +std::string trim(const std::string& s) { + size_t b = 0; + size_t e = s.size(); + while (b < e && is_ws(s[b])) b++; + while (e > b && is_ws(s[e - 1])) e--; + return s.substr(b, e - b); +} + +size_t skip_ws(const std::string& s, size_t pos) { + while (pos < s.size() && is_ws(s[pos])) pos++; + return pos; +} + +bool starts_with_at(const std::string& s, size_t pos, const std::string& prefix) { + return s.compare(pos, prefix.size(), prefix) == 0; +} + +/// \brief Drop a tail of s that could be the start of one of the markers +/// \note The tail is held back, not lost: the next parse sees more text and +/// either completes the marker or shows the tail as text. +std::string cut_partial_marker(const std::string& s, std::initializer_list markers) { + size_t cut = 0; + for (const std::string* marker : markers) { + if (marker->empty()) { + continue; + } + size_t max_len = std::min(s.size(), marker->size() - 1); + for (size_t len = max_len; len > cut; len--) { + if (s.compare(s.size() - len, len, *marker, 0, len) == 0) { + cut = len; + break; + } + } + } + return s.substr(0, s.size() - cut); +} + +/// \brief Index of the first marker found from pos, and which one it is +size_t find_first(const std::string& s, size_t pos, std::initializer_list markers, + const std::string** which) { + size_t best = npos; + for (const std::string* marker : markers) { + if (marker->empty()) { + continue; + } + size_t at = s.find(*marker, pos); + if (at < best) { + best = at; + *which = marker; + } + } + return best; +} + +/// \brief Collects the text parts of one kind and joins them +/// \note Each part is trimmed and empty parts are dropped. Trimming keeps the +/// prefix property: trailing whitespace only shows once text follows it. +struct text_parts { + std::vector parts; + + void add(const std::string& raw) { + parts.push_back(raw); + } + + std::string join() const { + std::string out; + for (const auto& raw : parts) { + std::string part = trim(raw); + if (part.empty()) { + continue; + } + if (!out.empty()) { + out += "\n\n"; + } + out += part; + } + return out; + } +}; + +/// \brief JSON schema types of tool parameters, from the request's tools +class param_types { +public: + explicit param_types(const json* tools) : tools_(tools) {} + + /// \return the declared type, or empty when the schema does not say + std::string of(const std::string& tool, const std::string& param) const { + if (tools_ == nullptr || !tools_->is_array()) { + return ""; + } + for (const auto& entry : *tools_) { + const json& fn = entry.contains("function") ? entry["function"] : entry; + if (!fn.is_object() || fn.value("name", "") != tool) { + continue; + } + if (!fn.contains("parameters") || !fn["parameters"].contains("properties")) { + return ""; + } + const json& props = fn["parameters"]["properties"]; + if (!props.is_object() || !props.contains(param)) { + return ""; + } + const json& type = props[param].contains("type") ? props[param]["type"] : json(); + if (type.is_string()) { + return type.get(); + } + if (type.is_array()) { + // ["string", "null"] and the like: the first non-null type decides + for (const auto& t : type) { + if (t.is_string() && t.get() != "null") { + return t.get(); + } + } + } + return ""; + } + return ""; + } + +private: + const json* tools_; +}; + +void add_call(std::vector& calls, const std::string& name, const std::string& arguments) { + chat_tool_call call; + call.name = name; + call.arguments = arguments; + calls.push_back(std::move(call)); +} + +// ---------------------------------------------------------------- hermes json + +/// \brief Decode a JSON string that may still be missing its end +/// \param s the text +/// \param quote the index of the opening quote +/// \return the decoded content seen so far +std::string decode_json_string_prefix(const std::string& s, size_t quote) { + size_t end = chat_json_value_end(s, quote); + try { + if (end != npos) { + return json::parse(s.substr(quote, end - quote)).get(); + } + // the relaxed reader holds back a half written escape, so what it + // produced so far is always a well formed JSON string body + const std::string head = "{\"a\":\""; + chat_relaxed_result r = chat_relaxed_args_to_json("a=" + s.substr(quote), 0, chat_relaxed_dialect::PYTHON, false); + if (r.json.size() <= head.size()) { + return ""; + } + return json::parse("\"" + r.json.substr(head.size()) + "\"").get(); + } catch (const json::exception&) { + return ""; + } +} + +/// \brief Parse one {"name": .., "arguments": ..} object starting at pos +/// \return index past the object, or npos if the text ends inside it +size_t parse_hermes_object(const std::string& s, size_t pos, std::vector& calls) { + std::string name; + bool have_name = false; + std::string arguments; + bool have_args = false; + size_t end = npos; + + size_t p = pos + 1; + while (true) { + p = skip_ws(s, p); + if (p >= s.size()) break; + if (s[p] == ',') { p++; continue; } + if (s[p] == '}') { end = p + 1; break; } + if (s[p] != '"') break; // not JSON we understand, stop here + size_t key_end = chat_json_value_end(s, p); + if (key_end == npos) break; + std::string key; + try { + key = json::parse(s.substr(p, key_end - p)).get(); + } catch (const json::exception&) { + break; + } + p = skip_ws(s, key_end); + if (p >= s.size() || s[p] != ':') break; + p = skip_ws(s, p + 1); + if (p >= s.size()) break; + size_t value_end = chat_json_value_end(s, p); + + if (key == "name") { + if (value_end == npos || s[p] != '"') break; + try { + name = json::parse(s.substr(p, value_end - p)).get(); + have_name = true; + } catch (const json::exception&) { + break; + } + } else if (key == "arguments" || key == "parameters") { + have_args = true; + if (s[p] == '"') { + // arguments sent as a JSON encoded string + arguments = decode_json_string_prefix(s, p); + } else { + arguments = s.substr(p, (value_end == npos ? s.size() : value_end) - p); + } + } + if (value_end == npos) break; + p = value_end; + } + + if (have_name) { + if (!have_args && end != npos) { + arguments = "{}"; + } + add_call(calls, name, arguments); + } + return end; +} + +void parse_hermes_block(const std::string& body, std::vector& calls) { + size_t p = skip_ws(body, 0); + if (p < body.size() && body[p] == '[') { + p++; + } + while (true) { + p = skip_ws(body, p); + while (p < body.size() && body[p] == ',') { + p = skip_ws(body, p + 1); + } + if (p >= body.size() || body[p] != '{') { + return; + } + p = parse_hermes_object(body, p, calls); + if (p == npos) { + return; + } + } +} + +// ---------------------------------------------------------------- qwen xml + +const std::string XML_FUNCTION_OPEN = " between begin and end +std::string parse_xml_arguments(const std::string& body, size_t begin, size_t end, bool function_closed, + const std::string& tool, const param_types& types) { + std::string out = "{"; + bool first = true; + size_t p = begin; + while (true) { + size_t open = body.find(XML_PARAM_OPEN, p); + if (open == npos || open >= end) { + break; + } + size_t name_begin = open + XML_PARAM_OPEN.size(); + size_t name_end = body.find('>', name_begin); + if (name_end == npos || name_end >= end) { + if (!function_closed) { + return out; + } + break; + } + std::string key = trim(body.substr(name_begin, name_end - name_begin)); + size_t value_begin = name_end + 1; + + size_t close = body.find(XML_PARAM_CLOSE, value_begin); + size_t next = body.find(XML_PARAM_OPEN, value_begin); + size_t value_end = std::min({close, next, end}); + bool value_closed = value_end < end || function_closed; + + std::string raw = body.substr(value_begin, value_end - value_begin); + if (!value_closed) { + raw = cut_partial_marker(raw, {&XML_PARAM_CLOSE, &XML_PARAM_OPEN, &XML_FUNCTION_CLOSE}); + } + std::string value = trim(raw); + std::string type = types.of(tool, key); + + if (!first) { + out += ','; + } + first = false; + out += '"'; + chat_json_escape_append(out, key); + out += "\":"; + if (!value_closed) { + if (type == "string") { + // strings stream as they come; anything else waits for its end + out += '"'; + chat_json_escape_append(out, value); + } + return out; + } + out += xml_value_to_json(value, type); + p = value_end == close ? close + XML_PARAM_CLOSE.size() : value_end; + } + if (function_closed) { + out += '}'; + } + return out; +} + +void parse_qwen_xml_block(const std::string& body, bool closed, const param_types& types, + std::vector& calls) { + size_t p = 0; + while (true) { + size_t open = body.find(XML_FUNCTION_OPEN, p); + if (open == npos) { + return; + } + size_t name_begin = open + XML_FUNCTION_OPEN.size(); + size_t name_end = body.find('>', name_begin); + if (name_end == npos) { + return; + } + std::string name = trim(body.substr(name_begin, name_end - name_begin)); + size_t function_end = body.find(XML_FUNCTION_CLOSE, name_end); + bool function_closed = function_end != npos || closed; + size_t args_end = function_end == npos ? body.size() : function_end; + add_call(calls, name, parse_xml_arguments(body, name_end + 1, args_end, function_closed, name, types)); + if (function_end == npos) { + return; + } + p = function_end + XML_FUNCTION_CLOSE.size(); + } +} + +// ---------------------------------------------------------------- lfm2 / gemma4 + +void parse_lfm2_block(const std::string& body, bool closed, std::vector& calls) { + size_t p = skip_ws(body, 0); + if (p < body.size() && body[p] == '[') { + p++; + } + while (true) { + p = skip_ws(body, p); + while (p < body.size() && body[p] == ',') { + p = skip_ws(body, p + 1); + } + if (p >= body.size() || body[p] == ']') { + return; + } + size_t paren = body.find('(', p); + if (paren == npos) { + std::string name = trim(body.substr(p)); + if (closed && !name.empty()) { + add_call(calls, name, "{}"); + } + return; + } + std::string name = trim(body.substr(p, paren - p)); + chat_relaxed_result r = chat_relaxed_args_to_json(body, paren + 1, chat_relaxed_dialect::PYTHON, closed); + add_call(calls, name, r.json); + if (!r.closed) { + return; + } + p = r.end; + } +} + +void parse_gemma4_block(const std::string& body, bool closed, std::vector& calls) { + static const std::string CALL_PREFIX = "call:"; + size_t p = 0; + while (true) { + p = skip_ws(body, p); + if (p >= body.size()) { + return; + } + if (starts_with_at(body, p, CALL_PREFIX)) { + p += CALL_PREFIX.size(); + } + size_t brace = body.find('{', p); + if (brace == npos) { + std::string name = trim(body.substr(p)); + if (closed && !name.empty()) { + add_call(calls, name, "{}"); + } + return; + } + std::string name = trim(body.substr(p, brace - p)); + chat_relaxed_result r = chat_relaxed_args_to_json(body, brace, chat_relaxed_dialect::GEMMA4, closed); + add_call(calls, name, r.json); + if (!r.closed) { + return; + } + p = r.end; + } +} + +void parse_tool_block(const chat_format_spec& spec, const std::string& body, bool closed, + const param_types& types, std::vector& calls) { + switch (spec.kind) { + case chat_format_kind::HERMES_JSON: + parse_hermes_block(body, calls); + break; + case chat_format_kind::QWEN_XML: + parse_qwen_xml_block(body, closed, types, calls); + break; + case chat_format_kind::LFM2_PYTHONIC: + parse_lfm2_block(body, closed, calls); + break; + case chat_format_kind::GEMMA4: + parse_gemma4_block(body, closed, calls); + break; + case chat_format_kind::CONTENT_ONLY: + case chat_format_kind::HARMONY: + break; + } +} + +// ---------------------------------------------------------------- tag based formats + +chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text, bool is_partial, + const chat_parse_options& options) { + std::string text = full_text; + if (!spec.stop_marker.empty()) { + size_t stop = text.find(spec.stop_marker); + if (stop != npos) { + // the turn is over at the marker, whatever the model wrote after it + text.resize(stop); + is_partial = false; + } + } + + const bool tools = options.parse_tool_calls && !spec.tool_start.empty(); + const std::string no_marker; + const std::string& reasoning_start = spec.has_reasoning() ? spec.reasoning_start : no_marker; + const std::string& tool_start = tools ? spec.tool_start : no_marker; + const std::string& stop_marker = spec.stop_marker; + const param_types types(options.tools); + + chat_msg msg; + text_parts reasoning; + text_parts content; + bool in_reasoning = false; + size_t pos = 0; + + while (pos <= text.size()) { + if (in_reasoning) { + // the model may open again what the prefill already opened + size_t first = skip_ws(text, pos); + if (starts_with_at(text, first, spec.reasoning_start)) { + pos = first + spec.reasoning_start.size(); + continue; + } + size_t rest_len = text.size() - first; + if (is_partial && rest_len > 0 && rest_len < spec.reasoning_start.size() && + spec.reasoning_start.compare(0, rest_len, text, first, rest_len) == 0) { + break; + } + size_t end = text.find(spec.reasoning_end, pos); + if (end == npos) { + std::string rest = text.substr(pos); + if (is_partial) { + rest = cut_partial_marker(rest, {&spec.reasoning_end, &stop_marker}); + } + reasoning.add(rest); + break; + } + reasoning.add(text.substr(pos, end - pos)); + pos = end + spec.reasoning_end.size(); + in_reasoning = false; + continue; + } + + const std::string* which = nullptr; + size_t at = find_first(text, pos, {&reasoning_start, &tool_start}, &which); + if (at == npos) { + std::string rest = text.substr(pos); + if (is_partial) { + rest = cut_partial_marker(rest, {&reasoning_start, &tool_start, &stop_marker}); + } + content.add(rest); + break; + } + content.add(text.substr(pos, at - pos)); + if (which == &reasoning_start) { + pos = at + reasoning_start.size(); + in_reasoning = true; + continue; + } + + size_t body_begin = at + tool_start.size(); + size_t body_end = text.find(spec.tool_end, body_begin); + if (body_end == npos) { + std::string body = text.substr(body_begin); + if (is_partial) { + body = cut_partial_marker(body, {&spec.tool_end, &stop_marker}); + } + // an unterminated block at the very end of a finished generation + // is parsed as far as it goes, like the stream saw it + parse_tool_block(spec, body, !is_partial, types, msg.tool_calls); + break; + } + parse_tool_block(spec, text.substr(body_begin, body_end - body_begin), true, types, msg.tool_calls); + pos = body_end + spec.tool_end.size(); + } + + msg.reasoning = reasoning.join(); + msg.content = content.join(); + return msg; +} + +// ---------------------------------------------------------------- harmony + +const std::string HARMONY_START = "<|start|>"; +const std::string HARMONY_CHANNEL = "<|channel|>"; +const std::string HARMONY_MESSAGE = "<|message|>"; +const std::string HARMONY_END = "<|end|>"; +const std::string HARMONY_CALL = "<|call|>"; +const std::string HARMONY_RETURN = "<|return|>"; +const std::string HARMONY_RECIPIENT = "to="; +const std::string HARMONY_FUNCTIONS = "functions."; + +struct harmony_header { + std::string channel; + std::string recipient; +}; + +harmony_header parse_harmony_header(const std::string& header) { + harmony_header h; + size_t channel = header.find(HARMONY_CHANNEL); + if (channel != npos) { + size_t b = channel + HARMONY_CHANNEL.size(); + size_t e = b; + while (e < header.size() && !is_ws(header[e]) && header[e] != '<') e++; + h.channel = header.substr(b, e - b); + } + // the recipient may sit in the role part or after the channel + size_t to = header.find(HARMONY_RECIPIENT); + if (to != npos) { + size_t b = to + HARMONY_RECIPIENT.size(); + size_t e = b; + while (e < header.size() && !is_ws(header[e]) && header[e] != '<') e++; + h.recipient = header.substr(b, e - b); + } + return h; +} + +chat_msg parse_harmony(const std::string& text, bool is_partial, const chat_parse_options& options) { + chat_msg msg; + text_parts reasoning; + text_parts content; + + // the prefill puts us right after a <|start|>, so the text opens with a header + size_t pos = 0; + while (pos < text.size()) { + size_t message = text.find(HARMONY_MESSAGE, pos); + if (message == npos) { + break; // header still forming + } + harmony_header header = parse_harmony_header(text.substr(pos, message - pos)); + size_t body_begin = message + HARMONY_MESSAGE.size(); + + const std::string* which = nullptr; + size_t body_end = find_first(text, body_begin, {&HARMONY_END, &HARMONY_CALL, &HARMONY_RETURN, &HARMONY_START}, &which); + bool body_closed = body_end != npos; + std::string body = text.substr(body_begin, (body_closed ? body_end : text.size()) - body_begin); + if (!body_closed && is_partial) { + body = cut_partial_marker(body, {&HARMONY_END, &HARMONY_CALL, &HARMONY_RETURN, &HARMONY_START}); + } + + if (header.recipient.compare(0, HARMONY_FUNCTIONS.size(), HARMONY_FUNCTIONS) == 0) { + if (options.parse_tool_calls) { + std::string args = trim(body); + if (args.empty() && (body_closed || !is_partial)) { + args = "{}"; + } + add_call(msg.tool_calls, header.recipient.substr(HARMONY_FUNCTIONS.size()), args); + } + } else if (header.channel == "analysis") { + reasoning.add(body); + } else { + // final, and commentary addressed to the user (preambles) + content.add(body); + } + + if (!body_closed) { + break; + } + pos = body_end + which->size(); + if (which != &HARMONY_START) { + // a closed message; the next one starts at its own <|start|> + size_t next = text.find(HARMONY_START, pos); + if (next == npos) { + break; + } + pos = next + HARMONY_START.size(); + } + } + + msg.reasoning = reasoning.join(); + msg.content = content.join(); + return msg; +} + +} // namespace + +chat_msg chat_parse(const chat_format_spec& spec, const std::string& text, bool is_partial, + const chat_parse_options& options) { + if (spec.kind == chat_format_kind::HARMONY) { + return parse_harmony(text, is_partial, options); + } + return parse_tagged(spec, text, is_partial, options); +} diff --git a/src/common/chat/chat_stream.cpp b/src/common/chat/chat_stream.cpp new file mode 100644 index 000000000..78606ac17 --- /dev/null +++ b/src/common/chat/chat_stream.cpp @@ -0,0 +1,37 @@ +/// \file chat_stream.cpp +/// \brief Per request streaming state + +#include "chat/chat_stream.hpp" + +#include + +std::string chat_new_tool_call_id() { + static const char ALPHABET[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + thread_local std::mt19937 gen{std::random_device{}()}; + std::uniform_int_distribution pick(0, sizeof(ALPHABET) - 2); + std::string id = "call_"; + for (int i = 0; i < 24; i++) { + id += ALPHABET[pick(gen)]; + } + return id; +} + +chat_stream::chat_stream(const chat_format_spec& spec, std::string prefill, chat_parse_options options) + : spec_(spec), options_(options), text_(std::move(prefill)) {} + +std::vector chat_stream::update(const std::string& text, bool is_final) { + text_ += text; + chat_msg next = chat_parse(spec_, text_, !is_final, options_); + + // ids are ours, not the model's: keep each call's id for the whole stream + for (size_t i = 0; i < next.tool_calls.size(); i++) { + if (i >= tool_ids_.size()) { + tool_ids_.push_back(chat_new_tool_call_id()); + } + next.tool_calls[i].id = tool_ids_[i]; + } + + std::vector deltas = chat_compute_deltas(msg_, next); + msg_ = std::move(next); + return deltas; +} diff --git a/src/include/chat/chat_args.hpp b/src/include/chat/chat_args.hpp new file mode 100644 index 000000000..0db10642a --- /dev/null +++ b/src/include/chat/chat_args.hpp @@ -0,0 +1,46 @@ +/// \file chat_args.hpp +/// \brief Incremental conversion of tool call arguments to JSON +/// \note Every converter here is prefix stable: converting a prefix of some +/// text yields a prefix of what converting the whole text yields. That is +/// what lets the stream send arguments piece by piece. Values whose JSON +/// form depends on text not seen yet (a bare number, true/false) are held +/// back until they end; strings are passed through as they arrive. +#pragma once + +#include +#include + +/// \brief Append s to out as JSON string content, without the quotes +void chat_json_escape_append(std::string& out, const std::string& s); + +/// \brief Find where the JSON value starting at pos ends +/// \return the index just past the value, or npos if the text ends first +size_t chat_json_value_end(const std::string& s, size_t pos); + +/// \brief Convert a bare scalar the way a model means it +/// \return JSON text: numbers, true/false/null (also Python's True/False/None) +/// as literals, anything else as a JSON string +std::string chat_scalar_to_json(const std::string& token); + +/// \brief Syntax of the relaxed, not quite JSON argument formats +enum class chat_relaxed_dialect { + GEMMA4, // {key:<|"|>text<|"|>,n:5,flag:true,list:[...]} + PYTHON, // key="text", n=5, flag=True, list=[...] (inside a call's parentheses) +}; + +/// \brief Result of converting relaxed arguments +struct chat_relaxed_result { + std::string json; // a prefix of the final JSON object, complete when closed is true + size_t end = 0; // index just past what was consumed + bool closed = false; // the argument list was properly terminated +}; + +/// \brief Convert relaxed arguments to a JSON object +/// \param s the text +/// \param pos for GEMMA4 the index of the opening '{'; for PYTHON the index just +/// past the call's '(' +/// \param dialect the syntax +/// \param input_done true when no more text will come; unterminated values and +/// containers are then closed so the result is valid JSON +chat_relaxed_result chat_relaxed_args_to_json(const std::string& s, size_t pos, + chat_relaxed_dialect dialect, bool input_done); diff --git a/src/include/chat/chat_constraint.hpp b/src/include/chat/chat_constraint.hpp new file mode 100644 index 000000000..879cb4d11 --- /dev/null +++ b/src/include/chat/chat_constraint.hpp @@ -0,0 +1,62 @@ +/// \file chat_constraint.hpp +/// \brief Build the decoding constraint a chat request asks for +/// \note The constraint comes from the same chat_format_spec as the output +/// parser, so whatever the constraint lets through, the parser reads. +#pragma once + +#include +#include +#include +#include + +#include + +#include "chat/chat_format.hpp" +#include "chat/decode_constraint.hpp" + +/// \brief OpenAI tool_choice +enum class chat_tool_choice { + AUTO, // the model decides + NONE, // no tool calls + REQUIRED, // at least one tool call + FUNCTION, // exactly the named function +}; + +/// \brief What the request asks to constrain +struct chat_constraint_request { + chat_tool_choice tool_choice = chat_tool_choice::AUTO; + std::string function_name; // for FUNCTION + const nlohmann::ordered_json* tools = nullptr; // the request's tools, may be null + bool parallel_tool_calls = true; + bool allow_reasoning = true; // false when the request turned thinking off + bool json_object = false; // response_format {"type": "json_object"} + nlohmann::ordered_json json_schema; // response_format json_schema's schema, null if none +}; + +/// \brief The model's vocabulary as the constraint builders need it +struct chat_vocab { + size_t size = 0; // width of the logits, may exceed the tokenizer's vocabulary + std::function(const std::string&)> encode; // without any BOS + std::vector stop_tokens; // end of turn ids + std::string tokenizer_json; // path of the HF tokenizer.json +}; + +class chat_grammar_engine; + +/// \brief Parse the OpenAI request fields into a constraint request +/// \param request the chat completion request body +/// \param tools the request's tools, must outlive the result +/// \param error set when a field is malformed +/// \return false on a malformed field +bool chat_constraint_request_from_openai(const nlohmann::ordered_json& request, + const nlohmann::ordered_json& tools, + chat_constraint_request& out, std::string& error); + +/// \brief Build the constraint for one request +/// \param grammar_engine returns the model's grammar engine, only called when a grammar is needed +/// \return null when nothing needs constraining +/// \throws std::invalid_argument when the request's grammar cannot be built +std::shared_ptr chat_make_constraint(const chat_format_spec& spec, + const chat_constraint_request& request, + const chat_vocab& vocab, + const std::function& grammar_engine); diff --git a/src/include/chat/chat_format.hpp b/src/include/chat/chat_format.hpp new file mode 100644 index 000000000..58af89a05 --- /dev/null +++ b/src/include/chat/chat_format.hpp @@ -0,0 +1,68 @@ +/// \file chat_format.hpp +/// \brief How a model's chat template lays out reasoning and tool calls +/// \note Detected once per model from the template source and the tokenizer's +/// special tokens; the output parser and the decoding constraints are both +/// derived from it, so they can never disagree about the syntax. +#pragma once + +#include +#include +#include + +/// \brief Tool call syntax families +enum class chat_format_kind { + CONTENT_ONLY, // no tool calls, optionally reasoning + HERMES_JSON, // {"name": .., "arguments": {..}} (Qwen3, Qwen2.5, Nanbeige) + QWEN_XML, // v (Qwen3.5+) + LFM2_PYTHONIC, // <|tool_call_start|>[f(a=1, b="x")]<|tool_call_end|> (LFM2) + GEMMA4, // <|tool_call>call:f{a:<|"|>x<|"|>} (Gemma4) + HARMONY, // <|channel|>analysis / commentary to=functions.f / final (gpt-oss) +}; + +/// \brief Everything the parser needs to know about one model's output syntax +struct chat_format_spec { + chat_format_kind kind = chat_format_kind::CONTENT_ONLY; + std::string reasoning_start; // empty when the model has no reasoning block + std::string reasoning_end; + std::string tool_start; // opens one tool call block, empty for CONTENT_ONLY and HARMONY + std::string tool_end; + std::string stop_marker; // text after it is never part of the turn, e.g. Gemma4's <|tool_response> + bool reasoning_forced = false; // the model always opens its turn by reasoning, tag or not (DeepSeek-R1) + std::vector extra_stops; // markers that end the turn but are not in the model's eos list + + bool has_reasoning() const { + return !reasoning_end.empty(); + } + bool has_tools() const { + return kind != chat_format_kind::CONTENT_ONLY; + } +}; + +/// \brief Stable name of a format kind, as used in logs and model_list.json +const char* chat_format_name(chat_format_kind kind); + +/// \brief Look a kind up by its name +/// \return false when the name is unknown +bool chat_format_from_name(const std::string& name, chat_format_kind& kind); + +/// \brief Build the spec of a known kind with its standard markers +/// \param kind the format kind +/// \param with_reasoning whether to recognise blocks (tag based kinds only) +chat_format_spec chat_format_make(chat_format_kind kind, bool with_reasoning); + +/// \brief Detect the format from the chat template and the tokenizer +/// \param template_src the jinja chat template source +/// \param is_token true if the string is a single token in the model's vocabulary +/// \return the detected spec, CONTENT_ONLY when nothing matches +/// \note The tokenizer is consulted because some templates never render the +/// markers the model emits, e.g. LFM2.5-Thinking prints tool calls as plain +/// content and the R1 distills never mention . +chat_format_spec chat_format_detect(const std::string& template_src, + const std::function& is_token); + +/// \brief The part of the prompt the parser has to see before the generated text +/// \param spec the model's format +/// \param prompt_tail the decoded end of the prompt as fed to the model +/// \return text to prepend to the output before parsing, e.g. "" when the +/// template already opened the reasoning block +std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail); diff --git a/src/include/chat/chat_grammar.hpp b/src/include/chat/chat_grammar.hpp new file mode 100644 index 000000000..ff06093f1 --- /dev/null +++ b/src/include/chat/chat_grammar.hpp @@ -0,0 +1,45 @@ +/// \file chat_grammar.hpp +/// \brief Grammar constraints for tool calls and response_format, on xgrammar +/// \note The grammars are xgrammar structural tags built from the same +/// chat_format_spec the output parser uses. For Hermes JSON, Qwen XML and +/// Harmony they follow xgrammar's own builtin model styles (qwen_3, +/// qwen_3_5, harmony); Gemma4 and LFM2 have hand written argument grammars. +#pragma once + +#include +#include + +#include + +#include "chat/chat_constraint.hpp" + +/// \brief The structural tag a request needs +/// \return the structural tag JSON, or null when the request needs no grammar +/// \throws std::invalid_argument when the request cannot be expressed for this format +nlohmann::ordered_json chat_structural_tag(const chat_format_spec& spec, const chat_constraint_request& request); + +/// \brief Per model grammar compiler; build once, it indexes the whole vocabulary +class chat_grammar_engine { +public: + /// \param vocab the model's vocabulary; tokenizer_json must point at its tokenizer.json + /// \param spec the model's format: markers its grammars match as text are kept + /// out of the stop tokens, whatever the eos list says + chat_grammar_engine(const chat_vocab& vocab, const chat_format_spec& spec); + ~chat_grammar_engine(); + + /// \brief Compile the request's grammar into a fresh constraint + /// \return null when the request needs no grammar + /// \throws std::invalid_argument when the grammar cannot be built, e.g. a bad schema + std::shared_ptr make(const chat_format_spec& spec, const chat_constraint_request& request); + + /// \brief Whether the request's grammar accepts a whole output, for tests and diagnostics + /// \param prefill the prefill as the constraint would get it in start() + /// \param text the generated text + /// \param error set to where it failed + bool accepts(const chat_format_spec& spec, const chat_constraint_request& request, + const std::string& prefill, const std::string& text, std::string& error); + +private: + struct impl; + std::unique_ptr impl_; +}; diff --git a/src/include/chat/chat_msg.hpp b/src/include/chat/chat_msg.hpp new file mode 100644 index 000000000..7d7eb6bcc --- /dev/null +++ b/src/include/chat/chat_msg.hpp @@ -0,0 +1,60 @@ +/// \file chat_msg.hpp +/// \brief Parsed assistant message and the deltas used to stream it +/// \note A message is rebuilt from the whole generated text on every update; +/// the parser guarantees each field only ever grows, so the difference +/// between two consecutive messages is always a plain suffix. +#pragma once + +#include +#include + +/// \brief One function call requested by the model +struct chat_tool_call { + std::string id; + std::string name; + std::string arguments; // JSON text, possibly incomplete while streaming + + bool operator==(const chat_tool_call& other) const { + return id == other.id && name == other.name && arguments == other.arguments; + } +}; + +/// \brief The assistant turn split into its OpenAI parts +struct chat_msg { + std::string reasoning; + std::string content; + std::vector tool_calls; + + bool empty() const { + return reasoning.empty() && content.empty() && tool_calls.empty(); + } + + bool operator==(const chat_msg& other) const { + return reasoning == other.reasoning && content == other.content && tool_calls == other.tool_calls; + } +}; + +/// \brief What changed between two consecutive parses of the same stream +/// \note At most one of the three groups is set: reasoning text, content text, +/// or one tool call. The tool call carries id and name only in its first +/// delta, as OpenAI clients expect. +struct chat_msg_delta { + std::string reasoning; + std::string content; + int tool_index = -1; + std::string tool_id; + std::string tool_name; + std::string tool_arguments; + + bool is_tool_call() const { + return tool_index >= 0; + } +}; + +/// \brief Compute the deltas that turn prev into cur +/// \param prev the message already sent to the client +/// \param cur the message parsed from the longer text +/// \return the deltas in emission order: reasoning, content, then tool calls +/// \note A field of cur that does not extend the same field of prev is a parser +/// bug; it is reported once and the field is skipped rather than resent. +std::vector chat_compute_deltas(const chat_msg& prev, const chat_msg& cur); diff --git a/src/include/chat/chat_parser.hpp b/src/include/chat/chat_parser.hpp new file mode 100644 index 000000000..c7d587dcd --- /dev/null +++ b/src/include/chat/chat_parser.hpp @@ -0,0 +1,38 @@ +/// \file chat_parser.hpp +/// \brief Turn a model's raw output into reasoning, content and tool calls +/// \note One parser serves both the streaming and the non-streaming path. The +/// stream re-parses the whole text on every update and sends the +/// difference, which is safe because a partial parse only ever yields a +/// prefix of what the full parse yields: +/// - a marker that may still be forming at the end of the text is held +/// back instead of being shown as text, +/// - reasoning and content are trimmed, so trailing whitespace waits +/// until something follows it, +/// - a tool call appears once its name is complete, its arguments grow +/// as JSON text (see chat_args.hpp). +#pragma once + +#include + +#include + +#include "chat/chat_format.hpp" +#include "chat/chat_msg.hpp" + +/// \brief Per request parsing options +struct chat_parse_options { + /// \brief the request's tools, used to type the arguments of formats that + /// do not say whether a value is a string; may be null + const nlohmann::ordered_json* tools = nullptr; + /// \brief extract tool calls; when false the tool markup stays in the content + bool parse_tool_calls = true; +}; + +/// \brief Parse the assistant output +/// \param spec the model's format +/// \param text the prefill (see chat_format_prefill) followed by the generated text +/// \param is_partial true while the generation is still running +/// \param options per request options +/// \return the parsed message; tool call ids are left empty for the caller +chat_msg chat_parse(const chat_format_spec& spec, const std::string& text, bool is_partial, + const chat_parse_options& options); diff --git a/src/include/chat/chat_stream.hpp b/src/include/chat/chat_stream.hpp new file mode 100644 index 000000000..152e49318 --- /dev/null +++ b/src/include/chat/chat_stream.hpp @@ -0,0 +1,43 @@ +/// \file chat_stream.hpp +/// \brief Per request streaming state: accumulate, re-parse, send the difference +#pragma once + +#include +#include + +#include "chat/chat_format.hpp" +#include "chat/chat_msg.hpp" +#include "chat/chat_parser.hpp" + +/// \brief Tracks one streamed assistant turn +/// \note The whole text is parsed again on every update. That keeps one code +/// path for streaming and non-streaming, and costs a scan of a few KB per +/// token, which is noise next to a decode step. +class chat_stream { +public: + /// \param spec the model's format + /// \param prefill the prompt part the parser must see first (chat_format_prefill) + /// \param options per request options; the tools pointer must outlive the stream + chat_stream(const chat_format_spec& spec, std::string prefill, chat_parse_options options); + + /// \brief Feed newly generated text + /// \param text the new text, may be empty + /// \param is_final true once generation has stopped + /// \return what changed since the previous update + std::vector update(const std::string& text, bool is_final); + + /// \brief The message as parsed so far, tool call ids filled in + const chat_msg& message() const { + return msg_; + } + +private: + chat_format_spec spec_; + chat_parse_options options_; + std::string text_; + chat_msg msg_; + std::vector tool_ids_; +}; + +/// \brief A fresh tool call id in the OpenAI style, "call_" and 24 random characters +std::string chat_new_tool_call_id(); diff --git a/src/include/chat/decode_constraint.hpp b/src/include/chat/decode_constraint.hpp new file mode 100644 index 000000000..5537e44fe --- /dev/null +++ b/src/include/chat/decode_constraint.hpp @@ -0,0 +1,66 @@ +/// \file decode_constraint.hpp +/// \brief Per request restrictions on which token may be sampled next +/// \note The decode loop asks for a mask before each sample and reports the +/// sampled token back. Masks are bitsets over the vocabulary so this +/// header needs nothing from the inference engine. +#pragma once + +#include +#include +#include +#include + +/// \brief One request's constraint, advanced token by token +class decode_constraint { +public: + virtual ~decode_constraint() = default; + + /// \brief Called once the prompt is in, before the first token is sampled + /// \param prefill the part of the turn the template already wrote, see chat_format_prefill + virtual void start(const std::string& /*prefill*/) {} + + /// \brief Allowed tokens for the next sample + /// \param mask resized to cover the vocabulary; bit i of word i/32 set means + /// token i may be sampled + /// \return false when every token is allowed, the mask is then left as is + virtual bool fill_mask(std::vector& mask) = 0; + + /// \brief Advance past the token that was sampled + virtual void accept(int token) = 0; + + /// \brief Advance past a token the model was made to emit, not sampled + /// \note A forced token may open what sampling was not allowed to, e.g. the + /// reasoning block of a thinking-only model whose request turned thinking off. + virtual void accept_forced(int token) { + accept(token); + } + + /// \brief Undo the last n accepted tokens + virtual void rollback(int /*n*/) {} +}; + +/// \brief Bans a fixed set of tokens, e.g. the tool call opener for tool_choice=none +class token_ban_constraint : public decode_constraint { +public: + token_ban_constraint(size_t vocab_size, std::vector banned) + : vocab_size_(vocab_size), banned_(std::move(banned)) {} + + bool fill_mask(std::vector& mask) override { + if (banned_.empty()) { + return false; + } + mask.assign((vocab_size_ + 31) / 32, 0xFFFFFFFFu); + for (int token : banned_) { + if (token >= 0 && static_cast(token) < vocab_size_) { + mask[token / 32] &= ~(1u << (token % 32)); + } + } + return true; + } + + void accept(int) override {} + +private: + size_t vocab_size_; + std::vector banned_; +}; From ed7473bbb43b2b8c70f3d96474668a0aa632107c Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 7 Oct 2026 14:48:37 -0600 Subject: [PATCH 04/18] Let the sampler take an allowed-token bitmask Sampler::sample and sample_greedy take an optional bitmask (bit i of word i/32 set when token i may be sampled). Tokens outside it are set to -inf in the sampler's own fp32 copy of the logits; the engine's buffer is only read. Without a mask the sampling path is unchanged. Co-Authored-By: Claude --- src/common/modules/sampler.cpp | 37 +++++++++++++++++++++++++++------ src/include/modules/sampler.hpp | 11 +++++++--- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/common/modules/sampler.cpp b/src/common/modules/sampler.cpp index 50b778ef1..fb960b744 100644 --- a/src/common/modules/sampler.cpp +++ b/src/common/modules/sampler.cpp @@ -352,7 +352,7 @@ void Sampler::ring_buffer_update_sparse(int sampled_index) { /// \brief Sample the token /// \param x the input buffer /// \return the sampled token -int Sampler::sample_greedy(buffer& x) { +int Sampler::sample_greedy(buffer& x, const std::vector* allowed) { const bool penalties_active = (this->repeat_last_n != 0) && (this->rep_penalty != 1.0f || this->freq_penalty != 0.0f || this->pre_penalty != 0.0f); @@ -360,9 +360,10 @@ int Sampler::sample_greedy(buffer& x) { float best = -std::numeric_limits::infinity(); int best_id = 0; - if (penalties_active) { + if (penalties_active || allowed != nullptr) { // penalties reorder the logits, so they have to land before the argmax. - // that needs the fp32 copy, but nothing past it does. + // that needs the fp32 copy, but nothing past it does. A mask needs the + // copy too: x is the engine's buffer and is never written. #if USEAVX2 const int simd_width = 8; int i = 0; @@ -379,7 +380,10 @@ int Sampler::sample_greedy(buffer& x) { this->logits[i] = x[i]; } #endif - sampler_penalty_apply_sparse(); + if (penalties_active) { + sampler_penalty_apply_sparse(); + } + sampler_allowed_apply(allowed); for (int i = 0; i < in_features; i++) { if (this->logits[i] > best) { best = this->logits[i]; @@ -442,10 +446,30 @@ int Sampler::sample_greedy(buffer& x) { return best_id; } -int Sampler::sample(buffer& x) { +void Sampler::sampler_allowed_apply(const std::vector* allowed) { + if (allowed == nullptr) { + return; + } + const float banned = -std::numeric_limits::infinity(); + const size_t words = allowed->size(); + for (int word = 0; word * 32 < in_features; word++) { + const uint32_t bits = static_cast(word) < words ? (*allowed)[word] : 0u; + if (bits == 0xFFFFFFFFu) { + continue; + } + const int end = std::min(in_features, (word + 1) * 32); + for (int i = word * 32; i < end; i++) { + if (!((bits >> (i % 32)) & 1u)) { + this->logits[i] = banned; + } + } + } +} + +int Sampler::sample(buffer& x, const std::vector* allowed) { // PRNG is seeded once at construction, no per-call seeding needed if (this->top_k == 1) { - return sample_greedy(x); + return sample_greedy(x, allowed); } // COPY FROM `x` → `this->logits[]` @@ -469,6 +493,7 @@ int Sampler::sample(buffer& x) { #endif sampler_penalty_apply_sparse(); + sampler_allowed_apply(allowed); sampler_topk_apply(this->top_k); if (this->use_optimized_sampling) { softmax_with_topp_minp(this->top_p, this->min_p); diff --git a/src/include/modules/sampler.hpp b/src/include/modules/sampler.hpp index 0d43d235a..df7997345 100644 --- a/src/include/modules/sampler.hpp +++ b/src/include/modules/sampler.hpp @@ -102,7 +102,7 @@ class Sampler{ /// Going through the general path would allocate and partial_sort a /// vocab-sized array of triples and run three full-vocab softmaxes for /// a result that never depends on any of it. - int sample_greedy(buffer& x); + int sample_greedy(buffer& x, const std::vector* allowed = nullptr); void sampler_topk_apply(int k); void sampler_topp_apply(float p); @@ -113,7 +113,12 @@ class Sampler{ void ring_buffer_update_sparse(int sampled_index); /// \brief Sample the token - /// \param x the input buffer + /// \param x the input buffer, only read + /// \param allowed optional bitmask, bit i of word i/32 set when token i may be + /// sampled; applied to the sampler's own copy of the logits /// \return the sampled token - int sample(buffer& x); + int sample(buffer& x, const std::vector* allowed = nullptr); + + /// \brief Push every token the bitmask does not allow to -inf in this->logits + void sampler_allowed_apply(const std::vector* allowed); }; \ No newline at end of file From ef7009f546567983a75c6c580028a377e508651d Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 7 Oct 2026 14:49:52 -0600 Subject: [PATCH 05/18] Parse chat output with the shared library and enforce tool_choice Wire src/common/chat into AutoModel, the models and /v1/chat/completions, and delete the per-model parsers it replaces (about 4,300 lines). Parsing: - AutoModel detects the chat format when it sets up the tokenizer (also for Phi-4 and Qwen3.5-Omni) and keeps the last 32 prompt tokens so the parser knows what the template already opened. Gemma4 now stops at <|tool_response> and GPT-OSS at <|call|>. - Streaming goes through chat_stream: OpenAI deltas for reasoning_content, content and tool_calls, with ids made once per call. Non-streamed replies use the same parser. Tool markup only counts as a call when the request has tools; content is null for a reply that is only tool calls. - Removed: parse_stream_content / parse_nstream_content in every model, parser state on AutoModel, openai_policy.hpp, the duplicated Gemma4 argument parsers and GPT-OSS's unused tool state machine. Constrained decoding: - tool_choice (auto, none, required, a named function), parallel_tool_calls and response_format (json_object, json_schema) become a decode_constraint on chat_meta_info_t. Malformed values, unknown functions, required without tools and unbuildable schemas return 400. - Every model samples through AutoModel::_sample, which applies the constraint's mask. _begin_turn feeds the prefill, _force_token reports the think tokens some generate()s force, and _skip_insert_sample drops the sample insert() made when generate() does not use it. - Qwen3.8-MTP turns speculative decoding off under a constraint. - The per-model tool start token ids are gone; tool_choice none bans the format's opener. The raw text models write to the stream is unchanged, so the CLI, /v1/completions and the Ollama endpoints behave as before. Co-Authored-By: Claude --- src/common/AutoModel/automodel.cpp | 286 ++++---- src/common/AutoModel/modeling_gemma4_12b.cpp | 626 +---------------- src/common/AutoModel/modeling_gemma4e.cpp | 639 +----------------- src/common/AutoModel/modeling_gpt_oss.cpp | 402 +---------- src/common/AutoModel/modeling_lfm2.cpp | 416 +----------- src/common/AutoModel/modeling_llama3.cpp | 77 +-- src/common/AutoModel/modeling_nanbeige.cpp | 199 +----- src/common/AutoModel/modeling_phi4.cpp | 1 + src/common/AutoModel/modeling_qwen3.cpp | 282 +------- .../AutoModel/modeling_qwen3_5_omni.cpp | 357 +--------- src/common/AutoModel/modeling_qwen3_5vl.cpp | 358 +--------- src/common/AutoModel/modeling_qwen3_6_moe.cpp | 358 +--------- src/common/AutoModel/modeling_qwen3_8mtp.cpp | 362 +--------- src/common/AutoModel/modeling_qwen3vl.cpp | 125 +--- src/include/AutoModel/automodel.hpp | 144 ++-- src/include/AutoModel/modeling_gemma4_12b.hpp | 12 - src/include/AutoModel/modeling_gemma4e.hpp | 8 - src/include/AutoModel/modeling_gpt_oss.hpp | 20 - src/include/AutoModel/modeling_lfm2.hpp | 3 - src/include/AutoModel/modeling_llama3.hpp | 2 - src/include/AutoModel/modeling_nanbeige.hpp | 2 - src/include/AutoModel/modeling_qwen3.hpp | 15 - .../AutoModel/modeling_qwen3_5_omni.hpp | 4 - src/include/AutoModel/modeling_qwen3_5vl.hpp | 11 - .../AutoModel/modeling_qwen3_6_moe.hpp | 11 - src/include/AutoModel/modeling_qwen3_8mtp.hpp | 5 +- src/include/AutoModel/modeling_qwen3vl.hpp | 8 - src/include/openai_policy.hpp | 164 ----- src/server/rest_handler.cpp | 91 ++- src/server/rest_handler.hpp | 5 +- src/server/streaming_ostream_openai.hpp | 182 +++-- 31 files changed, 379 insertions(+), 4796 deletions(-) delete mode 100644 src/include/openai_policy.hpp diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 404433d37..7b5f6b3bc 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -8,6 +8,9 @@ #include "AutoModel/automodel.hpp" #include "utils/npu_platform.hpp" +#include +#include + namespace { /// \brief the context length a model should actually be loaded with @@ -169,9 +172,41 @@ nlohmann::json AutoModel::_shared_setup_tokenizer(std::string model_path) { } this->user_system_prompt = ""; this->extra_context["user_system_prompt"] = this->user_system_prompt; + this->_detect_chat_format(tokenizer_config["chat_template"].get()); return tokenizer_config; } +void AutoModel::_detect_chat_format(const std::string& template_src) { + auto is_token = [this](const std::string& text) { + std::vector ids = this->tokenizer->encode(text); + if (this->has_bos_token && !ids.empty() && ids.front() == this->bos_token_id) { + ids.erase(ids.begin()); + } + return ids.size() == 1; + }; + this->chat_format = chat_format_detect(template_src, is_token); + + // markers that hand the turn to a tool end generation like eos does + std::string stops; + for (const std::string& marker : this->chat_format.extra_stops) { + std::vector ids = this->tokenizer->encode(marker); + if (this->has_bos_token && !ids.empty() && ids.front() == this->bos_token_id) { + ids.erase(ids.begin()); + } + if (ids.size() != 1) { + header_print("WARNING", "Stop marker " << marker << " is not a single token, generation will not stop on it"); + continue; + } + if (!this->is_eos(ids[0])) { + this->eos_token_ids.push_back(ids[0]); + } + stops += " " + marker; + } + header_print("FLM", "Chat format: " << chat_format_name(this->chat_format.kind) + << (this->chat_format.has_reasoning() ? " with reasoning" : "") + << (stops.empty() ? "" : ", also stops at" + stops)); +} + void AutoModel::_shared_load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { if (this->is_model_loaded && this->model_path == model_path) { @@ -295,7 +330,6 @@ void AutoModel::_shared_after_inference_failure(bool poisoned) { try { this->lm_engine->clear_context(); } catch (...) {} } if (this->sampler) this->sampler->reset_penalties(); - this->reset_parser(); } void AutoModel::_shared_guard_poisoned() const { @@ -348,6 +382,8 @@ void AutoModel::_shared_validate_capacity(std::size_t rendered_tokens, bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run, std::optional requested_max_new_tokens) { this->_shared_validate_capacity(tokens.size(), requested_max_new_tokens); + this->_begin_turn(meta_info, tokens); + // print token history // header_print("DEBUG", "Current token history: "); // for (size_t i = 0; i < this->token_history.size(); i++) { @@ -411,8 +447,8 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); - this->_apply_tool_choice_mask(y, meta_info); - this->last_token = this->sampler->sample(y); + this->last_token = this->_sample(y, meta_info); + meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); return true; } @@ -429,10 +465,6 @@ buffer AutoModel::_chunked_insert(chat_meta_info_t& meta_info, std::vector if (max_prefill_len < 512) { if (is_cancelled()) { meta_info.stop_reason = CANCEL_DETECTED; - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; return y; } y = this->lm_engine->prefill(tokens, payload); @@ -452,10 +484,6 @@ buffer AutoModel::_chunked_insert(chat_meta_info_t& meta_info, std::vector header_print("FLM", "Prefill chunk " + std::to_string(i+1) + "/" + std::to_string(chunks) + " with " + std::to_string(chunk_tokens.size()) + " tokens"); if (is_cancelled()) { meta_info.stop_reason = CANCEL_DETECTED; - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } buffer chunk_y = this->lm_engine->prefill(chunk_tokens, (i == 0)? payload : nullptr); @@ -470,18 +498,106 @@ buffer AutoModel::_chunked_insert(chat_meta_info_t& meta_info, std::vector return y; } -void AutoModel::_apply_tool_choice_mask(buffer& y, const chat_meta_info_t& meta_info) { - if (meta_info.tool_choice != TOOL_CHOICE_NONE) { - return; +void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& prompt_tokens) { + // keep the end of the prompt as the model sees it: generation continues from + // there, so it tells the output parser whether a reasoning block is already open + constexpr size_t PROMPT_TAIL_TOKENS = 32; + meta_info.prompt_tail.clear(); + for (size_t i = prompt_tokens.size() > PROMPT_TAIL_TOKENS ? prompt_tokens.size() - PROMPT_TAIL_TOKENS : 0; + i < prompt_tokens.size(); i++) { + try { + meta_info.prompt_tail += this->tokenizer->run_time_decoder(prompt_tokens[i]); + } catch (const std::runtime_error&) { + // added tokens outside the byte map, e.g. DeepSeek's <｜Assistant｜>, + // are stored as plain text; the full decoder handles them + meta_info.prompt_tail += this->tokenizer->decode({prompt_tokens[i]}); + } } - const int tool_start_token_id = this->get_tool_start_token_id(); - if (tool_start_token_id < 0 || (size_t)tool_start_token_id >= y.size()) { - return; + if (meta_info.constraint) { + meta_info.constraint->start(chat_format_prefill(this->chat_format, meta_info.prompt_tail)); + } +} + +int AutoModel::_sample(buffer& logits, chat_meta_info_t& meta_info) { + decode_constraint* constraint = meta_info.constraint.get(); + const bool masked = constraint != nullptr && constraint->fill_mask(this->constraint_mask_); + // The mask goes to the sampler, which applies it to its own copy: the logits + // live in the engine's device-mapped buffer and are never written here. + int token = this->sampler->sample(logits, masked ? &this->constraint_mask_ : nullptr); + auto allowed = [this](size_t i) { + return i / 32 < this->constraint_mask_.size() && ((this->constraint_mask_[i / 32] >> (i % 32)) & 1u); + }; + if (masked && (token < 0 || !allowed(static_cast(token)))) { + // Only happens when the logits themselves are broken (NaN everywhere + // makes the argmax land on 0): stay inside the grammar regardless. + int first = -1; + int best = -1; + float best_logit = -std::numeric_limits::infinity(); + for (size_t i = 0; i < logits.size(); i++) { + if (!allowed(i)) { + continue; + } + const float v = static_cast(logits[i]); + if (first < 0) first = static_cast(i); + if (v > best_logit) { // NaN never compares greater + best = static_cast(i); + best_logit = v; + } + } + if (first < 0) { + // nothing allowed at all: keep the sample, the constraint gives itself up on it + header_print("WARNING", "The decoding constraint allows no token here, keeping sampled token " << token); + } else { + header_print("WARNING", "Sampled token " << token << " is outside the decoding constraint, using " + << (best >= 0 ? best : first) << " instead"); + token = best >= 0 ? best : first; + } + } + if (constraint != nullptr) { + constraint->accept(token); + } + meta_info.insert_sample_pending = false; + return token; +} + +void AutoModel::_force_token(chat_meta_info_t& meta_info, int token) { + this->_skip_insert_sample(meta_info); + if (meta_info.constraint) { + meta_info.constraint->accept_forced(token); + } +} + +void AutoModel::_skip_insert_sample(chat_meta_info_t& meta_info) { + if (meta_info.constraint && meta_info.insert_sample_pending) { + meta_info.constraint->rollback(1); } - // A large negative logit rather than -inf: every sampling path either takes - // the argmax or runs this through softmax, and a finite value can never turn - // into the inf - inf that would poison the whole distribution. - y[tool_start_token_id] = bf16(-1e30f); + meta_info.insert_sample_pending = false; +} + +std::shared_ptr AutoModel::make_constraint(const chat_constraint_request& request) { + chat_vocab vocab; + vocab.size = static_cast(this->sampler->in_features); + vocab.encode = [this](const std::string& text) { + std::vector ids = this->tokenizer->encode(text); + if (this->has_bos_token && !ids.empty() && ids.front() == this->bos_token_id) { + ids.erase(ids.begin()); + } + return ids; + }; + vocab.stop_tokens = this->eos_token_ids; + vocab.tokenizer_json = (std::filesystem::path(this->model_path) / "tokenizer.json").string(); + + auto engine = [this, &vocab]() -> chat_grammar_engine& { + if (!this->grammar_engine_ || this->grammar_engine_path_ != vocab.tokenizer_json) { + auto start = time_utils::now(); + this->grammar_engine_ = std::make_unique(vocab, this->chat_format); + this->grammar_engine_path_ = vocab.tokenizer_json; + header_print("FLM", "Grammar engine ready in " + << time_utils::duration_ns(start, time_utils::now()).first / 1000000 << " ms"); + } + return *this->grammar_engine_; + }; + return chat_make_constraint(this->chat_format, request, vocab, engine); } std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { @@ -529,11 +645,6 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ } if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -541,8 +652,7 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -585,126 +695,6 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ return result; } -StreamResult AutoModel::_shared_think_tool_calling_pasrsed(const std::string content) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - - StreamResult result; - buffer_ += content; - - while (true) { - // check tool calling - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - // find the and output the content before it - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - // go to tool calling - is_in_tool_block_ = true; - tool_name_.clear(); - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - if (tool_end_pos != std::string::npos) { - tool_name_ += buffer_.substr(0, tool_end_pos); - buffer_ = buffer_.substr(tool_end_pos + MARKER_TOOL_END.length()); - is_in_tool_block_ = false; - - try { - auto j = nlohmann::json::parse(tool_name_); - - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - if (j.contains("name")) { - result.tool_name = j["name"].get(); - } - - if (j.contains("arguments")) { - if (j["arguments"].is_string()) { - result.tool_args_str = j["arguments"].get(); - } - else { - result.tool_args_str = j["arguments"].dump(); - } - } - - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - tool_name_ += buffer_; - buffer_.clear(); - result.type = StreamEventType::WAITING; - return result; - } - } - - // normal and reasoning - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} - /// \brief Clear the context /// \note The function will clear the context /// \note The function will reset the total tokens diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index 66178cf0a..ed2da6d97 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -10,396 +10,6 @@ #include "error_measure.hpp" #include -namespace { -std::string trim_gemma4_12b_tool_value(std::string value) { - size_t start = value.find_first_not_of(" \t\r\n"); - if (start == std::string::npos) { - return ""; - } - - size_t end = value.find_last_not_of(" \t\r\n"); - return value.substr(start, end - start + 1); -} - -// --- Gemma4 tool-args parser -------------------------------------------------- -// The model emits relaxed JSON for tool arguments: -// - keys are bare identifiers (e.g. `name:`) — sometimes also "..." quoted -// - string values are delimited by <|"|>...<|"|>, but the model frequently -// omits the opener and/or replaces the closer with a plain `"` -// - values can also be objects {..}, arrays [..], booleans, null, or numbers -// - inside a string value, raw `"`, raw newlines, and even the literal -// substring `<|"|>` can appear as content -// -// We rewrite the input into well-formed JSON via recursive-descent. -struct Gemma4_12BArgsParser { - const std::string& s; - size_t i = 0; - static constexpr size_t marker_len = 5; - static constexpr const char* quote_marker_lit = "<|\"|>"; - - explicit Gemma4_12BArgsParser(const std::string& src) : s(src) {} - - void skip_ws() { - while (i < s.size() && - std::isspace(static_cast(s[i]))) ++i; - } - void skip_ws_at(size_t& pos) const { - while (pos < s.size() && - std::isspace(static_cast(s[pos]))) ++pos; - } - bool match_marker(size_t pos) const { - return pos + marker_len <= s.size() && - s.compare(pos, marker_len, quote_marker_lit) == 0; - } - - static void json_escape_char(std::string& out, char c) { - switch (c) { - case '"': out.append("\\\""); break; - case '\\': out.append("\\\\"); break; - case '\n': out.append("\\n"); break; - case '\r': out.append("\\r"); break; - case '\t': out.append("\\t"); break; - case '\b': out.append("\\b"); break; - case '\f': out.append("\\f"); break; - default: - if (static_cast(c) < 0x20) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", - static_cast(c)); - out.append(buf); - } else { - out.push_back(c); - } - break; - } - } - - bool match_word(const char* w, size_t len) const { - if (i + len > s.size()) return false; - if (s.compare(i, len, w) != 0) return false; - if (i + len == s.size()) return true; - unsigned char nc = static_cast(s[i + len]); - return !(std::isalnum(nc) || nc == '_'); - } - - // Parse one JSON value. `terminators` is the set of chars that end a - // bare (undelimited) string at depth 0 (e.g. ",}" inside an object, - // ",]" inside an array, "" at the very top). - std::string parse_value(const std::string& terminators) { - skip_ws(); - if (i >= s.size()) return "null"; - char c = s[i]; - if (c == '{') return parse_object(); - if (c == '[') return parse_array(); - if (match_word("true", 4)) { i += 4; return "true"; } - if (match_word("false", 5)) { i += 5; return "false"; } - if (match_word("null", 4)) { i += 4; return "null"; } - if (c == '-' || (c >= '0' && c <= '9')) return parse_number(); - return parse_string(terminators); - } - - std::string parse_number() { - size_t start = i; - if (s[i] == '-') ++i; - while (i < s.size()) { - char c = s[i]; - if ((c >= '0' && c <= '9') || c == '.' || - c == 'e' || c == 'E' || c == '+' || c == '-') ++i; - else break; - } - return s.substr(start, i - start); - } - - std::string parse_key() { - skip_ws(); - std::string key; - if (i >= s.size()) return key; - if (match_marker(i)) { - i += marker_len; - while (i < s.size() && !match_marker(i)) key.push_back(s[i++]); - if (match_marker(i)) i += marker_len; - } else if (s[i] == '"') { - ++i; - while (i < s.size() && s[i] != '"') { - if (s[i] == '\\' && i + 1 < s.size()) { - key.push_back(s[i]); - key.push_back(s[i + 1]); - i += 2; - } else { - key.push_back(s[i++]); - } - } - if (i < s.size() && s[i] == '"') ++i; - } else { - while (i < s.size() && - (std::isalnum(static_cast(s[i])) || - s[i] == '_')) { - key.push_back(s[i++]); - } - } - return key; - } - - std::string parse_object() { - // assumes s[i] == '{' - ++i; - std::string out = "{"; - bool first = true; - while (i < s.size()) { - skip_ws(); - if (i >= s.size()) break; - if (s[i] == '}') { ++i; break; } - - std::string key = parse_key(); - if (key.empty()) { - // can't make progress; bail - if (i < s.size() && s[i] == '}') { ++i; } - break; - } - skip_ws(); - if (i < s.size() && s[i] == ':') ++i; - - std::string val = parse_value(",}"); - - if (!first) out.push_back(','); - first = false; - out.push_back('"'); - for (char kc : key) json_escape_char(out, kc); - out.append("\":"); - out.append(val); - - skip_ws(); - if (i < s.size() && s[i] == ',') ++i; - } - out.push_back('}'); - return out; - } - - std::string parse_array() { - // assumes s[i] == '[' - ++i; - std::string out = "["; - bool first = true; - while (i < s.size()) { - skip_ws(); - if (i >= s.size()) break; - if (s[i] == ']') { ++i; break; } - - std::string val = parse_value(",]"); - if (!first) out.push_back(','); - first = false; - out.append(val); - - skip_ws(); - if (i < s.size() && s[i] == ',') ++i; - } - out.push_back(']'); - return out; - } - - // Parse a string value. Accepts three forms: - // - <|"|>...<|"|> (or <|"|>...") -- marker opener, marker or " closer - // - "..." -- regular JSON string - // - bare text -- ends at depth-0 terminator - std::string parse_string(const std::string& terminators) { - enum Mode { MARKER, QUOTE, BARE }; - Mode mode = BARE; - if (match_marker(i)) { mode = MARKER; i += marker_len; } - else if (s[i] == '"') { mode = QUOTE; ++i; } - - auto is_terminator = [&](size_t pos) { - size_t k = pos; - skip_ws_at(k); - if (k >= s.size()) return true; - return terminators.find(s[k]) != std::string::npos; - }; - - // QUOTE-mode only: honor JSON-style backslash escapes verbatim. - auto consume_quote_escape = [&](std::string& out) { - // s[i] == '\\' - if (i + 1 >= s.size()) { - json_escape_char(out, s[i]); - ++i; - return; - } - char nc = s[i + 1]; - switch (nc) { - case '"': case '\\': case '/': - case 'b': case 'f': case 'n': case 'r': case 't': - out.push_back('\\'); - out.push_back(nc); - i += 2; - return; - case 'u': { - if (i + 5 < s.size() && - std::isxdigit(static_cast(s[i + 2])) && - std::isxdigit(static_cast(s[i + 3])) && - std::isxdigit(static_cast(s[i + 4])) && - std::isxdigit(static_cast(s[i + 5]))) { - out.append(s, i, 6); - i += 6; - return; - } - break; - } - default: break; - } - json_escape_char(out, s[i]); - ++i; - }; - - // For MARKER and BARE modes we first collect the raw content, then - // JSON-encode it. Backslash policy is decided per-string: - // - if every `\` is followed by a valid JSON escape char, treat - // them as escapes (so e.g. `\n` becomes a real newline) - // - otherwise treat every `\` as a literal char (so Windows paths - // like `C:\Users\nock9\Desktop\abc.txt` are preserved verbatim) - auto is_escape_char = [](char c) { - return c == '"' || c == '\\' || c == '/' || - c == 'b' || c == 'f' || c == 'n' || - c == 'r' || c == 't' || c == 'u'; - }; - auto encode_raw = [&](const std::string& raw, std::string& out) { - bool all_valid = true; - for (size_t k = 0; k < raw.size(); ++k) { - if (raw[k] == '\\') { - if (k + 1 >= raw.size() || !is_escape_char(raw[k + 1])) { - all_valid = false; - break; - } - ++k; - } - } - for (size_t k = 0; k < raw.size(); ++k) { - char c = raw[k]; - if (c == '\\' && all_valid && k + 1 < raw.size()) { - char nc = raw[k + 1]; - if (nc == 'u' && k + 5 < raw.size() && - std::isxdigit(static_cast(raw[k + 2])) && - std::isxdigit(static_cast(raw[k + 3])) && - std::isxdigit(static_cast(raw[k + 4])) && - std::isxdigit(static_cast(raw[k + 5]))) { - out.append(raw, k, 6); - k += 5; - } else { - out.push_back('\\'); - out.push_back(nc); - ++k; - } - } else { - json_escape_char(out, c); - } - } - }; - - std::string value; - std::string raw; // used in MARKER / BARE modes - int depth = 0; - while (i < s.size()) { - if (mode == MARKER) { - if (match_marker(i)) { - size_t after = i + marker_len; - if (is_terminator(after)) { i = after; break; } - raw.append(quote_marker_lit, marker_len); - i = after; - continue; - } - if (s[i] == '"' || s[i] == '`') { - // The model occasionally substitutes a plain `"` or even - // a backtick `` ` `` for the closing <|"|> marker. Treat - // either as a close only when followed by a terminator, - // so they remain valid string content otherwise. - if (is_terminator(i + 1)) { ++i; break; } - raw.push_back(s[i]); - ++i; - continue; - } - raw.push_back(s[i]); - ++i; - continue; - } - if (mode == QUOTE) { - char c = s[i]; - if (c == '\\') { - consume_quote_escape(value); - continue; - } - if (c == '"') { ++i; break; } - json_escape_char(value, c); - ++i; - continue; - } - // BARE - if (match_marker(i)) { - size_t after = i + marker_len; - if (depth == 0 && is_terminator(after)) { i = after; break; } - raw.append(quote_marker_lit, marker_len); - i = after; - continue; - } - char c = s[i]; - if (depth == 0 && terminators.find(c) != std::string::npos) break; - if (c == '{' || c == '[' || c == '(') ++depth; - else if ((c == '}' || c == ']' || c == ')') && depth > 0) --depth; - raw.push_back(c); - ++i; - } - - if (mode != QUOTE) { - encode_raw(raw, value); - } - - std::string out = "\""; - out.append(value); - out.push_back('"'); - return out; - } -}; - -std::pair parse_gemma4_12b_tool_content(std::string tool_content) { - tool_content = trim_gemma4_12b_tool_value(tool_content); - - const std::string prefix = "call:"; - if (tool_content.find(prefix) == 0) { - tool_content = trim_gemma4_12b_tool_value(tool_content.substr(prefix.length())); - } - - // only has tool name but no args - size_t brace_pos = tool_content.find('{'); - if (brace_pos == std::string::npos) { - return {trim_gemma4_12b_tool_value(tool_content), json::object()}; - } - - std::string tool_name = trim_gemma4_12b_tool_value(tool_content.substr(0, brace_pos)); - std::string args_str = trim_gemma4_12b_tool_value(tool_content.substr(brace_pos)); - - // Rewrite the relaxed args into well-formed JSON via recursive-descent. - Gemma4_12BArgsParser parser(args_str); - parser.skip_ws(); - std::string normalized; - if (parser.i < args_str.size() && args_str[parser.i] == '{') { - normalized = parser.parse_object(); - } else { - // Unexpected shape; wrap whatever we get into an object so the - // downstream JSON parse step still has a sensible structure. - normalized = parser.parse_value(""); - } - - // Final: parse to a real JSON object; fall back to empty object on failure. - json args_json = json::object(); - try { - args_json = json::parse(normalized); - } catch (const std::exception& e) { - std::cerr << "[WARNING] Failed to parse tool args as JSON: " - << e.what() << std::endl; - std::cerr << "Raw args: " << normalized << std::endl; - } - - return {tool_name, args_json}; -} -} - - /************ Gemma4_12B family **************/ Gemma4_12B::Gemma4_12B(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4_12B") {} @@ -897,6 +507,8 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, if(!enable_think) { this->token_history.push_back(boc_token_id); this->profiler_list[DECODING_TIME].start(); + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); this->lm_engine->forward(boc_token_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(boc_token_id); @@ -919,10 +531,7 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, buffer y = this->lm_engine->forward(eoc_token_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(eoc_token_id); - // First real token after the forced empty-thought preamble -- the model - // would open a tool call right here, so it needs the mask too. - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); this->total_tokens++; meta_info.generated_tokens++; @@ -953,11 +562,6 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -965,10 +569,7 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - // Gemma4_12B runs its own decode loop instead of _shared_generate, so the - // tool_choice mask has to be applied here as well as in _shared_insert. - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -1023,222 +624,3 @@ std::string Gemma4_12B::generate_with_prompt(chat_meta_info_t& meta_info, lm_uni } return this->generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult Gemma4_12B::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string think_start_tag = "<|channel>thought"; - std::string think_end_tag = ""; - std::string tool_start_tag = "<|tool_call>"; - std::string tool_end_tag = ""; - std::string tool_resp_tag = "<|tool_response>"; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - size_t tool_start_pos = response_text.find(tool_start_tag); - size_t tool_end_pos = response_text.find(tool_end_tag, tool_start_pos == std::string::npos ? 0 : tool_start_pos + tool_start_tag.length()); - - bool is_reasoning = (think_start_pos != std::string::npos && think_end_pos != std::string::npos && think_end_pos > think_start_pos); - bool is_tool = (tool_start_pos != std::string::npos); - - // 1. Parse Reasoning Content - if (is_reasoning) { - size_t start = think_start_pos + think_start_tag.length(); - result.reasoning_content = response_text.substr(start, think_end_pos - start); - } - - // 2. Parse Tool Calling - if (is_tool) { - size_t search_from = 0; - while (true) { - size_t start_pos = response_text.find(tool_start_tag, search_from); - if (start_pos == std::string::npos) break; - - size_t block_content_start = start_pos + tool_start_tag.length(); - size_t end_pos = response_text.find(tool_end_tag, block_content_start); - - size_t block_end; - if (end_pos != std::string::npos) { - block_end = end_pos; - search_from = end_pos + tool_end_tag.length(); - } else { - size_t resp_pos = response_text.find(tool_resp_tag, block_content_start); - block_end = (resp_pos != std::string::npos) ? resp_pos : response_text.length(); - search_from = block_end; - } - - std::string tool_content = response_text.substr(block_content_start, block_end - block_content_start); - auto parsed_tool = parse_gemma4_12b_tool_content(tool_content); - result.tool_calls_list.emplace_back(parsed_tool.first, parsed_tool.second.dump()); - } - - if (!result.tool_calls_list.empty()) { - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - } - } - // 3. Parse Normal Content - else { - if (is_reasoning) { - // Content is whatever comes AFTER the reasoning block - result.content = response_text.substr(think_end_pos + think_end_tag.length()); - } else { - // No reasoning, no tools -> the whole text is content - result.content = response_text; - } - - // Cleanup: Strip out <|tool_response> if the model accidentally hallucinated it into plain text - size_t resp_pos = 0; - while ((resp_pos = result.content.find(tool_resp_tag, resp_pos)) != std::string::npos) { - result.content.erase(resp_pos, tool_resp_tag.length()); - } - } - - return result; -} - - -// Stream -StreamResult Gemma4_12B::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Gemma4_12B::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Gemma4_12B::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = "<|channel>thought"; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = "<|tool_call>"; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_TOOL_RESP = "<|tool_response>"; - - StreamResult result; - buffer_ += content; - - while (true) { - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t tool_resp_pos = buffer_.find(MARKER_TOOL_RESP); - - if (tool_end_pos != std::string::npos || tool_resp_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - if (tool_resp_pos != std::string::npos && tool_resp_pos < actual_end_pos) { - actual_end_pos = tool_resp_pos; - skip_length = MARKER_TOOL_RESP.length(); - } - - std::string tool_content = buffer_.substr(0, actual_end_pos); - - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - result.type = StreamEventType::TOOL_DONE; - - static int tool_counter = 0; - result.tool_id = "call_" + std::to_string(std::time(nullptr)) + "_" + std::to_string(tool_counter++); - auto parsed_tool = parse_gemma4_12b_tool_content(tool_content); - result.tool_name = parsed_tool.first; - result.tool_args_str = parsed_tool.second.dump(); - return result; - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - // Find the earliest occurring marker in the buffer to avoid skipping tags - size_t pos_tool_start = buffer_.find(MARKER_TOOL_START); - size_t pos_tool_resp = buffer_.find(MARKER_TOOL_RESP); - size_t pos_think = std::string::npos; - - if (current_mode_ == StreamEventType::CONTENT) { - pos_think = buffer_.find(MARKER_THINK_START); - } - else if (current_mode_ == StreamEventType::REASONING) { - pos_think = buffer_.find(MARKER_THINK_END); - } - - size_t min_pos = std::string::npos; - if (pos_tool_start != std::string::npos) min_pos = std::min(min_pos, pos_tool_start); - if (pos_tool_resp != std::string::npos) min_pos = std::min(min_pos, pos_tool_resp); - if (pos_think != std::string::npos) min_pos = std::min(min_pos, pos_think); - - // Flush the text content before the earliest marker - if (min_pos != std::string::npos && min_pos > 0) { - result.content = buffer_.substr(0, min_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(min_pos); - return result; - } - - // Process the exact marker located at index 0 - if (min_pos == 0) { - if (pos_tool_resp == 0) { - buffer_ = buffer_.substr(MARKER_TOOL_RESP.length()); - continue; - } - if (pos_tool_start == 0) { - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - continue; - } - if (pos_think == 0) { - if (current_mode_ == StreamEventType::CONTENT) { - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - } else { - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - } - continue; - } - } - - // Safe Flush Mechanism - std::vector active_markers; - active_markers.push_back(MARKER_TOOL_START); - active_markers.push_back(MARKER_TOOL_RESP); - - if (current_mode_ == StreamEventType::CONTENT) { - active_markers.push_back(MARKER_THINK_START); - } - else if (current_mode_ == StreamEventType::REASONING) { - active_markers.push_back(MARKER_THINK_END); - } - - size_t safe_flush_len = buffer_.length(); - for (const auto& marker : active_markers) { - for (size_t i = 1; i <= marker.length() && i <= buffer_.length(); ++i) { - if (buffer_.compare(buffer_.length() - i, i, marker, 0, i) == 0) { - safe_flush_len = std::min(safe_flush_len, buffer_.length() - i); - } - } - } - - if (safe_flush_len > 0) { - result.content = buffer_.substr(0, safe_flush_len); - result.type = current_mode_; - buffer_ = buffer_.substr(safe_flush_len); - return result; - } - else if (buffer_.length() > 0) { - result.type = StreamEventType::WAITING; - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index 0803de4a3..f17c48da7 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -9,406 +9,6 @@ #include "AutoModel/modeling_gemma4e.hpp" #include "metrices.hpp" -namespace { -std::string trim_gemma4e_tool_value(std::string value) { - size_t start = value.find_first_not_of(" \t\r\n"); - if (start == std::string::npos) { - return ""; - } - - size_t end = value.find_last_not_of(" \t\r\n"); - return value.substr(start, end - start + 1); -} - -// --- Gemma4e tool-args parser ------------------------------------------------- -// The model emits relaxed JSON for tool arguments: -// - keys are bare identifiers (e.g. `name:`) — sometimes also "..." quoted -// - string values are delimited by <|"|>...<|"|>, but the model frequently -// omits the opener and/or replaces the closer with a plain `"` -// - values can also be objects {..}, arrays [..], booleans, null, or numbers -// - inside a string value, raw `"`, raw newlines, and even the literal -// substring `<|"|>` can appear as content -// -// We rewrite the input into well-formed JSON via recursive-descent. -struct Gemma4eArgsParser { - const std::string& s; - size_t i = 0; - static constexpr size_t marker_len = 5; - static constexpr const char* quote_marker_lit = "<|\"|>"; - - explicit Gemma4eArgsParser(const std::string& src) : s(src) {} - - void skip_ws() { - while (i < s.size() && - std::isspace(static_cast(s[i]))) ++i; - } - void skip_ws_at(size_t& pos) const { - while (pos < s.size() && - std::isspace(static_cast(s[pos]))) ++pos; - } - bool match_marker(size_t pos) const { - return pos + marker_len <= s.size() && - s.compare(pos, marker_len, quote_marker_lit) == 0; - } - - static void json_escape_char(std::string& out, char c) { - switch (c) { - case '"': out.append("\\\""); break; - case '\\': out.append("\\\\"); break; - case '\n': out.append("\\n"); break; - case '\r': out.append("\\r"); break; - case '\t': out.append("\\t"); break; - case '\b': out.append("\\b"); break; - case '\f': out.append("\\f"); break; - default: - if (static_cast(c) < 0x20) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", - static_cast(c)); - out.append(buf); - } else { - out.push_back(c); - } - break; - } - } - - bool match_word(const char* w, size_t len) const { - if (i + len > s.size()) return false; - if (s.compare(i, len, w) != 0) return false; - if (i + len == s.size()) return true; - unsigned char nc = static_cast(s[i + len]); - return !(std::isalnum(nc) || nc == '_'); - } - - // Parse one JSON value. `terminators` is the set of chars that end a - // bare (undelimited) string at depth 0 (e.g. ",}" inside an object, - // ",]" inside an array, "" at the very top). - std::string parse_value(const std::string& terminators) { - skip_ws(); - if (i >= s.size()) return "null"; - char c = s[i]; - if (c == '{') return parse_object(); - if (c == '[') return parse_array(); - if (match_word("true", 4)) { i += 4; return "true"; } - if (match_word("false", 5)) { i += 5; return "false"; } - if (match_word("null", 4)) { i += 4; return "null"; } - if (c == '-' || (c >= '0' && c <= '9')) return parse_number(); - return parse_string(terminators); - } - - std::string parse_number() { - size_t start = i; - if (s[i] == '-') ++i; - while (i < s.size()) { - char c = s[i]; - if ((c >= '0' && c <= '9') || c == '.' || - c == 'e' || c == 'E' || c == '+' || c == '-') ++i; - else break; - } - return s.substr(start, i - start); - } - - std::string parse_key() { - skip_ws(); - std::string key; - if (i >= s.size()) return key; - if (match_marker(i)) { - i += marker_len; - while (i < s.size() && !match_marker(i)) key.push_back(s[i++]); - if (match_marker(i)) i += marker_len; - } else if (s[i] == '"') { - ++i; - while (i < s.size() && s[i] != '"') { - if (s[i] == '\\' && i + 1 < s.size()) { - key.push_back(s[i]); - key.push_back(s[i + 1]); - i += 2; - } else { - key.push_back(s[i++]); - } - } - if (i < s.size() && s[i] == '"') ++i; - } else { - while (i < s.size() && - (std::isalnum(static_cast(s[i])) || - s[i] == '_')) { - key.push_back(s[i++]); - } - } - return key; - } - - std::string parse_object() { - // assumes s[i] == '{' - ++i; - std::string out = "{"; - bool first = true; - while (i < s.size()) { - skip_ws(); - if (i >= s.size()) break; - if (s[i] == '}') { ++i; break; } - - std::string key = parse_key(); - if (key.empty()) { - // can't make progress; bail - if (i < s.size() && s[i] == '}') { ++i; } - break; - } - skip_ws(); - if (i < s.size() && s[i] == ':') ++i; - - std::string val = parse_value(",}"); - - if (!first) out.push_back(','); - first = false; - out.push_back('"'); - for (char kc : key) json_escape_char(out, kc); - out.append("\":"); - out.append(val); - - skip_ws(); - if (i < s.size() && s[i] == ',') ++i; - } - out.push_back('}'); - return out; - } - - std::string parse_array() { - // assumes s[i] == '[' - ++i; - std::string out = "["; - bool first = true; - while (i < s.size()) { - skip_ws(); - if (i >= s.size()) break; - if (s[i] == ']') { ++i; break; } - - std::string val = parse_value(",]"); - if (!first) out.push_back(','); - first = false; - out.append(val); - - skip_ws(); - if (i < s.size() && s[i] == ',') ++i; - } - out.push_back(']'); - return out; - } - - // Parse a string value. Accepts three forms: - // - <|"|>...<|"|> (or <|"|>...") -- marker opener, marker or " closer - // - "..." -- regular JSON string - // - bare text -- ends at depth-0 terminator - std::string parse_string(const std::string& terminators) { - enum Mode { MARKER, QUOTE, BARE }; - Mode mode = BARE; - if (match_marker(i)) { mode = MARKER; i += marker_len; } - else if (s[i] == '"') { mode = QUOTE; ++i; } - - auto is_terminator = [&](size_t pos) { - size_t k = pos; - skip_ws_at(k); - if (k >= s.size()) return true; - return terminators.find(s[k]) != std::string::npos; - }; - - // QUOTE-mode only: honor JSON-style backslash escapes verbatim. - auto consume_quote_escape = [&](std::string& out) { - // s[i] == '\\' - if (i + 1 >= s.size()) { - json_escape_char(out, s[i]); - ++i; - return; - } - char nc = s[i + 1]; - switch (nc) { - case '"': case '\\': case '/': - case 'b': case 'f': case 'n': case 'r': case 't': - out.push_back('\\'); - out.push_back(nc); - i += 2; - return; - case 'u': { - if (i + 5 < s.size() && - std::isxdigit(static_cast(s[i + 2])) && - std::isxdigit(static_cast(s[i + 3])) && - std::isxdigit(static_cast(s[i + 4])) && - std::isxdigit(static_cast(s[i + 5]))) { - out.append(s, i, 6); - i += 6; - return; - } - break; - } - default: break; - } - json_escape_char(out, s[i]); - ++i; - }; - - // For MARKER and BARE modes we first collect the raw content, then - // JSON-encode it. Backslash policy is decided per-string: - // - if every `\` is followed by a valid JSON escape char, treat - // them as escapes (so e.g. `\n` becomes a real newline) - // - otherwise treat every `\` as a literal char (so Windows paths - // like `C:\Users\nock9\Desktop\abc.txt` are preserved verbatim) - auto is_escape_char = [](char c) { - return c == '"' || c == '\\' || c == '/' || - c == 'b' || c == 'f' || c == 'n' || - c == 'r' || c == 't' || c == 'u'; - }; - auto encode_raw = [&](const std::string& raw, std::string& out) { - bool all_valid = true; - for (size_t k = 0; k < raw.size(); ++k) { - if (raw[k] == '\\') { - if (k + 1 >= raw.size() || !is_escape_char(raw[k + 1])) { - all_valid = false; - break; - } - ++k; - } - } - for (size_t k = 0; k < raw.size(); ++k) { - char c = raw[k]; - if (c == '\\' && all_valid && k + 1 < raw.size()) { - char nc = raw[k + 1]; - if (nc == 'u' && k + 5 < raw.size() && - std::isxdigit(static_cast(raw[k + 2])) && - std::isxdigit(static_cast(raw[k + 3])) && - std::isxdigit(static_cast(raw[k + 4])) && - std::isxdigit(static_cast(raw[k + 5]))) { - out.append(raw, k, 6); - k += 5; - } else { - out.push_back('\\'); - out.push_back(nc); - ++k; - } - } else { - json_escape_char(out, c); - } - } - }; - - std::string value; - std::string raw; // used in MARKER / BARE modes - int depth = 0; - while (i < s.size()) { - if (mode == MARKER) { - if (match_marker(i)) { - size_t after = i + marker_len; - if (is_terminator(after)) { i = after; break; } - raw.append(quote_marker_lit, marker_len); - i = after; - continue; - } - if (s[i] == '"' || s[i] == '`') { - // The model occasionally substitutes a plain `"` or even - // a backtick `` ` `` for the closing <|"|> marker. Treat - // either as a close only when followed by a terminator, - // so they remain valid string content otherwise. - if (is_terminator(i + 1)) { ++i; break; } - raw.push_back(s[i]); - ++i; - continue; - } - raw.push_back(s[i]); - ++i; - continue; - } - if (mode == QUOTE) { - char c = s[i]; - if (c == '\\') { - consume_quote_escape(value); - continue; - } - if (c == '"') { ++i; break; } - json_escape_char(value, c); - ++i; - continue; - } - // BARE - if (match_marker(i)) { - size_t after = i + marker_len; - if (depth == 0 && is_terminator(after)) { i = after; break; } - raw.append(quote_marker_lit, marker_len); - i = after; - continue; - } - char c = s[i]; - if (depth == 0 && terminators.find(c) != std::string::npos) break; - if (c == '{' || c == '[' || c == '(') ++depth; - else if ((c == '}' || c == ']' || c == ')') && depth > 0) --depth; - raw.push_back(c); - ++i; - } - - if (mode != QUOTE) { - encode_raw(raw, value); - } - - std::string out = "\""; - out.append(value); - out.push_back('"'); - return out; - } -}; - -std::pair parse_gemma4e_tool_content(std::string tool_content) { - tool_content = trim_gemma4e_tool_value(tool_content); - - const std::string prefix = "call:"; - if (tool_content.find(prefix) == 0) { - tool_content = trim_gemma4e_tool_value(tool_content.substr(prefix.length())); - } - - // std::cout << "[DEBUG after prefix removal]" << std::endl; - // std::cout << tool_content << std::endl; - - // only has tool name but no args - size_t brace_pos = tool_content.find('{'); - if (brace_pos == std::string::npos) { - return {trim_gemma4e_tool_value(tool_content), json::object()}; - } - - std::string tool_name = trim_gemma4e_tool_value(tool_content.substr(0, brace_pos)); - std::string args_str = trim_gemma4e_tool_value(tool_content.substr(brace_pos)); - - // std::cout << "[DEBUG after parsing tool name and args]" << std::endl; - // std::cout << "Tool Name: " << tool_name << std::endl; - // std::cout << "Args Str: " << args_str << std::endl; - - // Rewrite the relaxed args into well-formed JSON via recursive-descent. - Gemma4eArgsParser parser(args_str); - parser.skip_ws(); - std::string normalized; - if (parser.i < args_str.size() && args_str[parser.i] == '{') { - normalized = parser.parse_object(); - } else { - // Unexpected shape; wrap whatever we get into an object so the - // downstream JSON parse step still has a sensible structure. - normalized = parser.parse_value(""); - } - - // std::cout << "[DEBUG normalized args]" << std::endl; - // std::cout << normalized << std::endl; - - // Final: parse to a real JSON object; fall back to empty object on failure. - json args_json = json::object(); - try { - args_json = json::parse(normalized); - } catch (const std::exception& e) { - std::cerr << "[WARNING] Failed to parse tool args as JSON: " - << e.what() << std::endl; - std::cerr << "Raw args: " << normalized << std::endl; - } - - return {tool_name, args_json}; -} -} - - /************ Gemma4e family **************/ Gemma4e::Gemma4e(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4e") {} @@ -518,8 +118,6 @@ void Gemma4e_Flash::_reset_turn() { } this->sampler->reset_penalties(); this->last_token = -1; - this->reset_parser(); - this->tool_name_.clear(); this->profiler_list[PREFILL_TIME].reset(); this->profiler_list[DECODING_TIME].reset(); this->profiler_list[SAMPLING_TIME].reset(); @@ -587,10 +185,6 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi while (this->total_tokens < this->MAX_L) { if (is_cancelled()) { reason = CANCEL_DETECTED; - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } @@ -599,7 +193,7 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -1113,11 +707,6 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -1125,10 +714,7 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - // Gemma4e runs its own decode loop instead of _shared_generate, so the - // tool_choice mask has to be applied here as well as in _shared_insert. - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -1183,224 +769,3 @@ std::string Gemma4e::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifor header_print_r("FLM", "Checkpoint before generation: " << checkpoint_idx << ", restore point: " << restore_idx << ", user context length: " << this->token_history.size()); return this->_shared_generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult Gemma4e::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string think_start_tag = "<|channel>thought"; - std::string think_end_tag = ""; - std::string tool_start_tag = "<|tool_call>"; - std::string tool_end_tag = ""; - std::string tool_resp_tag = "<|tool_response>"; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - size_t tool_start_pos = response_text.find(tool_start_tag); - size_t tool_end_pos = response_text.find(tool_end_tag, tool_start_pos == std::string::npos ? 0 : tool_start_pos + tool_start_tag.length()); - - bool is_reasoning = (think_start_pos != std::string::npos && think_end_pos != std::string::npos && think_end_pos > think_start_pos); - bool is_tool = (tool_start_pos != std::string::npos); - - // 1. Parse Reasoning Content - if (is_reasoning) { - size_t start = think_start_pos + think_start_tag.length(); - result.reasoning_content = response_text.substr(start, think_end_pos - start); - } - - // 2. Parse Tool Calling - if (is_tool) { - size_t search_from = 0; - while (true) { - size_t start_pos = response_text.find(tool_start_tag, search_from); - if (start_pos == std::string::npos) break; - - size_t block_content_start = start_pos + tool_start_tag.length(); - size_t end_pos = response_text.find(tool_end_tag, block_content_start); - - size_t block_end; - if (end_pos != std::string::npos) { - block_end = end_pos; - search_from = end_pos + tool_end_tag.length(); - } else { - size_t resp_pos = response_text.find(tool_resp_tag, block_content_start); - block_end = (resp_pos != std::string::npos) ? resp_pos : response_text.length(); - search_from = block_end; - } - - std::string tool_content = response_text.substr(block_content_start, block_end - block_content_start); - auto parsed_tool = parse_gemma4e_tool_content(tool_content); - result.tool_calls_list.emplace_back(parsed_tool.first, parsed_tool.second.dump()); - } - - if (!result.tool_calls_list.empty()) { - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - } - } - // 3. Parse Normal Content - else { - if (is_reasoning) { - // Content is whatever comes AFTER the reasoning block - result.content = response_text.substr(think_end_pos + think_end_tag.length()); - } else { - // No reasoning, no tools -> the whole text is content - result.content = response_text; - } - - // Cleanup: Strip out <|tool_response> if the model accidentally hallucinated it into plain text - size_t resp_pos = 0; - while ((resp_pos = result.content.find(tool_resp_tag, resp_pos)) != std::string::npos) { - result.content.erase(resp_pos, tool_resp_tag.length()); - } - } - - return result; -} - - -// Stream -StreamResult Gemma4e::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Gemma4e::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Gemma4e::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = "<|channel>thought"; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = "<|tool_call>"; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_TOOL_RESP = "<|tool_response>"; - - StreamResult result; - buffer_ += content; - - while (true) { - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t tool_resp_pos = buffer_.find(MARKER_TOOL_RESP); - - if (tool_end_pos != std::string::npos || tool_resp_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - if (tool_resp_pos != std::string::npos && tool_resp_pos < actual_end_pos) { - actual_end_pos = tool_resp_pos; - skip_length = MARKER_TOOL_RESP.length(); - } - - std::string tool_content = buffer_.substr(0, actual_end_pos); - - // std::cout << "DEBUG" << std::endl; - // std::cout << tool_content << std::endl; // For debugging: see the raw tool content extracted from the stream - - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - result.type = StreamEventType::TOOL_DONE; - - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - auto parsed_tool = parse_gemma4e_tool_content(tool_content); - result.tool_name = parsed_tool.first; - result.tool_args_str = parsed_tool.second.dump(); - return result; - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - // Find the earliest occurring marker in the buffer to avoid skipping tags - size_t pos_tool_start = buffer_.find(MARKER_TOOL_START); - size_t pos_tool_resp = buffer_.find(MARKER_TOOL_RESP); - size_t pos_think = std::string::npos; - - if (current_mode_ == StreamEventType::CONTENT) { - pos_think = buffer_.find(MARKER_THINK_START); - } - else if (current_mode_ == StreamEventType::REASONING) { - pos_think = buffer_.find(MARKER_THINK_END); - } - - size_t min_pos = std::string::npos; - if (pos_tool_start != std::string::npos) min_pos = std::min(min_pos, pos_tool_start); - if (pos_tool_resp != std::string::npos) min_pos = std::min(min_pos, pos_tool_resp); - if (pos_think != std::string::npos) min_pos = std::min(min_pos, pos_think); - - // Flush the text content before the earliest marker - if (min_pos != std::string::npos && min_pos > 0) { - result.content = buffer_.substr(0, min_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(min_pos); - return result; - } - - // Process the exact marker located at index 0 - if (min_pos == 0) { - if (pos_tool_resp == 0) { - buffer_ = buffer_.substr(MARKER_TOOL_RESP.length()); - continue; - } - if (pos_tool_start == 0) { - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - continue; - } - if (pos_think == 0) { - if (current_mode_ == StreamEventType::CONTENT) { - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - } else { - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - } - continue; - } - } - - // Safe Flush Mechanism - std::vector active_markers; - active_markers.push_back(MARKER_TOOL_START); - active_markers.push_back(MARKER_TOOL_RESP); - - if (current_mode_ == StreamEventType::CONTENT) { - active_markers.push_back(MARKER_THINK_START); - } - else if (current_mode_ == StreamEventType::REASONING) { - active_markers.push_back(MARKER_THINK_END); - } - - size_t safe_flush_len = buffer_.length(); - for (const auto& marker : active_markers) { - for (size_t i = 1; i <= marker.length() && i <= buffer_.length(); ++i) { - if (buffer_.compare(buffer_.length() - i, i, marker, 0, i) == 0) { - safe_flush_len = std::min(safe_flush_len, buffer_.length() - i); - } - } - } - - if (safe_flush_len > 0) { - result.content = buffer_.substr(0, safe_flush_len); - result.type = current_mode_; - buffer_ = buffer_.substr(safe_flush_len); - return result; - } - else if (buffer_.length() > 0) { - result.type = StreamEventType::WAITING; - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_gpt_oss.cpp b/src/common/AutoModel/modeling_gpt_oss.cpp index 31ad039f9..692f3ac23 100644 --- a/src/common/AutoModel/modeling_gpt_oss.cpp +++ b/src/common/AutoModel/modeling_gpt_oss.cpp @@ -48,7 +48,6 @@ std::string GPT_OSS::apply_chat_template(nlohmann::ordered_json& messages, nlohm return this->chat_tmpl->apply(inputs); } -json tools; bool GPT_OSS::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { // preprocess @@ -59,7 +58,6 @@ bool GPT_OSS::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - tools = input.tools; templated_text = this->apply_chat_template(input.messages); //templated_text = this->apply_chat_template(input.messages, input.tools); } @@ -127,11 +125,6 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -139,7 +132,7 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -175,113 +168,6 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std return "<|start|>assistant" + result + "<|end|>"; } -/* -// std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { -// const std::string MARKER_TOOL_START = "<|start|>assistant<|channel|>commentary"; -// bool is_constrained_mode = false; - -// os << "<|start|>" << std::flush; -// os << "assistant" << std::flush; -// std::string result = this->_shared_generate(meta_info, length_limit, os, is_cancelled); -// //std::vector sampled_tokens; -// //std::string result; -// //if (length_limit > 0) { -// // sampled_tokens.reserve(length_limit); -// //} -// //else { -// // sampled_tokens.reserve(4096); -// //} -// //assert(this->last_token != -1); - -// //stop_reason_t reason = EOT_DETECTED; -// //int last_sampled_token = this->last_token; -// //this->token_history.push_back(this->last_token); -// //auto decoding_start_time = time_utils::now(); -// //if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1) { -// // std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); -// // result += token_str; -// // os << token_str << std::flush; - -// //} -// //if (this->is_eos(last_sampled_token)) { -// // return result; -// //} -// //this->profiler_list[TKOEN_DECODE_TIME].stop(1); -// //if (this->total_tokens >= this->MAX_L) { -// // header_print("WARNING", "Max length reached, stopping generation..."); -// // reason = MAX_LENGTH_REACHED; -// // return result; -// //} -// //while (this->total_tokens < this->MAX_L) { -// // if (is_cancelled()) { -// // reason = CANCEL_DETECTED; -// // // reset stream content -// // buffer_.clear(); -// // current_mode_ = StreamEventType::CONTENT; -// // waiting_for_header_ = true; -// // break; -// // } -// // this->profiler_list[DECODING_TIME].start(); -// // buffer y = this->lm_engine->forward(last_sampled_token); -// // this->profiler_list[DECODING_TIME].stop(1); - -// // this->profiler_list[SAMPLING_TIME].start(); -// // // if MARKER_TOOL_START found -// // -// // if (!is_constrained_mode) { -// // if (result.length() >= MARKER_TOOL_START.length() && -// // result.compare(result.length() - MARKER_TOOL_START.length(), MARKER_TOOL_START.length(), MARKER_TOOL_START) == 0) { -// // is_constrained_mode = true; -// // reset_tool_grammar_state(); -// // header_print("INFO", "<|start|>assistant<|channel|>commentary detected! Switching to Constrained Decoding."); -// // } -// // } - -// // int sampled_token; -// // if (is_constrained_mode) { -// // sampled_token = _sample_in_tool(y); -// // if (current_state == STATE_COMPLETE) { -// // is_constrained_mode = false; -// // header_print("INFO", "Constrained Decoding complete, returning to normal mode"); -// // } -// // } -// // else -// // sampled_token = this->sampler->sample(y); -// // this->profiler_list[SAMPLING_TIME].stop(1); -// // this->total_tokens++; -// // last_sampled_token = sampled_token; - -// // this->profiler_list[TKOEN_DECODE_TIME].start(); -// // this->profiler_list[TKOEN_DECODE_TIME].stop(1); -// // if (this->is_normal_token(sampled_token)) { // filter out special tokens -// // std::string token_str = this->tokenizer->run_time_decoder(sampled_token); -// // os << token_str << std::flush; -// // result += token_str; -// // } -// // this->token_history.push_back(sampled_token); -// // if (this->is_eos(sampled_token)) { -// // this->lm_engine->forward(last_sampled_token); -// // break; -// // } -// // meta_info.generated_tokens++; -// // if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)) { -// // reason = MAX_LENGTH_REACHED; -// // break; -// // } -// //} - -// //auto decoding_end_time = time_utils::now(); -// //meta_info.decoding_duration = (uint64_t)time_utils::duration_ns(decoding_start_time, decoding_end_time).first; -// //meta_info.stop_reason = reason; -// //if (this->total_tokens >= this->MAX_L) { -// // header_print("WARNING", "Max length reached, stopping generation..."); -// //} - -// os << "<|end|>" << std::flush; -// return "<|start|>assistant" + result + "<|end|>"; -// } -*/ - std::string GPT_OSS::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { if (!this->insert(meta_info, input)) { return ""; @@ -292,289 +178,3 @@ std::string GPT_OSS::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifor os << "<|end|>" << std::flush; return "<|start|>assistant" + result + "<|end|>"; } - -NonStreamResult GPT_OSS::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - const std::string think_start_tag = "<|start|>assistant<|channel|>analysis<|message|>"; - const std::string think_end_tag = "<|end|>"; - const std::string final_start_tag = "<|start|>assistant<|channel|>final<|message|>"; - const std::string final_end_tag = "<|end|>"; - - // --- Parse reasoning content --- - size_t t_start = response_text.find(think_start_tag); - size_t t_end = response_text.find(think_end_tag, t_start + think_start_tag.size()); - - if (t_start != std::string::npos && t_end != std::string::npos) { - t_start += think_start_tag.size(); - result.reasoning_content = response_text.substr(t_start, t_end - t_start); - } - - // --- Parse final content --- - size_t f_start = response_text.find(final_start_tag); - size_t f_end = response_text.find(final_end_tag, f_start + final_start_tag.size()); - - if (f_start != std::string::npos && f_end != std::string::npos) { - f_start += final_start_tag.size(); - result.content = response_text.substr(f_start, f_end - f_start); - } - - return result; -} - -int GPT_OSS::_sample_in_tool(buffer& logits) { - // to=functions.get_current_weather <|constrain|>json<|message|>{"location":"San Francisco"}<|call|> - std::vector allowed_tokens; - switch (current_state) { - case STATE_EXPECT_TO_FUNCTIONS: { - allowed_tokens = get_to_functions_tokens(); - break; - } - case STATE_EXPECT_FUNCTION_NAME: { - allowed_tokens = get_function_name_tokens(); - break; - } - case STATE_EXPECT_CONSTRAIN: { - auto constrain_tokens = tokenizer->encode(" <|constrain|>"); - auto message_tokens = tokenizer->encode("json"); - allowed_tokens.insert(allowed_tokens.end(), - constrain_tokens.begin(), constrain_tokens.end()); - allowed_tokens.insert(allowed_tokens.end(), - message_tokens.begin(), message_tokens.end()); - break; - } - case STATE_EXPECT_MESSAGE: { - auto tokens = tokenizer->encode("<|message|>"); - allowed_tokens = { tokens[0] }; - break; - } - //case STATE_IN_JSON: { - // header_print("JSONSSS", "SSSS"); - // return this->sampler->sample(logits); - // //allowed_tokens = constrain_by_json_schema(logits); - // //break; - //} - } - mask_logits(logits, allowed_tokens); - int token_id = this->sampler->sample(logits); - update_state(token_id); - return token_id; - //return this->sampler->sample(logits); -} - -std::vector GPT_OSS::get_to_functions_tokens() { - const std::string target = " to=functions."; - - std::string remaining = target.substr(accumulated_text.length()); - - auto remaining_tokens = tokenizer->encode(remaining); - - if (remaining_tokens.empty()) { - return {}; - } - - return { remaining_tokens[0] }; -} - -std::vector GPT_OSS::get_function_name_tokens() { - std::vector allowed; - - for (const auto& tool : tools) { - std::string name = tool["function"]["name"]; - - auto tokens = this->tokenizer->encode(name); - for (int token : tokens) { - allowed.push_back(token); - } - } - - return allowed; -} - -void GPT_OSS::update_state(int token_id) { - std::string token_str = tokenizer->decode({ token_id }); - accumulated_text += token_str; - - switch (current_state) { - case STATE_EXPECT_TO_FUNCTIONS: { - if (accumulated_text.find("to=functions.") != std::string::npos) { - current_state = STATE_EXPECT_FUNCTION_NAME; - accumulated_text.clear(); - header_print("TOOLCALLING", "Completed 'to=functions.', expecting function name"); - } - break; - } - case STATE_EXPECT_FUNCTION_NAME: { - //bool found = false; - for (const auto& tool : tools) { - std::string name = tool["function"]["name"]; - if (accumulated_text == name) { - current_state = STATE_EXPECT_CONSTRAIN; - accumulated_text.clear(); - //found = true; - header_print("TOOLCALLING", "Matched function: " + name); - break; - } - } - break; - } - case STATE_EXPECT_CONSTRAIN: { - if (token_str.find("json") != std::string::npos) { - current_state = STATE_EXPECT_MESSAGE; - accumulated_text.clear(); - header_print("TOOLCALLING", "Starting MESSAGE generation"); - } - break; - } - case STATE_EXPECT_MESSAGE: { - if (token_str.find("<|message|>") != std::string::npos) { - current_state = STATE_COMPLETE; - accumulated_text.clear(); - } - break; - } - } -} - -void GPT_OSS::mask_logits(buffer& logits, const std::vector& allowed_tokens) { - const float NEG_INF = -std::numeric_limits::infinity(); - - std::unordered_set allowed_set(allowed_tokens.begin(), allowed_tokens.end()); - - for (int i = 0; i < logits.size(); i++) { - if (allowed_set.find(i) == allowed_set.end()) { - logits[i] = static_cast(NEG_INF); - } - } -} - -StreamResult GPT_OSS::parse_stream_content(const std::string content) { - //header_print("GPTOSSHERE", content); - - const std::string MARKER_REASONING = "<|start|>assistant<|channel|>analysis<|message|>"; - const std::string MARKER_NORMAL = "<|start|>assistant<|channel|>final<|message|>"; - const std::string MARKER_END = "<|end|>"; - - // Markers for Tool Calling - const std::string MARKER_TOOL_START = "<|start|>assistant<|channel|>commentary to=functions."; - //const std::string MARKER_TOOL_START = "<|start|>assistant<|channel|>commentary <|constrain|>functions."; - const std::string MARKER_TOOL_SPLIT = "<|message|>"; - const std::string MARKER_TOOL_END = "<|end|>"; - - StreamResult result; - buffer_ += content; // Append new chunk to buffer - - // for test - //result.content = content; - //result.type = current_mode_; - //return result; - // for test - - while (true) { - if (waiting_for_header_) { - size_t pos_reason = buffer_.find(MARKER_REASONING); - size_t pos_normal = buffer_.find(MARKER_NORMAL); - size_t pos_tool = buffer_.find(MARKER_TOOL_START); - - // Find which marker appears first - size_t pos_first = std::string::npos; - StreamEventType next_mode = StreamEventType::REASONING; // default - size_t marker_len = 0; - - if (pos_reason != std::string::npos) { - pos_first = pos_reason; - next_mode = StreamEventType::REASONING; - marker_len = MARKER_REASONING.length(); - } - if (pos_normal != std::string::npos && (pos_first == std::string::npos || pos_normal < pos_first)) { - pos_first = pos_normal; - next_mode = StreamEventType::CONTENT; - marker_len = MARKER_NORMAL.length(); - } - if (pos_tool != std::string::npos && (pos_first == std::string::npos || pos_tool < pos_first)) { - pos_first = pos_tool; - next_mode = StreamEventType::WAITING; // Special mode for buffering tool - marker_len = MARKER_TOOL_START.length(); - } - - // found - if (pos_first != std::string::npos) { - waiting_for_header_ = false; - current_mode_ = next_mode; - - // Remove garbage before the marker and the marker itself - buffer_.erase(0, pos_first + marker_len); - continue; - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - - if (current_mode_ == StreamEventType::WAITING) { - size_t pos_end = buffer_.find(MARKER_TOOL_END); - - if (pos_end != std::string::npos) { - // Format: Name <|constrain|>... <|message|> {JSON} - std::string full_tool_str = buffer_.substr(0, pos_end); - - size_t pos_split = full_tool_str.find(MARKER_TOOL_SPLIT); - if (pos_split != std::string::npos) { - // 1. Extract Name (Left side) - std::string meta_part = full_tool_str.substr(0, pos_split); - // Name is typically the first word - size_t name_end = meta_part.find_first_of(" <"); - result.tool_name = meta_part.substr(0, name_end); - - // 2. Extract JSON (Right side) - result.tool_args_str = full_tool_str.substr(pos_split + MARKER_TOOL_SPLIT.length()); - - // 3. Set Result - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - } - else { - // Fallback if format is wrong - result.content = "[Tool Parse Error]"; - result.type = StreamEventType::CONTENT; - } - - // Clean up and reset - buffer_.erase(0, pos_end + MARKER_TOOL_END.length()); - waiting_for_header_ = true; - - // Return immediately, don't stream rest - return result; - } - else { - // Tool block not finished yet. Wait. - result.type = StreamEventType::WAITING; - return result; - } - } - else { - size_t pos_end = buffer_.find(MARKER_END); - - if (pos_end != std::string::npos) { - // End marker found - result.content += buffer_.substr(0, pos_end); // Output content before marker - result.type = current_mode_; - - // Remove content + marker, ready for next section - buffer_ = buffer_.substr(pos_end + MARKER_END.length()); - waiting_for_header_ = true; // Go back to waiting for next header - } - else { - // No end marker, flush everything immediately - result.content += buffer_; - result.type = current_mode_; - buffer_.clear(); - break; // Done with this chunk - } - } - } - - return result; -} \ No newline at end of file diff --git a/src/common/AutoModel/modeling_lfm2.cpp b/src/common/AutoModel/modeling_lfm2.cpp index 349eaf541..de496e6b2 100644 --- a/src/common/AutoModel/modeling_lfm2.cpp +++ b/src/common/AutoModel/modeling_lfm2.cpp @@ -84,176 +84,6 @@ std::string LFM2::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_i return this->generate(meta_info, length_limit, os); } -StreamResult LFM2::parse_stream_content(const std::string content) { - const std::string TOOL_START = "<|tool_call_start|>"; - const std::string TOOL_END = "<|tool_call_end|>"; - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(TOOL_START); - - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - tool_name_.clear(); - buffer_ = buffer_.substr(TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - else { - if (!buffer_.empty()) { - result.content = buffer_; - result.type = StreamEventType::CONTENT; - buffer_.clear(); - return result; - } - break; - } - } - - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(TOOL_END); - - if (tool_end_pos != std::string::npos) { - tool_name_ += buffer_.substr(0, tool_end_pos); - buffer_ = buffer_.substr(tool_end_pos + TOOL_END.length()); - is_in_tool_block_ = false; - - std::string raw = tool_name_; - - size_t start = raw.find_first_not_of(" \t\n\r["); - if (start == std::string::npos) { - result.type = StreamEventType::CONTENT; - result.content = "[Error: Empty tool call]"; - return result; - } - raw = raw.substr(start); - - size_t pos_open = raw.find('('); - if (pos_open == std::string::npos) { - result.type = StreamEventType::CONTENT; - result.content = "[Error: No function bracket found]"; - return result; - } - - std::string fn_name = raw.substr(0, pos_open); - size_t name_end = fn_name.find_last_not_of(" \t\n\r"); - fn_name = fn_name.substr(0, name_end + 1); - - int paren_count = 0; - size_t pos_close = pos_open; - for (size_t i = pos_open; i < raw.length(); ++i) { - if (raw[i] == '(') paren_count++; - if (raw[i] == ')') { - paren_count--; - if (paren_count == 0) { - pos_close = i; - break; - } - } - } - - std::string args_part = raw.substr(pos_open + 1, pos_close - pos_open - 1); - - json args_json = json::object(); - - size_t pos = 0; - while (pos < args_part.length()) { - while (pos < args_part.length() && - (args_part[pos] == ' ' || args_part[pos] == ',' || - args_part[pos] == '\t' || args_part[pos] == '\n')) { - pos++; - } - if (pos >= args_part.length()) break; - - size_t eq_pos = args_part.find('=', pos); - if (eq_pos == std::string::npos) break; - - std::string key = args_part.substr(pos, eq_pos - pos); - size_t key_start = key.find_first_not_of(" \t\n\r"); - size_t key_end = key.find_last_not_of(" \t\n\r"); - if (key_start != std::string::npos) { - key = key.substr(key_start, key_end - key_start + 1); - } - - pos = eq_pos + 1; - while (pos < args_part.length() && - (args_part[pos] == ' ' || args_part[pos] == '\t')) { - pos++; - } - - std::string value; - if (pos < args_part.length() && args_part[pos] == '"') { - pos++; - size_t value_end = pos; - while (value_end < args_part.length() && args_part[value_end] != '"') { - if (args_part[value_end] == '\\') value_end++; - value_end++; - } - value = args_part.substr(pos, value_end - pos); - pos = value_end + 1; - args_json[key] = value; - } - else { - size_t value_end = pos; - while (value_end < args_part.length() && - args_part[value_end] != ',' && args_part[value_end] != ')') { - value_end++; - } - value = args_part.substr(pos, value_end - pos); - size_t v_start = value.find_first_not_of(" \t\n\r"); - size_t v_end = value.find_last_not_of(" \t\n\r"); - if (v_start != std::string::npos) { - value = value.substr(v_start, v_end - v_start + 1); - } - args_json[key] = value; - pos = value_end; - } - } - - size_t next_tool_start = pos_close + 1; - while (next_tool_start < raw.length() && - (raw[next_tool_start] == ',' || raw[next_tool_start] == ' ' || - raw[next_tool_start] == '\t' || raw[next_tool_start] == '\n' || - raw[next_tool_start] == ']')) { - next_tool_start++; - } - - if (next_tool_start < raw.length()) { - buffer_ = TOOL_START + "[" + raw.substr(next_tool_start) + "]" + TOOL_END + buffer_; - } - - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - result.tool_name = fn_name; - result.tool_args_str = args_json.dump(); - - return result; - - } - else { - tool_name_ += buffer_; - buffer_.clear(); - result.type = StreamEventType::WAITING; - return result; - } - } - } - - result.type = StreamEventType::WAITING; - return result; -} - - /*********** LFM2_5_TK family ***********/ LFM2_5_TK::LFM2_5_TK(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "LFM2_5_TK") {} @@ -358,6 +188,7 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, think_start_id); this->lm_engine->forward(think_start_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -367,11 +198,12 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s // \n this->token_history.push_back(708); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, 708); buffer y = this->lm_engine->forward(708); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(708); result += token_str; - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; this->total_tokens++; @@ -389,11 +221,6 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -401,7 +228,7 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -441,238 +268,3 @@ std::string LFM2_5_TK::generate_with_prompt(chat_meta_info_t& meta_info, lm_unif } return this->generate(meta_info, length_limit, os); } - -NonStreamResult LFM2_5_TK::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string content, reasoning_content; - - std::string think_start_tag = ""; - std::string think_end_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - - - think_start_pos += think_start_tag.length(); - std::string reasoning_str = response_text.substr(think_start_pos, think_end_pos - think_start_pos); - result.reasoning_content = reasoning_str; - - std::string content_str = response_text.substr(think_end_pos + think_end_tag.length()); - result.content = content_str; - - return result; -} - -StreamResult LFM2_5_TK::parse_stream_content(const std::string content) { - const std::string TOOL_START = "<|tool_call_start|>"; - const std::string TOOL_END = "<|tool_call_end|>"; - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(TOOL_START); - - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - tool_name_.clear(); - buffer_ = buffer_.substr(TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - //else { - // if (!buffer_.empty()) { - // result.content = buffer_; - // result.type = StreamEventType::CONTENT; - // buffer_.clear(); - // return result; - // } - // break; - //} - } - - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(TOOL_END); - - if (tool_end_pos != std::string::npos) { - tool_name_ += buffer_.substr(0, tool_end_pos); - buffer_ = buffer_.substr(tool_end_pos + TOOL_END.length()); - is_in_tool_block_ = false; - - std::string raw = tool_name_; - - size_t start = raw.find_first_not_of(" \t\n\r["); - if (start == std::string::npos) { - result.type = StreamEventType::CONTENT; - result.content = "[Error: Empty tool call]"; - return result; - } - raw = raw.substr(start); - - size_t pos_open = raw.find('('); - if (pos_open == std::string::npos) { - result.type = StreamEventType::CONTENT; - result.content = "[Error: No function bracket found]"; - return result; - } - - std::string fn_name = raw.substr(0, pos_open); - size_t name_end = fn_name.find_last_not_of(" \t\n\r"); - fn_name = fn_name.substr(0, name_end + 1); - - int paren_count = 0; - size_t pos_close = pos_open; - for (size_t i = pos_open; i < raw.length(); ++i) { - if (raw[i] == '(') paren_count++; - if (raw[i] == ')') { - paren_count--; - if (paren_count == 0) { - pos_close = i; - break; - } - } - } - - std::string args_part = raw.substr(pos_open + 1, pos_close - pos_open - 1); - - json args_json = json::object(); - - size_t pos = 0; - while (pos < args_part.length()) { - while (pos < args_part.length() && - (args_part[pos] == ' ' || args_part[pos] == ',' || - args_part[pos] == '\t' || args_part[pos] == '\n')) { - pos++; - } - if (pos >= args_part.length()) break; - - size_t eq_pos = args_part.find('=', pos); - if (eq_pos == std::string::npos) break; - - std::string key = args_part.substr(pos, eq_pos - pos); - size_t key_start = key.find_first_not_of(" \t\n\r"); - size_t key_end = key.find_last_not_of(" \t\n\r"); - if (key_start != std::string::npos) { - key = key.substr(key_start, key_end - key_start + 1); - } - - pos = eq_pos + 1; - while (pos < args_part.length() && - (args_part[pos] == ' ' || args_part[pos] == '\t')) { - pos++; - } - - std::string value; - if (pos < args_part.length() && args_part[pos] == '"') { - pos++; - size_t value_end = pos; - while (value_end < args_part.length() && args_part[value_end] != '"') { - if (args_part[value_end] == '\\') value_end++; - value_end++; - } - value = args_part.substr(pos, value_end - pos); - pos = value_end + 1; - args_json[key] = value; - } - else { - size_t value_end = pos; - while (value_end < args_part.length() && - args_part[value_end] != ',' && args_part[value_end] != ')') { - value_end++; - } - value = args_part.substr(pos, value_end - pos); - size_t v_start = value.find_first_not_of(" \t\n\r"); - size_t v_end = value.find_last_not_of(" \t\n\r"); - if (v_start != std::string::npos) { - value = value.substr(v_start, v_end - v_start + 1); - } - args_json[key] = value; - pos = value_end; - } - } - - size_t next_tool_start = pos_close + 1; - while (next_tool_start < raw.length() && - (raw[next_tool_start] == ',' || raw[next_tool_start] == ' ' || - raw[next_tool_start] == '\t' || raw[next_tool_start] == '\n' || - raw[next_tool_start] == ']')) { - next_tool_start++; - } - - if (next_tool_start < raw.length()) { - buffer_ = TOOL_START + "[" + raw.substr(next_tool_start) + "]" + TOOL_END + buffer_; - } - - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - result.tool_name = fn_name; - result.tool_args_str = args_json.dump(); - - return result; - - } - else { - tool_name_ += buffer_; - buffer_.clear(); - result.type = StreamEventType::WAITING; - return result; - } - } - - // normal and reasoning - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - - } - - result.type = current_mode_; - return result; -} \ No newline at end of file diff --git a/src/common/AutoModel/modeling_llama3.cpp b/src/common/AutoModel/modeling_llama3.cpp index e9656f565..0de60286e 100644 --- a/src/common/AutoModel/modeling_llama3.cpp +++ b/src/common/AutoModel/modeling_llama3.cpp @@ -184,10 +184,6 @@ std::string DeepSeek_r1_8b::generate(chat_meta_info_t& meta_info, int length_lim while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -195,7 +191,7 @@ std::string DeepSeek_r1_8b::generate(chat_meta_info_t& meta_info, int length_lim this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -236,74 +232,3 @@ std::string DeepSeek_r1_8b::generate_with_prompt(chat_meta_info_t& meta_info, lm } return this->generate(meta_info, length_limit, os); } - -NonStreamResult DeepSeek_r1_8b::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string content, reasoning_content; - - std::string think_start_tag = ""; - std::string think_end_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - - - think_start_pos += think_start_tag.length(); - std::string reasoning_str = response_text.substr(think_start_pos, think_end_pos - think_start_pos); - result.reasoning_content = reasoning_str; - - std::string content_str = response_text.substr(think_end_pos + think_end_tag.length()); - result.content = content_str; - - return result; -} - -StreamResult DeepSeek_r1_8b::parse_stream_content(const std::string content) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - - StreamResult result; - buffer_ += content; - - while (true) { - if (current_mode_ == StreamEventType::CONTENT) { - // Check for the start of a thought block - size_t pos = buffer_.find(MARKER_THINK_START); - - if (pos != std::string::npos) { - // Emit content before the tag - result.content += buffer_.substr(0, pos); - result.type = StreamEventType::CONTENT; - - // Remove "\n" and switch mode - buffer_ = buffer_.substr(pos + MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - // Check for the end of a thought block - size_t pos = buffer_.find(MARKER_THINK_END); - - if (pos != std::string::npos) { - // Emit content before the tag - result.content += buffer_.substr(0, pos); - result.type = StreamEventType::REASONING; - - // Remove "\n" and switch mode - buffer_ = buffer_.substr(pos + MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - // Flush remaining buffer - result.content += buffer_; - result.type = current_mode_; - buffer_.clear(); - break; - } - - return result; -} \ No newline at end of file diff --git a/src/common/AutoModel/modeling_nanbeige.cpp b/src/common/AutoModel/modeling_nanbeige.cpp index 7eeac2a07..d1286e9a5 100644 --- a/src/common/AutoModel/modeling_nanbeige.cpp +++ b/src/common/AutoModel/modeling_nanbeige.cpp @@ -128,11 +128,6 @@ std::string Nanbeige::generate(chat_meta_info_t& meta_info, int length_limit, st while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -140,7 +135,7 @@ std::string Nanbeige::generate(chat_meta_info_t& meta_info, int length_limit, st this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -212,7 +207,7 @@ std::string Nanbeige::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifo this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -242,193 +237,3 @@ std::string Nanbeige::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifo } return result; } - -NonStreamResult Nanbeige::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string name, arguments; - std::string content, reasoning_content; - - std::string think_start_tag = ""; - std::string think_end_tag = ""; - std::string tool_start_tag = ""; - std::string tool_end_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - size_t tool_start_pos = response_text.find(tool_start_tag); - size_t tool_end_pos = response_text.find(tool_end_tag); - - bool is_reasoning = !(think_start_pos == std::string::npos || think_end_pos == std::string::npos); - bool is_tool = !(tool_start_pos == std::string::npos || tool_end_pos == std::string::npos); - bool is_content = !is_tool; - - if (is_reasoning) { - // Find reasoning part - think_start_pos += think_start_tag.length(); - std::string reasoning_str = response_text.substr(think_start_pos, think_end_pos - think_start_pos); - result.reasoning_content = reasoning_str; - } - - if (is_tool) { - // Find tool calling part - tool_start_pos += tool_start_tag.length(); - std::string json_str = response_text.substr(tool_start_pos, tool_end_pos - tool_start_pos); - // Parse "name" - std::string key_name = "\"name\": \""; - size_t name_start = json_str.find(key_name); - if (name_start != std::string::npos) { - name_start += key_name.length(); - size_t name_end = json_str.find("\"", name_start); - if (name_end != std::string::npos) { - name = json_str.substr(name_start, name_end - name_start); - } - } - // Parse "arguments" - std::string key_args = "\"arguments\":"; - size_t args_pos = json_str.find(key_args); - if (args_pos != std::string::npos) { - size_t brace_start = json_str.find("{", args_pos); - size_t brace_end = json_str.rfind("}"); // Find the last closing brace - - if (brace_start != std::string::npos && brace_end != std::string::npos && brace_end > brace_start) { - arguments = json_str.substr(brace_start, brace_end - brace_start); - } - } - - result.tool_name = name; - result.tool_args = arguments; - - } - else if (is_content) { - std::string content_str = response_text.substr(think_end_pos + think_end_tag.length()); - result.content = content_str; - } - - return result; -} - - -StreamResult Nanbeige::parse_stream_content(const std::string content) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - - StreamResult result; - buffer_ += content; - - while (true) { - // Keep data in buffer_ and wait until a complete TOOL_END is found. Never clear buffer_ midway. - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - if (tool_end_pos != std::string::npos) { - std::string tool_content = buffer_.substr(0, tool_end_pos); - buffer_ = buffer_.substr(tool_end_pos + MARKER_TOOL_END.length()); - is_in_tool_block_ = false; - - try { - auto j = nlohmann::json::parse(tool_content); - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - if (j.contains("name")) { - result.tool_name = j["name"].get(); - } - if (j.contains("arguments")) { - if (j["arguments"].is_string()) { - result.tool_args_str = j["arguments"].get(); - } else { - result.tool_args_str = j["arguments"].dump(); - } - } - return result; - } catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } else { - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool start - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - continue; - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - // 4. Safe Flush mechanism: Handle fragmented tag headers - // E.g., if the current chunk ends with " active_markers; - if (current_mode_ == StreamEventType::CONTENT) { - active_markers.push_back(MARKER_THINK_START); - active_markers.push_back(MARKER_TOOL_START); - } else if (current_mode_ == StreamEventType::REASONING) { - active_markers.push_back(MARKER_THINK_END); - active_markers.push_back(MARKER_TOOL_START); - } - - size_t safe_flush_len = buffer_.length(); - for (const auto& marker : active_markers) { - for (size_t i = 1; i <= marker.length() && i <= buffer_.length(); ++i) { - if (buffer_.compare(buffer_.length() - i, i, marker, 0, i) == 0) { - safe_flush_len = std::min(safe_flush_len, buffer_.length() - i); - } - } - } - - if (safe_flush_len > 0) { - result.content = buffer_.substr(0, safe_flush_len); - result.type = current_mode_; - buffer_ = buffer_.substr(safe_flush_len); - return result; - } else if (buffer_.length() > 0) { - result.type = StreamEventType::WAITING; - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index 021a3062e..acd384bac 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -89,6 +89,7 @@ void Phi4::setup_tokenizer(const nlohmann::json& config) { chat_tmpl = std::move(chat); user_system_prompt.clear(); extra_context["user_system_prompt"] = user_system_prompt; + _detect_chat_format(config["chat_template"].get()); } std::string Phi4::apply_chat_template(nlohmann::ordered_json& messages, diff --git a/src/common/AutoModel/modeling_qwen3.cpp b/src/common/AutoModel/modeling_qwen3.cpp index d308d1178..d03095068 100644 --- a/src/common/AutoModel/modeling_qwen3.cpp +++ b/src/common/AutoModel/modeling_qwen3.cpp @@ -144,6 +144,8 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os return result; } this->token_history.push_back(think_start_id); + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); this->lm_engine->forward(think_start_id); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -165,7 +167,7 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os buffer y = this->lm_engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); // All four went through the engine, so all four occupy the context. this->total_tokens += 4; @@ -197,11 +199,6 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os while (this->total_tokens < decode_cap){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -209,7 +206,7 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -253,89 +250,6 @@ std::string Qwen3::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_ return this->_shared_generate(meta_info, length_limit, os); } -static void qwen3_parse_tool_blocks(const std::string& response_text, NonStreamResult& result) { - const std::string tool_start_tag = ""; - const std::string tool_end_tag = ""; - - size_t search_from = 0; - while (true) { - size_t start_pos = response_text.find(tool_start_tag, search_from); - if (start_pos == std::string::npos) break; - - size_t block_content_start = start_pos + tool_start_tag.length(); - size_t end_pos = response_text.find(tool_end_tag, block_content_start); - size_t block_end = (end_pos != std::string::npos) ? end_pos : response_text.length(); - search_from = (end_pos != std::string::npos) ? end_pos + tool_end_tag.length() : block_end; - - std::string json_str = response_text.substr(block_content_start, block_end - block_content_start); - - std::string name, arguments; - try { - auto j = nlohmann::json::parse(json_str); - if (j.contains("name")) name = j["name"].get(); - if (j.contains("arguments")) { - arguments = j["arguments"].is_string() - ? j["arguments"].get() - : j["arguments"].dump(); - } - } catch (...) { - // fallback: manual string extraction - std::string key_name = "\"name\": \""; - size_t ns = json_str.find(key_name); - if (ns != std::string::npos) { - ns += key_name.length(); - size_t ne = json_str.find("\"", ns); - if (ne != std::string::npos) name = json_str.substr(ns, ne - ns); - } - std::string key_args = "\"arguments\":"; - size_t ap = json_str.find(key_args); - if (ap != std::string::npos) { - size_t bs = json_str.find("{", ap); - size_t be = json_str.rfind("}"); - if (bs != std::string::npos && be != std::string::npos && be > bs) - arguments = json_str.substr(bs, be - bs + 1); - } - } - - result.tool_calls_list.emplace_back(name, arguments); - } - - if (!result.tool_calls_list.empty()) { - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - } -} - -NonStreamResult Qwen3::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - const std::string think_start_tag = ""; - const std::string think_end_tag = ""; - const std::string tool_start_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - bool is_reasoning = (think_start_pos != std::string::npos && think_end_pos != std::string::npos); - - if (is_reasoning) { - size_t start = think_start_pos + think_start_tag.length(); - result.reasoning_content = response_text.substr(start, think_end_pos - start); - } - - qwen3_parse_tool_blocks(response_text, result); - - if (result.tool_calls_list.empty()) { - size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; - result.content = response_text.substr(content_start); - } - - return result; -} - -StreamResult Qwen3::parse_stream_content(const std::string content) { - return _shared_think_tool_calling_pasrsed(content); -} - /************ Qwen3_IT family **************/ Qwen3_IT::Qwen3_IT(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} @@ -407,74 +321,6 @@ std::string Qwen3_IT::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifo return this->_shared_generate(meta_info, length_limit, os); } -// Non-stream -NonStreamResult Qwen3_IT::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - qwen3_parse_tool_blocks(response_text, result); - - if (result.tool_calls_list.empty()) { - result.content = response_text; - } - - return result; -} - -// Stream -StreamResult Qwen3_IT::parse_stream_content(const std::string content) { - std::string tool_start_tag = ""; - std::string tool_end_tag = ""; - - StreamResult result; - result.type = StreamEventType::CONTENT; - - if (content.find(tool_start_tag) != std::string::npos) { - is_in_tool_block_ = true; - tool_name_.clear(); - result.type = StreamEventType::WAITING; - return result; - } - - if (content.find("") != std::string::npos) { - is_in_tool_block_ = false; - - try { - auto j = nlohmann::json::parse(tool_name_); - - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - if (j.contains("name")) { - result.tool_name = j["name"].get(); - } - - if (j.contains("arguments")) { - if (j["arguments"].is_string()) { - result.tool_args_str = j["arguments"].get(); - } - else { - result.tool_args_str = j["arguments"].dump(); - } - } - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - } - return result; - } - - if (is_in_tool_block_) { - tool_name_ += content; - result.type = StreamEventType::WAITING; - return result; - } - - result.content = content; - return result; - -} - /************ Qwen3_TK family **************/ Qwen3_TK::Qwen3_TK(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} @@ -571,6 +417,7 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, think_start_id); this->lm_engine->forward(think_start_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -580,11 +427,12 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st // \n this->token_history.push_back(198); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, 198); buffer y = this->lm_engine->forward(198); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(198); result += token_str; - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; this->total_tokens++; @@ -603,11 +451,6 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -615,7 +458,7 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -656,36 +499,6 @@ std::string Qwen3_TK::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifo return this->generate(meta_info, length_limit, os); } -NonStreamResult Qwen3_TK::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - const std::string think_start_tag = ""; - const std::string think_end_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - bool is_reasoning = (think_start_pos != std::string::npos && think_end_pos != std::string::npos); - - if (is_reasoning) { - size_t start = think_start_pos + think_start_tag.length(); - result.reasoning_content = response_text.substr(start, think_end_pos - start); - } - - qwen3_parse_tool_blocks(response_text, result); - - if (result.tool_calls_list.empty()) { - size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; - result.content = response_text.substr(content_start); - } - - return result; -} - - -StreamResult Qwen3_TK::parse_stream_content(const std::string content) { - return _shared_think_tool_calling_pasrsed(content); -} - /************ DeepSeek_r1_0528_8b family **************/ DeepSeek_r1_0528_8b::DeepSeek_r1_0528_8b(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} @@ -792,11 +605,6 @@ std::string DeepSeek_r1_0528_8b::generate(chat_meta_info_t& meta_info, int lengt while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -804,7 +612,7 @@ std::string DeepSeek_r1_0528_8b::generate(chat_meta_info_t& meta_info, int lengt this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -844,75 +652,3 @@ std::string DeepSeek_r1_0528_8b::generate_with_prompt(chat_meta_info_t& meta_inf } return this->_shared_generate(meta_info, length_limit, os); } - -NonStreamResult DeepSeek_r1_0528_8b::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string content, reasoning_content; - - std::string think_start_tag = ""; - std::string think_end_tag = ""; - - size_t think_start_pos = response_text.find(think_start_tag); - size_t think_end_pos = response_text.find(think_end_tag); - - - think_start_pos += think_start_tag.length(); - std::string reasoning_str = response_text.substr(think_start_pos, think_end_pos - think_start_pos); - result.reasoning_content = reasoning_str; - - std::string content_str = response_text.substr(think_end_pos + think_end_tag.length()); - result.content = content_str; - - return result; -} - - -StreamResult DeepSeek_r1_0528_8b::parse_stream_content(const std::string content) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - - StreamResult result; - buffer_ += content; - - while (true) { - if (current_mode_ == StreamEventType::CONTENT) { - // Check for the start of a thought block - size_t pos = buffer_.find(MARKER_THINK_START); - - if (pos != std::string::npos) { - // Emit content before the tag - result.content += buffer_.substr(0, pos); - result.type = StreamEventType::CONTENT; - - // Remove "\n" and switch mode - buffer_ = buffer_.substr(pos + MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - // Check for the end of a thought block - size_t pos = buffer_.find(MARKER_THINK_END); - - if (pos != std::string::npos) { - // Emit content before the tag - result.content += buffer_.substr(0, pos); - result.type = StreamEventType::REASONING; - - // Remove "\n" and switch mode - buffer_ = buffer_.substr(pos + MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - // Flush remaining buffer - result.content += buffer_; - result.type = current_mode_; - buffer_.clear(); - break; - } - - return result; -} \ No newline at end of file diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index bccee9c3e..bf4e364b1 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -61,6 +61,7 @@ void Qwen3_5_Omni::setup_tokenizer(std::string model_path) { } this->user_system_prompt = ""; this->extra_context["user_system_prompt"] = this->user_system_prompt; + this->_detect_chat_format(tokenizer_config["chat_template"].get()); } void Qwen3_5_Omni::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption, const std::string& backend) { @@ -474,6 +475,7 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input size_t n = tokens.size(); tokens.resize(n - 4); + this->_begin_turn(meta_info, tokens); // prefix check for tokens and token history to see if we can skip some tokens const size_t idx = this->token_history.size(); @@ -531,11 +533,6 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input for (int i = 0; i < chunks; i++) { if (is_cancelled()) { meta_info.stop_reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } int start = i * max_prefill_len; @@ -561,7 +558,8 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input this->total_tokens += tokens.size(); this->profiler_list[SAMPLING_TIME].start(); - this->last_token = this->sampler->sample(this->last_thinker_result.logits); + this->last_token = this->_sample(this->last_thinker_result.logits, meta_info); + meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); this->checkpoint_his = this->token_history; @@ -590,6 +588,8 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit // manually decoding \n <\think> \n { this->token_history.push_back(think_start_id); + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); this->engine->forward(think_start_id); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -611,7 +611,7 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit this->last_thinker_result = this->engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); - sampled_token = this->sampler->sample(this->last_thinker_result.logits); + sampled_token = this->_sample(this->last_thinker_result.logits, meta_info); this->total_tokens++; meta_info.generated_tokens++; @@ -630,11 +630,6 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit while (this->total_tokens < this->MAX_L) { if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -642,7 +637,7 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - int sampled_token = this->sampler->sample(this->last_thinker_result.logits); + int sampled_token = this->_sample(this->last_thinker_result.logits, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -757,339 +752,3 @@ std::pair> Qwen3_5_Omni::get_history() { std::string all_context = this->tokenizer->decode(history); return std::make_pair(all_context, history); } - -// Non-stream -NonStreamResult Qwen3_5_Omni::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string start_tag = ""; - std::string end_tag = ""; - std::string func_end_tag = ""; - std::string func_open = " fallback - size_t func_end_pos = response_text.find(func_end_tag, block_content_start); - if (func_end_pos != std::string::npos) { - block_end = func_end_pos + func_end_tag.length(); - } else { - block_end = response_text.length(); - } - search_from = block_end; - } - - std::string block = response_text.substr(block_content_start, block_end - block_content_start); - - std::string tool_name; - size_t func_start = block.find(func_open); - if (func_start != std::string::npos) { - func_start += func_open.length(); - size_t func_name_end = block.find(">", func_start); - if (func_name_end != std::string::npos) { - tool_name = block.substr(func_start, func_name_end - func_start); - } - } - - nlohmann::json args = nlohmann::json::object(); - size_t pos = 0; - - while (true) { - size_t param_start = block.find(param_open, pos); - if (param_start == std::string::npos) break; - - param_start += param_open.length(); - size_t param_name_end = block.find(">", param_start); - if (param_name_end == std::string::npos) break; - - std::string param_name = block.substr(param_start, param_name_end - param_start); - size_t value_start = param_name_end + 1; - size_t value_end = block.find(param_close, value_start); - - size_t next_param_pos = block.find(param_open, value_start); - size_t func_boundary_pos = block.find(func_end_tag, value_start); - - auto use_earlier_boundary = [&value_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { - value_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - - if (value_end == std::string::npos) { - value_end = block.length(); - } - - std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); - - try { - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - pos = value_end; - if (block.compare(value_end, param_close.length(), param_close) == 0) { - pos += param_close.length(); - } - } - - result.tool_calls_list.emplace_back(tool_name, args.dump()); - } - - if (result.tool_calls_list.empty()) { - result.content = response_text; - } else { - // Populate legacy single-tool fields from the first call for backward compatibility - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - // Extract content before the first - size_t first_tool = response_text.find(start_tag); - if (first_tool != std::string::npos && first_tool > 0) { - result.content = trim_tool_value(response_text.substr(0, first_tool)); - } - } - - return result; -} - -// Stream -StreamResult Qwen3_5_Omni::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Qwen3_5_Omni::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Qwen3_5_Omni::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_FUNC_END = ""; - - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); - if (stray_end_pos != std::string::npos) { - buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); - } - } - - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t func_end_pos = buffer_.find(MARKER_FUNC_END); - - if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - else if (func_end_pos != std::string::npos) { - actual_end_pos = func_end_pos; - skip_length = MARKER_FUNC_END.length(); - } - - std::string block = buffer_.substr(0, actual_end_pos + skip_length); - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - try { - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - // parse function name - std::string func_open = "", func_start); - if (func_end != std::string::npos) { - result.tool_name = block.substr(func_start, func_end - func_start); - } - } - - // parse parameters - nlohmann::json args = nlohmann::json::object(); - std::string param_open = "", p_start); - if (p_name_end == std::string::npos) break; - std::string param_name = block.substr(p_start, p_name_end - p_start); - - size_t val_start = p_name_end + 1; - if (val_start < block.size() && block[val_start] == '\n') val_start++; - - size_t param_close_pos = block.find(param_close, val_start); - size_t val_end = param_close_pos; - - size_t next_param_pos = block.find(param_open, val_start); - size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); - size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); - - auto use_earlier_boundary = [&val_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { - val_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - use_earlier_boundary(tool_boundary_pos); - - if (val_end == std::string::npos && is_final) { - val_end = block.size(); - } - if (val_end == std::string::npos) break; - - std::string param_value = block.substr(val_start, val_end - val_start); - - // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter - while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { - param_value.pop_back(); - } - - try { - // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - search_pos = param_close_pos != std::string::npos && val_end == param_close_pos - ? val_end + param_close.length() - : val_end; - } - result.tool_args_str = args.dump(); - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - size_t last_lt = buffer_.rfind('<'); - // If '<' appears at the end (possibly an incomplete or tag) - if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { - if (last_lt > 0) { - // Only output the content before '<' - result.content = buffer_.substr(0, last_lt); - result.type = current_mode_; - buffer_ = buffer_.substr(last_lt); - return result; - } else { - // If '<' is the first character in the buffer, directly wait for the next chunk - result.type = StreamEventType::WAITING; - return result; - } - } - - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} \ No newline at end of file diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index a09f09731..ab3b31dba 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -319,6 +319,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, think_start_id); this->lm_engine->forward(think_start_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -328,17 +329,19 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s // \n this->token_history.push_back(198); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, 198); buffer y = this->lm_engine->forward(198); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(198); result += token_str; // First token of the reasoning block. - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; } else{ this->token_history.push_back(think_start_id); + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); this->lm_engine->forward(think_start_id); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -360,10 +363,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s buffer y = this->lm_engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); - // First real token after the forced empty think block -- the model would - // open a tool call right here, so it needs the mask too. - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); } this->total_tokens++; meta_info.generated_tokens++; @@ -380,11 +380,6 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -392,10 +387,7 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - // Qwen3_5VL runs its own decode loop instead of _shared_generate, so the - // tool_choice mask has to be applied here as well as in _shared_insert. - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -444,339 +436,3 @@ std::string Qwen3_5VL::generate_with_prompt(chat_meta_info_t& meta_info, lm_unif } return this->_shared_generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult Qwen3_5VL::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string start_tag = ""; - std::string end_tag = ""; - std::string func_end_tag = ""; - std::string func_open = " fallback - size_t func_end_pos = response_text.find(func_end_tag, block_content_start); - if (func_end_pos != std::string::npos) { - block_end = func_end_pos + func_end_tag.length(); - } else { - block_end = response_text.length(); - } - search_from = block_end; - } - - std::string block = response_text.substr(block_content_start, block_end - block_content_start); - - std::string tool_name; - size_t func_start = block.find(func_open); - if (func_start != std::string::npos) { - func_start += func_open.length(); - size_t func_name_end = block.find(">", func_start); - if (func_name_end != std::string::npos) { - tool_name = block.substr(func_start, func_name_end - func_start); - } - } - - nlohmann::json args = nlohmann::json::object(); - size_t pos = 0; - - while (true) { - size_t param_start = block.find(param_open, pos); - if (param_start == std::string::npos) break; - - param_start += param_open.length(); - size_t param_name_end = block.find(">", param_start); - if (param_name_end == std::string::npos) break; - - std::string param_name = block.substr(param_start, param_name_end - param_start); - size_t value_start = param_name_end + 1; - size_t value_end = block.find(param_close, value_start); - - size_t next_param_pos = block.find(param_open, value_start); - size_t func_boundary_pos = block.find(func_end_tag, value_start); - - auto use_earlier_boundary = [&value_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { - value_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - - if (value_end == std::string::npos) { - value_end = block.length(); - } - - std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); - - try { - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - pos = value_end; - if (block.compare(value_end, param_close.length(), param_close) == 0) { - pos += param_close.length(); - } - } - - result.tool_calls_list.emplace_back(tool_name, args.dump()); - } - - if (result.tool_calls_list.empty()) { - result.content = response_text; - } else { - // Populate legacy single-tool fields from the first call for backward compatibility - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - // Extract content before the first - size_t first_tool = response_text.find(start_tag); - if (first_tool != std::string::npos && first_tool > 0) { - result.content = trim_tool_value(response_text.substr(0, first_tool)); - } - } - - return result; -} - -// Stream -StreamResult Qwen3_5VL::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Qwen3_5VL::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Qwen3_5VL::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_FUNC_END = ""; - - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); - if (stray_end_pos != std::string::npos) { - buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); - } - } - - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t func_end_pos = buffer_.find(MARKER_FUNC_END); - - if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - else if (func_end_pos != std::string::npos) { - actual_end_pos = func_end_pos; - skip_length = MARKER_FUNC_END.length(); - } - - std::string block = buffer_.substr(0, actual_end_pos + skip_length); - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - try { - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - // parse function name - std::string func_open = "", func_start); - if (func_end != std::string::npos) { - result.tool_name = block.substr(func_start, func_end - func_start); - } - } - - // parse parameters - nlohmann::json args = nlohmann::json::object(); - std::string param_open = "", p_start); - if (p_name_end == std::string::npos) break; - std::string param_name = block.substr(p_start, p_name_end - p_start); - - size_t val_start = p_name_end + 1; - if (val_start < block.size() && block[val_start] == '\n') val_start++; - - size_t param_close_pos = block.find(param_close, val_start); - size_t val_end = param_close_pos; - - size_t next_param_pos = block.find(param_open, val_start); - size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); - size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); - - auto use_earlier_boundary = [&val_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { - val_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - use_earlier_boundary(tool_boundary_pos); - - if (val_end == std::string::npos && is_final) { - val_end = block.size(); - } - if (val_end == std::string::npos) break; - - std::string param_value = block.substr(val_start, val_end - val_start); - - // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter - while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { - param_value.pop_back(); - } - - try { - // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - search_pos = param_close_pos != std::string::npos && val_end == param_close_pos - ? val_end + param_close.length() - : val_end; - } - result.tool_args_str = args.dump(); - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - size_t last_lt = buffer_.rfind('<'); - // If '<' appears at the end (possibly an incomplete or tag) - if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { - if (last_lt > 0) { - // Only output the content before '<' - result.content = buffer_.substr(0, last_lt); - result.type = current_mode_; - buffer_ = buffer_.substr(last_lt); - return result; - } else { - // If '<' is the first character in the buffer, directly wait for the next chunk - result.type = StreamEventType::WAITING; - return result; - } - } - - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 4c00fe310..d7accfff1 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -320,6 +320,7 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, if(this->enable_think) { this->token_history.push_back(think_start_id); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, think_start_id); this->lm_engine->forward(think_start_id); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -329,17 +330,19 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, // \n this->token_history.push_back(198); this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, 198); buffer y = this->lm_engine->forward(198); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(198); result += token_str; // First token of the reasoning block. - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; } else{ this->token_history.push_back(think_start_id); + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); this->lm_engine->forward(think_start_id); token_str = this->tokenizer->run_time_decoder(think_start_id); @@ -361,10 +364,7 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, buffer y = this->lm_engine->forward(271); this->profiler_list[DECODING_TIME].stop(1); token_str = this->tokenizer->run_time_decoder(271); - // First real token after the forced empty think block -- the model would - // open a tool call right here, so it needs the mask too. - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); } this->total_tokens++; meta_info.generated_tokens++; @@ -381,11 +381,6 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } this->profiler_list[DECODING_TIME].start(); @@ -393,10 +388,7 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - // Qwen3_6_MOE runs its own decode loop instead of _shared_generate, so the - // tool_choice mask has to be applied here as well as in _shared_insert. - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; last_sampled_token = sampled_token; @@ -446,339 +438,3 @@ std::string Qwen3_6_MOE::generate_with_prompt(chat_meta_info_t& meta_info, lm_un } return this->_shared_generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult Qwen3_6_MOE::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - std::string start_tag = ""; - std::string end_tag = ""; - std::string func_end_tag = ""; - std::string func_open = " fallback - size_t func_end_pos = response_text.find(func_end_tag, block_content_start); - if (func_end_pos != std::string::npos) { - block_end = func_end_pos + func_end_tag.length(); - } else { - block_end = response_text.length(); - } - search_from = block_end; - } - - std::string block = response_text.substr(block_content_start, block_end - block_content_start); - - std::string tool_name; - size_t func_start = block.find(func_open); - if (func_start != std::string::npos) { - func_start += func_open.length(); - size_t func_name_end = block.find(">", func_start); - if (func_name_end != std::string::npos) { - tool_name = block.substr(func_start, func_name_end - func_start); - } - } - - nlohmann::json args = nlohmann::json::object(); - size_t pos = 0; - - while (true) { - size_t param_start = block.find(param_open, pos); - if (param_start == std::string::npos) break; - - param_start += param_open.length(); - size_t param_name_end = block.find(">", param_start); - if (param_name_end == std::string::npos) break; - - std::string param_name = block.substr(param_start, param_name_end - param_start); - size_t value_start = param_name_end + 1; - size_t value_end = block.find(param_close, value_start); - - size_t next_param_pos = block.find(param_open, value_start); - size_t func_boundary_pos = block.find(func_end_tag, value_start); - - auto use_earlier_boundary = [&value_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { - value_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - - if (value_end == std::string::npos) { - value_end = block.length(); - } - - std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); - - try { - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - pos = value_end; - if (block.compare(value_end, param_close.length(), param_close) == 0) { - pos += param_close.length(); - } - } - - result.tool_calls_list.emplace_back(tool_name, args.dump()); - } - - if (result.tool_calls_list.empty()) { - result.content = response_text; - } else { - // Populate legacy single-tool fields from the first call for backward compatibility - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - // Extract content before the first - size_t first_tool = response_text.find(start_tag); - if (first_tool != std::string::npos && first_tool > 0) { - result.content = trim_tool_value(response_text.substr(0, first_tool)); - } - } - - return result; -} - -// Stream -StreamResult Qwen3_6_MOE::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Qwen3_6_MOE::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Qwen3_6_MOE::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_FUNC_END = ""; - - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); - if (stray_end_pos != std::string::npos) { - buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); - } - } - - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t func_end_pos = buffer_.find(MARKER_FUNC_END); - - if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - else if (func_end_pos != std::string::npos) { - actual_end_pos = func_end_pos; - skip_length = MARKER_FUNC_END.length(); - } - - std::string block = buffer_.substr(0, actual_end_pos + skip_length); - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - try { - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - // parse function name - std::string func_open = "", func_start); - if (func_end != std::string::npos) { - result.tool_name = block.substr(func_start, func_end - func_start); - } - } - - // parse parameters - nlohmann::json args = nlohmann::json::object(); - std::string param_open = "", p_start); - if (p_name_end == std::string::npos) break; - std::string param_name = block.substr(p_start, p_name_end - p_start); - - size_t val_start = p_name_end + 1; - if (val_start < block.size() && block[val_start] == '\n') val_start++; - - size_t param_close_pos = block.find(param_close, val_start); - size_t val_end = param_close_pos; - - size_t next_param_pos = block.find(param_open, val_start); - size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); - size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); - - auto use_earlier_boundary = [&val_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { - val_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - use_earlier_boundary(tool_boundary_pos); - - if (val_end == std::string::npos && is_final) { - val_end = block.size(); - } - if (val_end == std::string::npos) break; - - std::string param_value = block.substr(val_start, val_end - val_start); - - // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter - while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { - param_value.pop_back(); - } - - try { - // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - search_pos = param_close_pos != std::string::npos && val_end == param_close_pos - ? val_end + param_close.length() - : val_end; - } - result.tool_args_str = args.dump(); - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - size_t last_lt = buffer_.rfind('<'); - // If '<' appears at the end (possibly an incomplete or tag) - if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { - if (last_lt > 0) { - // Only output the content before '<' - result.content = buffer_.substr(0, last_lt); - result.type = current_mode_; - buffer_ = buffer_.substr(last_lt); - return result; - } else { - // If '<' is the first character in the buffer, directly wait for the next chunk - result.type = StreamEventType::WAITING; - return result; - } - } - - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_qwen3_8mtp.cpp b/src/common/AutoModel/modeling_qwen3_8mtp.cpp index 96155032d..efa20953a 100644 --- a/src/common/AutoModel/modeling_qwen3_8mtp.cpp +++ b/src/common/AutoModel/modeling_qwen3_8mtp.cpp @@ -428,7 +428,9 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l // runs and silently produces greedy output instead. A checkpoint with no // MTP head is unaffected: speculate() just returns {} every step and // everything falls through to the ordinary path below. - const bool spec_enabled = true; + // Off under a decoding constraint: speculate() commits the drafts the base + // model agrees with, so they would never pass the constraint's mask. + const bool spec_enabled = !meta_info.constraint; // Draft depth; also the hit-rate denominator (see report_speculation_stats). const int SPEC_MAX_DRAFT = 7; if (spec_enabled) { @@ -493,11 +495,6 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } @@ -540,8 +537,7 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); if (!consume(sampled_token)) break; @@ -620,353 +616,3 @@ std::string Qwen3_8MTP::generate_with_prompt(chat_meta_info_t& meta_info, lm_uni } return this->_speculative_generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult Qwen3_8MTP::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - // Qwen3.5-style tool syntax: - // VALUE - std::string start_tag = ""; - std::string end_tag = ""; - std::string func_end_tag = ""; - std::string func_open = " fallback - size_t func_end_pos = response_text.find(func_end_tag, block_content_start); - if (func_end_pos != std::string::npos) { - block_end = func_end_pos + func_end_tag.length(); - } else { - block_end = response_text.length(); - } - search_from = block_end; - } - - std::string block = response_text.substr(block_content_start, block_end - block_content_start); - - std::string tool_name; - size_t func_start = block.find(func_open); - if (func_start != std::string::npos) { - func_start += func_open.length(); - size_t func_name_end = block.find(">", func_start); - if (func_name_end != std::string::npos) { - tool_name = block.substr(func_start, func_name_end - func_start); - } - } - - nlohmann::json args = nlohmann::json::object(); - size_t pos = 0; - - while (true) { - size_t param_start = block.find(param_open, pos); - if (param_start == std::string::npos) break; - - param_start += param_open.length(); - size_t param_name_end = block.find(">", param_start); - if (param_name_end == std::string::npos) break; - - std::string param_name = block.substr(param_start, param_name_end - param_start); - size_t value_start = param_name_end + 1; - size_t value_end = block.find(param_close, value_start); - - size_t next_param_pos = block.find(param_open, value_start); - size_t func_boundary_pos = block.find(func_end_tag, value_start); - - auto use_earlier_boundary = [&value_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { - value_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - - if (value_end == std::string::npos) { - value_end = block.length(); - } - - std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); - - try { - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - pos = value_end; - if (block.compare(value_end, param_close.length(), param_close) == 0) { - pos += param_close.length(); - } - } - - result.tool_calls_list.emplace_back(tool_name, args.dump()); - } - - if (result.tool_calls_list.empty()) { - size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; - result.content = response_text.substr(content_start); - } else { - // Populate legacy single-tool fields from the first call for backward compatibility - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - // Extract content before the first - size_t first_tool = response_text.find(start_tag); - size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; - if (first_tool != std::string::npos && first_tool > content_start) { - result.content = trim_tool_value(response_text.substr(content_start, first_tool - content_start)); - } - } - - return result; -} - -// Stream -StreamResult Qwen3_8MTP::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult Qwen3_8MTP::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult Qwen3_8MTP::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_FUNC_END = ""; - - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); - if (stray_end_pos != std::string::npos) { - buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); - } - } - - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t func_end_pos = buffer_.find(MARKER_FUNC_END); - - if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - else if (func_end_pos != std::string::npos) { - actual_end_pos = func_end_pos; - skip_length = MARKER_FUNC_END.length(); - } - - std::string block = buffer_.substr(0, actual_end_pos + skip_length); - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - try { - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - // parse function name - std::string func_open = "", func_start); - if (func_end != std::string::npos) { - result.tool_name = block.substr(func_start, func_end - func_start); - } - } - - // parse parameters - nlohmann::json args = nlohmann::json::object(); - std::string param_open = "", p_start); - if (p_name_end == std::string::npos) break; - std::string param_name = block.substr(p_start, p_name_end - p_start); - - size_t val_start = p_name_end + 1; - if (val_start < block.size() && block[val_start] == '\n') val_start++; - - size_t param_close_pos = block.find(param_close, val_start); - size_t val_end = param_close_pos; - - size_t next_param_pos = block.find(param_open, val_start); - size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); - size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); - - auto use_earlier_boundary = [&val_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { - val_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - use_earlier_boundary(tool_boundary_pos); - - if (val_end == std::string::npos && is_final) { - val_end = block.size(); - } - if (val_end == std::string::npos) break; - - std::string param_value = block.substr(val_start, val_end - val_start); - - // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter - while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { - param_value.pop_back(); - } - - try { - // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - search_pos = param_close_pos != std::string::npos && val_end == param_close_pos - ? val_end + param_close.length() - : val_end; - } - result.tool_args_str = args.dump(); - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - size_t last_lt = buffer_.rfind('<'); - // If '<' appears at the end (possibly an incomplete or tag) - if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { - if (last_lt > 0) { - // Only output the content before '<' - result.content = buffer_.substr(0, last_lt); - result.type = current_mode_; - buffer_ = buffer_.substr(last_lt); - return result; - } else { - // If '<' is the first character in the buffer, directly wait for the next chunk - result.type = StreamEventType::WAITING; - return result; - } - } - - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/common/AutoModel/modeling_qwen3vl.cpp b/src/common/AutoModel/modeling_qwen3vl.cpp index 01ff0d4c6..3dc4895bb 100644 --- a/src/common/AutoModel/modeling_qwen3vl.cpp +++ b/src/common/AutoModel/modeling_qwen3vl.cpp @@ -296,121 +296,6 @@ std::string Qwen3VL::generate_with_prompt(chat_meta_info_t& meta_info, lm_unifor return this->_shared_generate(meta_info, length_limit, os); } -// Non-stream -NonStreamResult Qwen3VL::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - const std::string tool_start_tag = ""; - const std::string tool_end_tag = ""; - - size_t search_from = 0; - while (true) { - size_t start_pos = response_text.find(tool_start_tag, search_from); - if (start_pos == std::string::npos) break; - - size_t block_content_start = start_pos + tool_start_tag.length(); - size_t end_pos = response_text.find(tool_end_tag, block_content_start); - size_t block_end = (end_pos != std::string::npos) ? end_pos : response_text.length(); - search_from = (end_pos != std::string::npos) ? end_pos + tool_end_tag.length() : block_end; - - std::string json_str = response_text.substr(block_content_start, block_end - block_content_start); - - std::string name, arguments; - try { - auto j = nlohmann::json::parse(json_str); - if (j.contains("name")) name = j["name"].get(); - if (j.contains("arguments")) { - arguments = j["arguments"].is_string() - ? j["arguments"].get() - : j["arguments"].dump(); - } - } catch (...) { - std::string key_name = "\"name\": \""; - size_t ns = json_str.find(key_name); - if (ns != std::string::npos) { - ns += key_name.length(); - size_t ne = json_str.find("\"", ns); - if (ne != std::string::npos) name = json_str.substr(ns, ne - ns); - } - std::string key_args = "\"arguments\":"; - size_t ap = json_str.find(key_args); - if (ap != std::string::npos) { - size_t bs = json_str.find("{", ap); - size_t be = json_str.rfind("}"); - if (bs != std::string::npos && be != std::string::npos && be > bs) - arguments = json_str.substr(bs, be - bs + 1); - } - } - - result.tool_calls_list.emplace_back(name, arguments); - } - - if (result.tool_calls_list.empty()) { - result.content = response_text; - } else { - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - } - - return result; -} - -// Stream -StreamResult Qwen3VL::parse_stream_content(const std::string content) { - std::string tool_start_tag = ""; - std::string tool_end_tag = ""; - - StreamResult result; - result.type = StreamEventType::CONTENT; - - if (content.find(tool_start_tag) != std::string::npos) { - is_in_tool_block_ = true; - tool_name_.clear(); - result.type = StreamEventType::WAITING; - return result; - } - - if (content.find("") != std::string::npos) { - is_in_tool_block_ = false; - - try { - auto j = nlohmann::json::parse(tool_name_); - - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - if (j.contains("name")) { - result.tool_name = j["name"].get(); - } - - if (j.contains("arguments")) { - if (j["arguments"].is_string()) { - result.tool_args_str = j["arguments"].get(); - } - else { - result.tool_args_str = j["arguments"].dump(); - } - } - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - } - return result; - } - - if (is_in_tool_block_) { - tool_name_ += content; - result.type = StreamEventType::WAITING; - return result; - } - - result.content = content; - return result; - -} - - /************ Qwen3VL_Flash **************/ int Qwen3VL_Flash::_pin_system_prefix(const std::string& system_text) { @@ -485,8 +370,6 @@ void Qwen3VL_Flash::_reset_turn() { } this->sampler->reset_penalties(); this->last_token = -1; - this->reset_parser(); - this->tool_name_.clear(); this->profiler_list[PREFILL_TIME].reset(); this->profiler_list[DECODING_TIME].reset(); this->profiler_list[SAMPLING_TIME].reset(); @@ -555,6 +438,7 @@ bool Qwen3VL_Flash::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& inpu } this->token_history = tokens; + this->_begin_turn(meta_info, tokens); std::vector tokens_to_prefill(tokens.begin() + skip, tokens.end()); auto prefill_start_time = this->profiler_list[PREFILL_TIME].start(); @@ -577,7 +461,8 @@ bool Qwen3VL_Flash::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& inpu this->total_tokens = static_cast(tokens.size()); this->profiler_list[SAMPLING_TIME].start(); - this->last_token = this->sampler->sample(y); + this->last_token = this->_sample(y, meta_info); + meta_info.insert_sample_pending = true; this->profiler_list[SAMPLING_TIME].stop(1); return true; } @@ -608,8 +493,6 @@ std::string Qwen3VL_Flash::generate(chat_meta_info_t& meta_info, int length_limi while (true) { if (is_cancelled()) { reason = CANCEL_DETECTED; - this->reset_parser(); - this->tool_name_.clear(); break; } @@ -618,7 +501,7 @@ std::string Qwen3VL_Flash::generate(chat_meta_info_t& meta_info, int length_limi this->profiler_list[DECODING_TIME].stop(1); this->profiler_list[SAMPLING_TIME].start(); - last_sampled_token = this->sampler->sample(y); + last_sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); this->total_tokens++; diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 32bb9ac7c..e15924bff 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -51,34 +51,14 @@ #include "tensor_utils/q4_npu_eXpress.hpp" #include "npu_utils/npu_utils.hpp" #include "minja/chat-template.hpp" +#include "chat/chat_format.hpp" +#include "chat/chat_constraint.hpp" +#include "chat/chat_grammar.hpp" #include // Forward declaration struct CancellationToken; -enum class StreamEventType { - WAITING, - CONTENT, - REASONING, - TOOL_DONE, -}; - -struct StreamResult { - StreamEventType type; - std::string content; - std::string tool_id; - std::string tool_name; - std::string tool_args_str; -}; - -struct NonStreamResult { - std::string content; - std::string reasoning_content; - std::string tool_name; - std::string tool_args; - std::vector> tool_calls_list; // (name, args) for multiple tool calls -}; - typedef enum { EOT_DETECTED, MAX_LENGTH_REACHED, @@ -104,15 +84,6 @@ inline std::string stop_reason_to_string(stop_reason_t reason){ } } -/// \brief What the request allows the model to do about tool calls. -/// \note Only the two modes the server parses today; "required" and the -/// per-function object form are reported as unsupported and fall back -/// to TOOL_CHOICE_AUTO. -typedef enum { - TOOL_CHOICE_AUTO, // the model may emit tool calls (default) - TOOL_CHOICE_NONE // tool call tokens are masked out of the logits -} tool_choice_t; - struct chat_meta_info_t { int max_prefill_len; int prompt_tokens; // whole prompt, cached prefix included @@ -124,9 +95,17 @@ struct chat_meta_info_t { uint64_t decoding_duration; // in nanoseconds stop_reason_t stop_reason; bool restore_allowed; - tool_choice_t tool_choice; - - chat_meta_info_t() : max_prefill_len(0), prompt_tokens(0), cached_prompt_tokens(0), generated_tokens(0), total_duration(0), load_duration(0), prefill_duration(0), decoding_duration(0), stop_reason(EOT_DETECTED), restore_allowed(false), tool_choice(TOOL_CHOICE_AUTO) {} + /// \brief Restricts sampling for this request (tool_choice, response_format); may be null + std::shared_ptr constraint; + /// \brief insert() sampled a token that generate() has not used yet + /// \note Models that make the model emit preamble tokens in generate() + /// throw that sample away; _force_token() takes it back out of the constraint. + bool insert_sample_pending = false; + /// \brief Decoded tail of the prompt as fed to the model; the output parser + /// reads it to tell whether generation starts inside a reasoning block + std::string prompt_tail; + + chat_meta_info_t() : max_prefill_len(0), prompt_tokens(0), cached_prompt_tokens(0), generated_tokens(0), total_duration(0), load_duration(0), prefill_duration(0), decoding_duration(0), stop_reason(EOT_DETECTED), restore_allowed(false) {} }; typedef enum { @@ -216,6 +195,7 @@ class AutoModel { uint32_t total_tokens = 0; std::unique_ptr lm_config = nullptr; std::unique_ptr chat_tmpl = nullptr; + chat_format_spec chat_format; std::string bos_token; std::string eos_token; @@ -242,18 +222,13 @@ class AutoModel { std::vector profiler_list; time_utils::time_with_unit last_prefill_time; - std::string tool_name_; - bool is_in_tool_block_ = false; - std::string buffer_; - StreamEventType current_mode_ = StreamEventType::CONTENT; - bool waiting_for_header_ = true; - - void _shared_load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false); void _shared_initialize_model_state(std::string model_path, json model_info, int context_length); void _shared_initialize_legacy_npu(bool enable_preemption); nlohmann::json _shared_setup_tokenizer(std::string model_path); + /// \brief Detect chat_format from the template; the tokenizer and bos id must be set up + void _detect_chat_format(const std::string& template_src); /// \brief Load a model onto its chosen backend /// \param model_path the model directory @@ -305,14 +280,35 @@ class AutoModel { buffer _chunked_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); std::string _shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }); - /// \brief Push the tool start token out of contention before sampling. - /// \param y the logits of the token about to be sampled - /// \param meta_info the meta information of the chat, carrying the tool choice - /// \note No-op unless the request asked for tool_choice=none and this model - /// reports a tool start token, so the default path is untouched. - void _apply_tool_choice_mask(buffer& y, const chat_meta_info_t& meta_info); - - StreamResult _shared_think_tool_calling_pasrsed(const std::string content); + /// \brief Note where this turn's output starts, once the prompt is final + /// \param meta_info the request; gets the prompt tail, and its constraint is started + /// \param prompt_tokens the whole prompt as the model sees it + /// \note Every insert() calls this before sampling the first token. + void _begin_turn(chat_meta_info_t& meta_info, const std::vector& prompt_tokens); + + /// \brief Sample the next token under the request's constraint + /// \param logits the logits of the token about to be sampled, masked in place + /// \param meta_info the request, carrying the constraint + /// \return the sampled token, already reported to the constraint + int _sample(buffer& logits, chat_meta_info_t& meta_info); + + /// \brief Report a token the model is made to emit rather than samples + /// \note For the think preamble some generate()s feed after insert(); the + /// constraint has to see it like any generated text. The first forced + /// token of a turn also drops the sample insert() made, which such a + /// generate() never uses. + void _force_token(chat_meta_info_t& meta_info, int token); + + /// \brief Drop the sample insert() made when generate() feeds tokens instead + /// \note For preamble tokens that never reach the output, like the empty + /// think block fed when thinking is off: the constraint does not see + /// them, as the output parser does not either. + void _skip_insert_sample(chat_meta_info_t& meta_info); + /// \brief Scratch bitmask the constraint fills before each sample + std::vector constraint_mask_; + /// \brief Grammar compiler over this model's vocabulary, built on first use + std::unique_ptr grammar_engine_; + std::string grammar_engine_path_; public: //************ Shared by all models *************/ @@ -325,12 +321,6 @@ class AutoModel { AutoModel(flm_rt::device* npu_device_inst, std::string current_model = ""); - void reset_parser() { - buffer_.clear(); - is_in_tool_block_ = false; - current_mode_ = StreamEventType::CONTENT; - } - /// \brief Clear the context virtual void clear_context(); @@ -494,20 +484,23 @@ class AutoModel { virtual void load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false, const std::string& backend = "") {} virtual std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) = 0; + /// \brief Turn structure of the prompt, used by the prompt cache only + /// \note Output parsing goes by get_chat_format(), not by this virtual chat_template_type_t get_chat_template_type() { return chat_template_type_t::chat_ml; } + /// \brief How this model writes reasoning and tool calls + /// \note Detected from the chat template and the tokenizer when the tokenizer is set up + const chat_format_spec& get_chat_format() const { + return this->chat_format; + } virtual bool check_using_checkpint() { return true; } - /// \brief The token that opens a tool call for this model - /// \return the token id, or -1 if this model has no single such token - /// \note Models that report an id can have tool calling suppressed through - /// tool_choice=none; the rest keep emitting tool calls either way. - virtual int get_tool_start_token_id() const { - return -1; - } + /// \brief Build the sampling constraint a request asks for + /// \return null when the request needs no constraint + std::shared_ptr make_constraint(const chat_constraint_request& request); /// \brief Insert the tokens /// \param tokens the tokens /// \param is_system_prompt the is system prompt @@ -548,31 +541,6 @@ class AutoModel { virtual std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) = 0; - virtual NonStreamResult parse_nstream_content(const std::string response_text) { - NonStreamResult result; - result.content = response_text; - return result; - } - - virtual StreamResult parse_stream_content(const std::string content) { - //header_print("AUTOMODEL PARSING", content); - - StreamResult result; - result.type = StreamEventType::CONTENT; - result.content = content; - - return result; - } - - virtual StreamResult parse_stream_content_final(const std::string content) { - if (!content.empty()) { - return parse_stream_content(content); - } - - StreamResult result; - result.type = StreamEventType::WAITING; - return result; - } }; diff --git a/src/include/AutoModel/modeling_gemma4_12b.hpp b/src/include/AutoModel/modeling_gemma4_12b.hpp index ff971ec44..e7754a11e 100644 --- a/src/include/AutoModel/modeling_gemma4_12b.hpp +++ b/src/include/AutoModel/modeling_gemma4_12b.hpp @@ -28,7 +28,6 @@ class Gemma4_12B : public AutoModel { void setup_tokenizer(std::string model_path); - StreamResult parse_stream_content_impl(const std::string content, bool is_final); static constexpr int boc_token_id = 100; // begin of channel token id static constexpr int eoc_token_id = 101; // end of channel token id @@ -44,7 +43,6 @@ class Gemma4_12B : public AutoModel { static constexpr int audio_token_id = 258881; // audio token id static constexpr int eoa_token_id = 258883; // end of audio token id - static constexpr int tool_start_token_id = 48; ImageReader image_reader_; gemma4_12b_image_t load_image(const std::string& filename); @@ -106,20 +104,10 @@ class Gemma4_12B : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text) override; - StreamResult parse_stream_content(const std::string content) override; - StreamResult parse_stream_content_final(const std::string content) override; chat_template_type_t get_chat_template_type() override { return chat_template_type_t::gemma4; } - /// \note Gemma4-12B opens every tool call with <|tool_call>, so masking that - /// one token is enough to honour tool_choice=none. Unlike Gemma4e there - /// is no size-dependent enable_tool gate -- this checkpoint always - /// supports tool calling, so the id is reported unconditionally. - int get_tool_start_token_id() const override { - return tool_start_token_id; - } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_gemma4e.hpp b/src/include/AutoModel/modeling_gemma4e.hpp index 1a32378b8..6ccff23c7 100644 --- a/src/include/AutoModel/modeling_gemma4e.hpp +++ b/src/include/AutoModel/modeling_gemma4e.hpp @@ -53,7 +53,6 @@ class Gemma4e : public AutoModel { static constexpr int think_start_id = 100; static constexpr int think_end_id = 101; - static constexpr int tool_start_token_id = 48; bool enable_think = false; bool enable_tool = false; @@ -101,7 +100,6 @@ class Gemma4e : public AutoModel { int pooling_kernel_size ); - StreamResult parse_stream_content_impl(const std::string content, bool is_final); @@ -126,9 +124,6 @@ class Gemma4e : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); - StreamResult parse_stream_content_final(const std::string content) override; chat_template_type_t get_chat_template_type() { return chat_template_type_t::gemma4; } @@ -136,9 +131,6 @@ class Gemma4e : public AutoModel { return this->enable_think; } - int get_tool_start_token_id() const override { - return this->enable_tool ? tool_start_token_id : -1; - } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_gpt_oss.hpp b/src/include/AutoModel/modeling_gpt_oss.hpp index ea6747771..607abc56d 100644 --- a/src/include/AutoModel/modeling_gpt_oss.hpp +++ b/src/include/AutoModel/modeling_gpt_oss.hpp @@ -19,19 +19,6 @@ class GPT_OSS : public AutoModel { void setup_tokenizer(std::string model_path); - enum ToolGrammarState { - STATE_EXPECT_TO_FUNCTIONS, - STATE_EXPECT_FUNCTION_NAME, - STATE_EXPECT_CONSTRAIN, - STATE_EXPECT_MESSAGE, - STATE_COMPLETE - }; - ToolGrammarState current_state = STATE_EXPECT_TO_FUNCTIONS; - std::string accumulated_text; - void reset_tool_grammar_state() { - current_state = STATE_EXPECT_TO_FUNCTIONS; - accumulated_text.clear(); - } public: GPT_OSS(flm_rt::device* npu_device_inst); @@ -40,13 +27,6 @@ class GPT_OSS : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - int _sample_in_tool(buffer& logits); - std::vector get_to_functions_tokens(); - std::vector get_function_name_tokens(); - void update_state(int token_id); - void mask_logits(buffer& logits, const std::vector& allowed_tokens); - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); chat_template_type_t get_chat_template_type() { return chat_template_type_t::harmony; } diff --git a/src/include/AutoModel/modeling_lfm2.hpp b/src/include/AutoModel/modeling_lfm2.hpp index 7567eccdd..3f105ead9 100644 --- a/src/include/AutoModel/modeling_lfm2.hpp +++ b/src/include/AutoModel/modeling_lfm2.hpp @@ -32,7 +32,6 @@ class LFM2 : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - StreamResult parse_stream_content(const std::string content); bool check_using_checkpint() { return false; } @@ -64,6 +63,4 @@ class LFM2_5_TK : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_llama3.hpp b/src/include/AutoModel/modeling_llama3.hpp index 510979de2..dfb19dbba 100644 --- a/src/include/AutoModel/modeling_llama3.hpp +++ b/src/include/AutoModel/modeling_llama3.hpp @@ -47,6 +47,4 @@ class DeepSeek_r1_8b : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_nanbeige.hpp b/src/include/AutoModel/modeling_nanbeige.hpp index 09b8c7ced..a9fa71ff0 100644 --- a/src/include/AutoModel/modeling_nanbeige.hpp +++ b/src/include/AutoModel/modeling_nanbeige.hpp @@ -24,6 +24,4 @@ class Nanbeige : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); }; diff --git a/src/include/AutoModel/modeling_qwen3.hpp b/src/include/AutoModel/modeling_qwen3.hpp index 6d639ce21..011691ab5 100644 --- a/src/include/AutoModel/modeling_qwen3.hpp +++ b/src/include/AutoModel/modeling_qwen3.hpp @@ -36,8 +36,6 @@ class Qwen3 : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); /// \brief Override configure_parameter to handle Qwen3-specific parameters bool configure_parameter(std::string parameter_name, const std::any& value) override { @@ -85,7 +83,6 @@ class Qwen3 : public AutoModel { class Qwen3_IT : public AutoModel { private: std::string current_model = "Qwen3_IT"; - static constexpr int tool_start_token_id = 151657; void setup_tokenizer(std::string model_path); @@ -98,18 +95,10 @@ class Qwen3_IT : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); bool check_using_checkpint() { return false; } - /// \note Qwen3-IT opens every tool call with , so masking that one - /// token is enough to honour tool_choice=none. Reported unconditionally - /// -- unlike the VL/MoE wrappers this class has no enable_tool gate. - int get_tool_start_token_id() const override { - return tool_start_token_id; - } }; class Qwen3_TK : public AutoModel { @@ -131,8 +120,6 @@ class Qwen3_TK : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); }; @@ -156,6 +143,4 @@ class DeepSeek_r1_0528_8b : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_qwen3_5_omni.hpp b/src/include/AutoModel/modeling_qwen3_5_omni.hpp index 9b9ab064b..6e10d7907 100644 --- a/src/include/AutoModel/modeling_qwen3_5_omni.hpp +++ b/src/include/AutoModel/modeling_qwen3_5_omni.hpp @@ -46,9 +46,6 @@ class Qwen3_5_Omni : public AutoModel { /// \brief insert() followed by generate(). std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); - StreamResult parse_stream_content_final(const std::string content) override; /// \brief Synthesize speech from the last thinker result (talker path). /// \note Talker/codec/WAV-writer are not implemented in the engine yet; this @@ -194,6 +191,5 @@ class Qwen3_5_Omni : public AutoModel { std::unique_ptr engine = nullptr; qwen3_5_omni_thinker_result_t last_thinker_result; - StreamResult parse_stream_content_impl(const std::string content, bool is_final); }; diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index 430ad1bfd..545920157 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -28,7 +28,6 @@ class Qwen3_5VL : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; - static constexpr int tool_start_token_id = 248058; void setup_tokenizer(std::string model_path); @@ -58,21 +57,11 @@ class Qwen3_5VL : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); - StreamResult parse_stream_content_final(const std::string content) override; private: - StreamResult parse_stream_content_impl(const std::string content, bool is_final); public: - /// \note Qwen3.5-VL opens every tool call with , so masking that - /// one token is enough to honour tool_choice=none. Gated on enable_tool - /// to match apply_chat_template, which only renders tools when it is set. - int get_tool_start_token_id() const override { - return this->enable_tool ? tool_start_token_id : -1; - } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_6_moe.hpp b/src/include/AutoModel/modeling_qwen3_6_moe.hpp index 0887e73f0..101325110 100644 --- a/src/include/AutoModel/modeling_qwen3_6_moe.hpp +++ b/src/include/AutoModel/modeling_qwen3_6_moe.hpp @@ -28,7 +28,6 @@ class Qwen3_6_MOE : public AutoModel { bool enable_tool = false; int think_start_id = 248068; int think_end_id = 248069; - static constexpr int tool_start_token_id = 248058; void setup_tokenizer(std::string model_path); @@ -58,21 +57,11 @@ class Qwen3_6_MOE : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); - StreamResult parse_stream_content_final(const std::string content) override; private: - StreamResult parse_stream_content_impl(const std::string content, bool is_final); public: - /// \note Qwen3.6-MoE opens every tool call with , so masking that - /// one token is enough to honour tool_choice=none. Gated on enable_tool - /// to match apply_chat_template, which only renders tools when it is set. - int get_tool_start_token_id() const override { - return this->enable_tool ? tool_start_token_id : -1; - } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_8mtp.hpp b/src/include/AutoModel/modeling_qwen3_8mtp.hpp index 3e3683d8b..ca117462c 100644 --- a/src/include/AutoModel/modeling_qwen3_8mtp.hpp +++ b/src/include/AutoModel/modeling_qwen3_8mtp.hpp @@ -69,6 +69,7 @@ class Qwen3_8MTP : public AutoModel { /// prefilled instead of what the flag says now. std::vector preamble_to_stream; + void setup_tokenizer(std::string model_path); // ---- image pipeline --------------------------------------------------- @@ -150,9 +151,6 @@ class Qwen3_8MTP : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); - StreamResult parse_stream_content_final(const std::string content) override; /// \brief the base profile plus the MTP draft/verify/replay breakdown /// \note Overridden because "Decoding time" is one number for a model with @@ -176,7 +174,6 @@ class Qwen3_8MTP : public AutoModel { void clear_context() override; private: - StreamResult parse_stream_content_impl(const std::string content, bool is_final); /// \brief this model's decode loop: MTP speculation, ordinary sampling as /// the fallback diff --git a/src/include/AutoModel/modeling_qwen3vl.hpp b/src/include/AutoModel/modeling_qwen3vl.hpp index 25da2faed..fa9cec704 100644 --- a/src/include/AutoModel/modeling_qwen3vl.hpp +++ b/src/include/AutoModel/modeling_qwen3vl.hpp @@ -41,7 +41,6 @@ class Qwen3VL : public AutoModel { protected: static constexpr int IMAGE_SOFT_TOKEN_ID = 151655; - static constexpr int TOOL_START_TOKEN_ID = 151657; qwen3vl_image_t load_image(const std::string& filename); qwen3vl_image_t load_image_base64(const std::string& base64_string); @@ -83,17 +82,10 @@ class Qwen3VL : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text); - StreamResult parse_stream_content(const std::string content); bool check_using_checkpint() { return false; } - /// \note Qwen3-VL opens every tool call with , so masking that one - /// token is enough to honour tool_choice=none. - int get_tool_start_token_id() const override { - return TOOL_START_TOKEN_ID; - } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/openai_policy.hpp b/src/include/openai_policy.hpp deleted file mode 100644 index cbb8a69a8..000000000 --- a/src/include/openai_policy.hpp +++ /dev/null @@ -1,164 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "AutoModel/automodel.hpp" - -// content type -enum class openai_content_type { - CONTENT, - REASONING, - TOOL -}; - -// reasoning tags -struct reasoning_tags { - std::string start_tag; - std::string end_tag; -}; - -// response struct -struct openai_content { - std::string text; - openai_content_type type; -}; - - - -class openai_policy { -public: - explicit openai_policy(chat_template_type_t template_type, bool start_in_reasoning = false); - - static reasoning_tags get_reasoning_tags(chat_template_type_t template_type); - - std::vector parse(const std::string& text); - std::vector flush(); - void reset(bool start_in_reasoning = false); - - bool in_reasoning_part() const; - -private: - static void append(std::vector& parts, - openai_content_type type, - const std::string& text); - - static std::size_t partial_tag_length(const std::string& buffer, const std::string& tag); - - openai_content_type current_type() const; - const std::string& pending_tag() const; - - chat_template_type_t template_type; - reasoning_tags tags; - std::string buffer; - bool in_reasoning; -}; - -inline openai_policy::openai_policy(chat_template_type_t template_type, bool start_in_reasoning) - : template_type(template_type), - tags(get_reasoning_tags(template_type)), - in_reasoning(start_in_reasoning) { - buffer.reserve(256); -} - -inline reasoning_tags openai_policy::get_reasoning_tags(chat_template_type_t template_type) { - switch (template_type) { - case chat_template_type_t::chat_ml: - return reasoning_tags{"", ""}; - case chat_template_type_t::harmony: - return reasoning_tags{"<|start|>assistant<|channel|>analysis<|message|>", - "<|end|><|start|>assistant<|channel|>final<|message|>"}; - case chat_template_type_t::gemma4: - return reasoning_tags{"<|channel>thought", ""}; - case chat_template_type_t::unknown_template: - return reasoning_tags{"", ""}; - } - return reasoning_tags{"", ""}; -} - -inline std::vector openai_policy::parse(const std::string& text) { - std::vector parts; - // carry over whatever was held back last call, a tag may straddle two calls - buffer += text; - - // consume every complete tag sitting in the buffer, one call can cross - // several boundaries; pending_tag() is the start tag while we are outside - // reasoning and the end tag while we are inside it - while (!pending_tag().empty()) { - const std::string& tag = pending_tag(); - std::size_t pos = buffer.find(tag); - if (pos == std::string::npos) { - // no boundary yet, the rest is handled below - break; - } - // everything before the tag belongs to the part we are leaving - append(parts, current_type(), buffer.substr(0, pos)); - // drop the tag itself, it is a marker and never reaches the client - buffer.erase(0, pos + tag.size()); - in_reasoning = !in_reasoning; - } - - // the tail may be the beginning of the next tag, hold that many bytes back - // so a split marker is not emitted as visible text - std::size_t held = partial_tag_length(buffer, pending_tag()); - if (buffer.size() > held) { - std::size_t emit_len = buffer.size() - held; - append(parts, current_type(), buffer.substr(0, emit_len)); - buffer.erase(0, emit_len); - } - - // whatever stays in buffer is a partial tag, flush() releases it at end of stream - return parts; -} - -inline std::vector openai_policy::flush() { - std::vector parts; - append(parts, current_type(), buffer); - buffer.clear(); - return parts; -} - -inline void openai_policy::reset(bool start_in_reasoning) { - buffer.clear(); - in_reasoning = start_in_reasoning; -} - -inline bool openai_policy::in_reasoning_part() const { - return in_reasoning; -} - -inline void openai_policy::append(std::vector& parts, - openai_content_type type, - const std::string& text) { - if (text.empty()) { - return; - } - if (!parts.empty() && parts.back().type == type) { - parts.back().text += text; - return; - } - parts.push_back(openai_content{text, type}); -} - -inline std::size_t openai_policy::partial_tag_length(const std::string& buffer, const std::string& tag) { - if (tag.empty() || buffer.empty()) { - return 0; - } - std::size_t max_len = std::min(buffer.size(), tag.size() - 1); - for (std::size_t len = max_len; len > 0; --len) { - if (buffer.compare(buffer.size() - len, len, tag, 0, len) == 0) { - return len; - } - } - return 0; -} - -inline openai_content_type openai_policy::current_type() const { - return in_reasoning ? openai_content_type::REASONING : openai_content_type::CONTENT; -} - -inline const std::string& openai_policy::pending_tag() const { - return in_reasoning ? tags.end_tag : tags.start_tag; -} diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 8fc9180df..ee60b8009 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -569,56 +569,40 @@ void RestHandler::configure_chat_engine_parameters(const json& options, const js } } -json RestHandler::build_nstream_response(std::string response_text, chat_meta_info_t& meta_info) { - // Get tool info - NonStreamResult result = auto_chat_engine->parse_nstream_content(response_text); +json RestHandler::build_nstream_response(const std::string& response_text, chat_meta_info_t& meta_info, + const chat_parse_options& parse_options) { + const chat_format_spec& spec = auto_chat_engine->get_chat_format(); + chat_msg parsed = chat_parse(spec, chat_format_prefill(spec, meta_info.prompt_tail) + response_text, + false, parse_options); json message; message["role"] = "assistant"; - - bool is_reasoning = !result.reasoning_content.empty(); - bool is_tool_call = !result.tool_calls_list.empty() || !result.tool_name.empty(); - - if (is_reasoning) { - message["reasoning_content"] = result.reasoning_content; + if (!parsed.reasoning.empty()) { + message["reasoning_content"] = parsed.reasoning; } + const bool is_tool_call = !parsed.tool_calls.empty(); if (is_tool_call) { json tool_calls_json = json::array(); - if (!result.tool_calls_list.empty()) { - int idx = 0; - for (const auto& tc : result.tool_calls_list) { - tool_calls_json.push_back({ - {"index", idx++}, - {"id", "call_" + std::to_string(std::time(nullptr)) + "_" + std::to_string(idx)}, - {"type", "function"}, - {"function", { - {"name", tc.first}, - {"arguments", tc.second} - }} - }); - } - } else { + for (size_t i = 0; i < parsed.tool_calls.size(); i++) { tool_calls_json.push_back({ - {"index", 0}, - {"id", "call_" + std::to_string(std::time(nullptr))}, + {"index", i}, + {"id", chat_new_tool_call_id()}, {"type", "function"}, {"function", { - {"name", result.tool_name}, - {"arguments", result.tool_args} + {"name", parsed.tool_calls[i].name}, + {"arguments", parsed.tool_calls[i].arguments} }} }); } message["tool_calls"] = tool_calls_json; - if (!result.content.empty()) { - message["content"] = result.content; - } + // OpenAI sends null rather than "" when the turn is only tool calls + message["content"] = parsed.content.empty() ? json(nullptr) : json(parsed.content); } else { - message["content"] = result.content; + message["content"] = parsed.content; } - // Follow meta_info for the finish reason, same as the streaming path. // A tool call is only visible after parsing the generated text, so promote // the stop reason here; every other case (length, cancel, error) is already @@ -1153,20 +1137,22 @@ void RestHandler::handle_openai_chat_completion(const json& request, json tools = request.value("tools", json::array()); json options = request.value("options", json::object()); - // Only "auto" and "none" are honoured; "required" and the per-function - // object form are not implemented yet, and a request asking for one is - // served as "auto" rather than refused. - json tool_choice = request.value("tool_choice", json("auto")); - tool_choice_t tool_choice_mode = TOOL_CHOICE_AUTO; - if (tool_choice.is_string() && tool_choice.get() == "none") { - tool_choice_mode = TOOL_CHOICE_NONE; + // tool_choice, parallel_tool_calls and response_format become a decoding constraint + chat_constraint_request constraint_request; + std::string request_error; + if (!chat_constraint_request_from_openai(request, tools, constraint_request, request_error)) { + send_response({{"error", {{"message", request_error}, {"type", "invalid_request_error"}, {"code", 400}}}}); + return; } - else if (!(tool_choice.is_string() && tool_choice.get() == "auto")) { - header_print("Warning", "Unsupported tool_choice " + tool_choice.dump() + ", falling back to auto."); + if (constraint_request.tool_choice == chat_tool_choice::NONE) { + // the model is not shown the tools at all: banning the call opener alone + // leaves it writing the call's JSON as plain text + tools = json::array(); } auto load_start_time = time_utils::now(); - if (!ensure_model_loaded(model)) { + // a request may name the backend to run on, see docs/instructions/cli.md + if (!ensure_model_loaded(model, request.value("backend", std::string()))) { json error_response = {{"error", "Failed to load " + model + " model!"}}; send_response(error_response); return; @@ -1241,18 +1227,27 @@ void RestHandler::handle_openai_chat_completion(const json& request, normalize_requested_max_new_tokens(openai_chat_budget); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; - meta_info.tool_choice = tool_choice_mode; + try { + meta_info.constraint = auto_chat_engine->make_constraint(constraint_request); + } catch (const std::invalid_argument& e) { + // the request asks for something this model's format cannot do, or a bad schema + send_response({{"error", {{"message", e.what()}, {"type", "invalid_request_error"}, {"code", 400}}}}); + return; + } + // tool markup is only a tool call when the request offered tools + chat_parse_options parse_options; + parse_options.tools = &tools; + parse_options.parse_tool_calls = !tools.empty() && constraint_request.tool_choice != chat_tool_choice::NONE; if (stream){ // Create a wrapper callback that passes the pre-formatted SSE string directly cancellation_token->reset(); - auto_chat_engine->reset_parser(); bool stream_started = false; auto openai_stream_callback = [&send_streaming_response, &stream_started](const std::string& data, bool is_final) { stream_started = true; json data_json = data; send_streaming_response(data_json, is_final); }; - streaming_ostream_openai_chat ostream(model, auto_chat_engine.get(), openai_stream_callback); // streaming in chat completion format + streaming_ostream_openai_chat ostream(model, auto_chat_engine.get(), openai_stream_callback, parse_options); // streaming in chat completion format header_print("FLM", "Start prefill..."); try { @@ -1286,6 +1281,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.reset(); return; } + ostream.start(meta_info); header_print("FLM", "Start generating..."); try { auto_chat_engine->generate(meta_info, length_limit, ostream, [&] { return cancellation_token->cancelled(); }); @@ -1359,7 +1355,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, return; } // check response_text - json choices = build_nstream_response(response_text, meta_info); + json choices = build_nstream_response(response_text, meta_info, parse_options); response = { {"id", "fastflowlm-chat-completion"}, {"object", "chat.completion"}, @@ -1485,7 +1481,8 @@ void RestHandler::handle_openai_completion(const json& request, int length_limit = request.value("max_tokens", 4096); - if (!ensure_model_loaded(model)) { + // a request may name the backend to run on, see docs/instructions/cli.md + if (!ensure_model_loaded(model, request.value("backend", std::string()))) { json error_response = {{"error", "Failed to load " + model + " model!"}}; send_response(error_response); return; diff --git a/src/server/rest_handler.hpp b/src/server/rest_handler.hpp index 83730a98e..7f2ba92a6 100644 --- a/src/server/rest_handler.hpp +++ b/src/server/rest_handler.hpp @@ -14,6 +14,8 @@ #include "AutoEmbeddingModel/all_embedding_model.hpp" #endif #include "model_list.hpp" +#include "chat/chat_parser.hpp" +#include "chat/chat_stream.hpp" #include "program_args.hpp" @@ -114,7 +116,8 @@ class RestHandler { void ensure_asr_model_loaded(const std::string& model_tag); void ensure_embed_model_loaded(const std::string& model_tag); void configure_chat_engine_parameters(const json& options, const json& request); - json build_nstream_response(std::string response_text, chat_meta_info_t& meta_info); + json build_nstream_response(const std::string& response_text, chat_meta_info_t& meta_info, + const chat_parse_options& parse_options); std::unique_ptr auto_chat_engine; diff --git a/src/server/streaming_ostream_openai.hpp b/src/server/streaming_ostream_openai.hpp index ca93e4849..d00774a9d 100644 --- a/src/server/streaming_ostream_openai.hpp +++ b/src/server/streaming_ostream_openai.hpp @@ -1,6 +1,6 @@ /*! * Copyright (c) 2026 Advanced Micro Devices, Inc. - * \file streaming_ostream.hpp + * \file streaming_ostream_openai.hpp * \brief Custom ostream for streaming * \author FastFlowLM Team * \date 2025-06-24 @@ -18,7 +18,7 @@ #include #include #include "AutoModel/automodel.hpp" -#include "harmony_filter.hpp" +#include "chat/chat_stream.hpp" using json = nlohmann::ordered_json; @@ -255,24 +255,34 @@ class streaming_ostream_openai : public std::ostream { // v1/chat/completions -///@brief Custom streambuf that captures tokens and sends them immediately -///@param model the model -///@param callback the callback -///@param is_chat_format the is chat format -///@return the streaming buf -///@note The packet is sent every std::flush, but only when UTF-8 sequences are complete +///@brief Custom streambuf that parses the generated text and streams it as chat completion chunks +///@note The packet is sent every std::flush, but only when UTF-8 sequences are complete. +/// Every packet is parsed with the model's chat format; reasoning, content and +/// tool call arguments go out as the deltas between consecutive parses. class streaming_buf_openai_chat : public std::streambuf { public: ///@brief StreamCallback using StreamCallback = std::function; - streaming_buf_openai_chat(const std::string& model, AutoModel* auto_chat_engine, StreamCallback callback) - : model_name(model), auto_chat_engine(auto_chat_engine), stream_callback(callback), first_chunk(true) { + ///@param model the model name reported to the client + ///@param auto_chat_engine the engine generating the text + ///@param callback receives each server-sent event + ///@param options per request parsing options; the tools must outlive the stream + streaming_buf_openai_chat(const std::string& model, AutoModel* auto_chat_engine, StreamCallback callback, + chat_parse_options options) + : model_name(model), auto_chat_engine(auto_chat_engine), stream_callback(callback), + parse_options(options) { // Generate a unique ID for this stream generate_stream_id(); generate_created(); generate_system_fingerprint(); - harmony_filter_inst = std::make_unique(); + } + + ///@brief Call this once the prompt is inserted, before generating + ///@param meta_info the request's meta info, its prompt tail tells where the output starts + void start(const chat_meta_info_t& meta_info) { + const chat_format_spec& spec = auto_chat_engine->get_chat_format(); + stream = std::make_unique(spec, chat_format_prefill(spec, meta_info.prompt_tail), parse_options); } protected: @@ -289,27 +299,23 @@ class streaming_buf_openai_chat : public std::streambuf { ///@brief Called when stream is flushed ///@return 0 int sync() override { - flush_complete_utf8_sequences(false); + flush_complete_utf8_sequences(); return 0; } public: ///@brief Call this when generation is complete void finalize(chat_meta_info_t& meta_info) { - // Send all remaining content, including incomplete sequences - if (!buffer.empty()) { - send_response(buffer, false, true); - buffer.clear(); - } - else { - send_response("", false, true); - } + // Send all remaining content, including incomplete sequences, and let + // the parser release whatever it held back + send_response(buffer, true); + buffer.clear(); // Only promote to a tool-call finish reason on an otherwise clean stop; // length/cancel/error must win, so a truncated or aborted generation is // never reported to the client as a complete tool call. - if (stream_stop_reason == stop_reason_t::TOOL_DETECTED + if (stream && !stream->message().tool_calls.empty() && meta_info.stop_reason == stop_reason_t::EOT_DETECTED) { - meta_info.stop_reason = stream_stop_reason; + meta_info.stop_reason = stop_reason_t::TOOL_DETECTED; } send_final_response(meta_info); } @@ -350,10 +356,6 @@ class streaming_buf_openai_chat : public std::streambuf { system_fingerprint = ss.str(); } - void tool_calling_id() { - - } - ///@brief Get UTF-8 sequence length from first byte ///@param first_byte the first byte ///@return sequence length, or 0 if invalid @@ -374,8 +376,7 @@ class streaming_buf_openai_chat : public std::streambuf { } ///@brief Flush only complete UTF-8 sequences - ///@param is_final the is final - void flush_complete_utf8_sequences(bool is_final) { + void flush_complete_utf8_sequences() { if (buffer.empty()) return; std::string complete_content; @@ -405,7 +406,7 @@ class streaming_buf_openai_chat : public std::streambuf { // Send complete sequences if any if (!complete_content.empty()) { - send_response(complete_content, is_final); + send_response(complete_content, false); } // Remove processed bytes from buffer @@ -414,68 +415,51 @@ class streaming_buf_openai_chat : public std::streambuf { } } - ///@brief Send the response - ///@param content the content - ///@param is_final the is final - void send_response(const std::string& content, bool is_final, bool parser_final = false) { - json response; - - StreamResult result = parser_final - ? auto_chat_engine->parse_stream_content_final(content) - : auto_chat_engine->parse_stream_content(content); - if (result.type == StreamEventType::WAITING) { - return; + ///@brief Parse the new text and send what changed + ///@param content the newly generated text + ///@param generation_done true once the model has stopped + void send_response(const std::string& content, bool generation_done) { + if (!stream) { + // start() was skipped; parse as if nothing was prefilled + stream = std::make_unique(auto_chat_engine->get_chat_format(), "", parse_options); } - json delta; - if (result.type == StreamEventType::TOOL_DONE) { - stream_stop_reason = stop_reason_t::TOOL_DETECTED; - delta = { - {"tool_calls", json::array({ + for (const chat_msg_delta& d : stream->update(content, generation_done)) { + json delta = {{"role", "assistant"}}; + if (d.is_tool_call()) { + json call = {{"index", d.tool_index}}; + if (!d.tool_id.empty()) { + // the first delta of a call names it; later ones only add arguments + call["id"] = d.tool_id; + call["type"] = "function"; + call["function"] = {{"name", d.tool_name}, {"arguments", d.tool_arguments}}; + } + else { + call["function"] = {{"arguments", d.tool_arguments}}; + } + delta["tool_calls"] = json::array({call}); + } + else if (!d.reasoning.empty()) { + delta["reasoning_content"] = d.reasoning; + } + else { + delta["content"] = d.content; + } + + json response = { + {"id", stream_id}, + {"object", "chat.completion.chunk"}, + {"created", created}, + {"model", model_name}, + {"choices", json::array({ { - {"index", index++}, - {"id", result.tool_id}, - {"type", "function"}, - {"function", { - {"name", result.tool_name}, - {"arguments", result.tool_args_str} - }} + {"index", 0}, + {"delta", delta}, + {"finish_reason", nullptr} } })} }; - header_print("FLM", "Tool name: " + result.tool_name); - header_print("FLM", "Tool args: " + result.tool_args_str); - header_print("FLM", "Tool JSON: " + delta["tool_calls"][0]["function"].dump()); + stream_callback("data: " + response.dump() + "\n\n", false); } - else if (result.type == StreamEventType::REASONING) { - delta = { - {"role", "assistant"}, - {"reasoning_content", result.content} - }; - } - else if (result.type == StreamEventType::CONTENT) { - if (result.content.empty()) return; - - delta = { - {"role", "assistant"}, - {"content", result.content} - }; - } - - response = { - {"id", stream_id}, - {"object", "chat.completion.chunk"}, - {"created", created}, - {"model", model_name}, - {"choices", json::array({ - { - {"index", 0}, - {"delta", delta}, - {"finish_reason", nullptr} - } - })} - }; - - stream_callback("data: " + response.dump() + "\n\n", is_final); } ///@brief Send the chat final response @@ -530,24 +514,23 @@ class streaming_buf_openai_chat : public std::streambuf { std::string stream_id; long created; std::string system_fingerprint; - ///@brief First chunk flag - bool first_chunk; - stop_reason_t stream_stop_reason = stop_reason_t::EOT_DETECTED; - ///@brief tool_calls delta index, scoped to this stream - int index = 0; - - std::unique_ptr harmony_filter_inst; + ///@brief Per request parsing options + chat_parse_options parse_options; + ///@brief Parse state of this response, created by start() + std::unique_ptr stream; }; -///@brief Custom ostream for streaming -///@param model the model -///@param callback the callback -///@param is_chat_format the is chat format -///@return the streaming ostream +///@brief Custom ostream for streaming chat completions class streaming_ostream_openai_chat : public std::ostream { public: - streaming_ostream_openai_chat(const std::string& model, AutoModel* auto_chat_engine, streaming_buf_openai_chat::StreamCallback callback) - : std::ostream(&buf), buf(model, auto_chat_engine, callback) {} + streaming_ostream_openai_chat(const std::string& model, AutoModel* auto_chat_engine, streaming_buf_openai_chat::StreamCallback callback, + chat_parse_options options) + : std::ostream(&buf), buf(model, auto_chat_engine, callback, options) {} + + ///@brief Call once the prompt is inserted, before generating + void start(const chat_meta_info_t& meta_info) { + buf.start(meta_info); + } ///@brief Finalize the chat void finalize(chat_meta_info_t& meta_info) { @@ -558,4 +541,3 @@ class streaming_ostream_openai_chat : public std::ostream { ///@brief Buffer streaming_buf_openai_chat buf; }; - From 48ad359a56409e19af968a9d8f9567aea0a20d01 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 12 Oct 2026 12:49:53 -0600 Subject: [PATCH 06/18] Report streaming usage only when stream_options.include_usage asks for it Following OpenAI, a streamed chat or text completion no longer carries usage unless the request sets stream_options.include_usage. When it does, every chunk has a null usage and one final chunk with no choices carries the usage of the whole request before [DONE]. The server log still prints the usage either way. The usage contract test and the sweep script now request it, and the contract test checks the chunk layout. Co-Authored-By: Claude --- .github/script/sweep_common.py | 8 +- docs/docs/instructions/server/openapi.md | 16 +++ src/server/rest_handler.cpp | 16 ++- src/server/streaming_ostream_openai.hpp | 109 ++++++++++++++----- src/test/openai_usage/test_usage_contract.py | 46 +++++--- 5 files changed, 146 insertions(+), 49 deletions(-) diff --git a/.github/script/sweep_common.py b/.github/script/sweep_common.py index 238001f9a..e4b428944 100644 --- a/.github/script/sweep_common.py +++ b/.github/script/sweep_common.py @@ -703,10 +703,8 @@ def collect_stream(self, response) -> tuple[str, str]: f"stream exceeded the {self.args.request_timeout:.0f}s " f"request timeout after {len(output_content)} characters" ) - # FLM attaches usage to the final chunk, which also carries a - # choices entry with a null delta. Read it before the choices - # guard below, so a server that instead sends usage on its own - # terminal chunk (the stock OpenAI shape) still gets picked up. + # With include_usage the usage arrives on its own terminal chunk + # with no choices, so read it before the choices guard below. if getattr(chunk, "usage", None): usage = chunk.usage if not chunk.choices: @@ -730,6 +728,8 @@ def chat(self, model: str, messages: list, stream: bool = True) -> tuple[str, st model=model, messages=messages, stream=stream, + # a stream reports usage only when asked, as OpenAI's does + **({"stream_options": {"include_usage": True}} if stream else {}), **self.completion_kwargs(), ) if stream: diff --git a/docs/docs/instructions/server/openapi.md b/docs/docs/instructions/server/openapi.md index ac5c7afd1..ec4cf0330 100644 --- a/docs/docs/instructions/server/openapi.md +++ b/docs/docs/instructions/server/openapi.md @@ -205,6 +205,22 @@ del stream, client gc.collect() ``` +As with OpenAI, a stream reports no token usage unless you ask for it with `stream_options={"include_usage": True}` (`stream_usage=True` in LangChain). Then every chunk carries `usage: null`, and one last chunk before `data: [DONE]` has an empty `choices` list and the usage of the whole request. On `/v1/chat/completions` that usage also includes FLM's timing and KV cache figures (`prefill_speed_tps`, `decoding_speed_tps`, `active_kv_tokens`, ...). Since that last chunk has no choices, guard `chunk.choices` before indexing it: + +```python +stream = client.chat.completions.create( + model="llama3.2:1b", + messages=[{"role": "user", "content": "Hi!"}], + stream=True, + stream_options={"include_usage": True}, +) +for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) + if chunk.usage: + print(f"\n{chunk.usage.prompt_tokens} prompt + {chunk.usage.completion_tokens} completion tokens") +``` + --- ## 📄 Example: Use a File as the Prompt diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index ee60b8009..1fb53e62d 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -219,6 +219,17 @@ static json try_parse_json_value(const std::string& s) { } } +///@brief Whether a streaming OpenAI request asked for usage through stream_options.include_usage. +/// Following OpenAI, a stream reports no usage unless asked; a malformed option counts as not asked. +static bool stream_include_usage(const json& request) { + auto options = request.find("stream_options"); + if (options == request.end() || !options->is_object()) { + return false; + } + auto include_usage = options->find("include_usage"); + return include_usage != options->end() && include_usage->is_boolean() && include_usage->get(); +} + ///@brief Convert OpenAI-style assistant tool_calls + following tool messages into the /// Gemma4 chat-template format, which expects a single assistant message containing /// both `tool_calls` and `tool_responses` (with `{name, response}` entries). @@ -1247,7 +1258,8 @@ void RestHandler::handle_openai_chat_completion(const json& request, json data_json = data; send_streaming_response(data_json, is_final); }; - streaming_ostream_openai_chat ostream(model, auto_chat_engine.get(), openai_stream_callback, parse_options); // streaming in chat completion format + streaming_ostream_openai_chat ostream(model, auto_chat_engine.get(), openai_stream_callback, parse_options, + stream_include_usage(request)); // streaming in chat completion format header_print("FLM", "Start prefill..."); try { @@ -1508,7 +1520,7 @@ void RestHandler::handle_openai_completion(const json& request, json data_json = data; send_streaming_response(data_json, is_final); }; - streaming_ostream_openai ostream(model, openai_stream_callback); // streaming in completion format + streaming_ostream_openai ostream(model, openai_stream_callback, stream_include_usage(request)); // streaming in completion format uniformed_input.prompt = prompt; try { bool success = auto_chat_engine->insert(meta_info, uniformed_input, [&] { return cancellation_token && cancellation_token->cancelled(); }); diff --git a/src/server/streaming_ostream_openai.hpp b/src/server/streaming_ostream_openai.hpp index d00774a9d..62e0a56cd 100644 --- a/src/server/streaming_ostream_openai.hpp +++ b/src/server/streaming_ostream_openai.hpp @@ -37,8 +37,11 @@ class streaming_buf_openai : public std::streambuf { ///@brief StreamCallback using StreamCallback = std::function; - streaming_buf_openai(const std::string& model, StreamCallback callback) - : model_name(model), stream_callback(callback), first_chunk(true) { + ///@param model the model name reported to the client + ///@param callback receives each server-sent event + ///@param include_usage stream_options.include_usage: send usage in its own chunk before [DONE] + streaming_buf_openai(const std::string& model, StreamCallback callback, bool include_usage) + : model_name(model), stream_callback(callback), include_usage(include_usage), first_chunk(true) { // Generate a unique ID for this stream generate_stream_id(); generate_created(); @@ -186,6 +189,9 @@ class streaming_buf_openai : public std::streambuf { } })} }; + if (include_usage) { + response["usage"] = nullptr; + } stream_callback("data: " + response.dump() + "\n\n", is_final); @@ -204,15 +210,31 @@ class streaming_buf_openai : public std::streambuf { { {"finish_reason", stop_reason_to_string(meta_info.stop_reason)}, } - })}, - {"usage", { - {"prompt_tokens", meta_info.prompt_tokens}, - {"prompt_tokens_details", {{"cached_tokens", meta_info.cached_prompt_tokens}}}, - {"completion_tokens", meta_info.generated_tokens}, - {"total_tokens", meta_info.prompt_tokens + meta_info.generated_tokens} - }} + })} }; + if (include_usage) { + // OpenAI: every chunk carries a null usage, then one extra chunk + // with no choices carries the usage of the whole request + final_response["usage"] = nullptr; + } stream_callback("data: " + final_response.dump() + "\n\n", false); + if (include_usage) { + json usage_response = { + {"id", stream_id}, + {"object", "text_completion"}, + {"created", created}, + {"system_fingerprint", system_fingerprint}, + {"model", model_name}, + {"choices", json::array()}, + {"usage", { + {"prompt_tokens", meta_info.prompt_tokens}, + {"prompt_tokens_details", {{"cached_tokens", meta_info.cached_prompt_tokens}}}, + {"completion_tokens", meta_info.generated_tokens}, + {"total_tokens", meta_info.prompt_tokens + meta_info.generated_tokens} + }} + }; + stream_callback("data: " + usage_response.dump() + "\n\n", false); + } // Send the [DONE] message stream_callback("data: [DONE]\n\n", true); } @@ -224,6 +246,8 @@ class streaming_buf_openai : public std::streambuf { std::string model_name; ///@brief Stream callback StreamCallback stream_callback; + ///@brief Whether the client asked for usage, see stream_options.include_usage + bool include_usage; ///@brief Stream ID std::string stream_id; long created; @@ -239,8 +263,8 @@ class streaming_buf_openai : public std::streambuf { ///@return the streaming ostream class streaming_ostream_openai : public std::ostream { public: - streaming_ostream_openai(const std::string& model, streaming_buf_openai::StreamCallback callback) - : std::ostream(&buf), buf(model, callback) {} + streaming_ostream_openai(const std::string& model, streaming_buf_openai::StreamCallback callback, bool include_usage) + : std::ostream(&buf), buf(model, callback, include_usage) {} ///@brief Finalize the chat void finalize(chat_meta_info_t& meta_info) { @@ -268,10 +292,11 @@ class streaming_buf_openai_chat : public std::streambuf { ///@param auto_chat_engine the engine generating the text ///@param callback receives each server-sent event ///@param options per request parsing options; the tools must outlive the stream + ///@param include_usage stream_options.include_usage: send usage in its own chunk before [DONE] streaming_buf_openai_chat(const std::string& model, AutoModel* auto_chat_engine, StreamCallback callback, - chat_parse_options options) + chat_parse_options options, bool include_usage) : model_name(model), auto_chat_engine(auto_chat_engine), stream_callback(callback), - parse_options(options) { + include_usage(include_usage), parse_options(options) { // Generate a unique ID for this stream generate_stream_id(); generate_created(); @@ -458,6 +483,9 @@ class streaming_buf_openai_chat : public std::streambuf { } })} }; + if (include_usage) { + response["usage"] = nullptr; + } stream_callback("data: " + response.dump() + "\n\n", false); } } @@ -479,24 +507,43 @@ class streaming_buf_openai_chat : public std::streambuf { //{"logprobs", nullptr}, {"finish_reason", stop_reason_to_string(meta_info.stop_reason)} } - })}, - {"usage", { - {"prompt_tokens", meta_info.prompt_tokens}, - {"prompt_tokens_details", {{"cached_tokens", meta_info.cached_prompt_tokens}}}, - {"completion_tokens", meta_info.generated_tokens}, - {"total_tokens", meta_info.prompt_tokens + meta_info.generated_tokens}, - {"active_kv_tokens", this->auto_chat_engine->get_current_context_length()}, - {"max_kv_token_capacity", this->auto_chat_engine->get_max_length()}, - {"kv_token_occupancy_rate_percentage", (float)this->auto_chat_engine->get_current_context_length() / (float)this->auto_chat_engine->get_max_length() * 100}, - {"load_duration", static_cast(meta_info.load_duration) / 1'000'000'000}, - {"prefill_duration_ttft", static_cast(meta_info.prefill_duration) / 1'000'000'000}, - {"decoding_duration", static_cast(meta_info.decoding_duration) / 1'000'000'000}, - {"prefill_speed_tps", static_cast(meta_info.prompt_tokens - meta_info.cached_prompt_tokens) / static_cast(meta_info.prefill_duration) * 1'000'000'000}, - {"decoding_speed_tps", static_cast(meta_info.generated_tokens) / static_cast(meta_info.decoding_duration) * 1'000'000'000}, - }} + })} + }; + json usage = { + {"prompt_tokens", meta_info.prompt_tokens}, + {"prompt_tokens_details", {{"cached_tokens", meta_info.cached_prompt_tokens}}}, + {"completion_tokens", meta_info.generated_tokens}, + {"total_tokens", meta_info.prompt_tokens + meta_info.generated_tokens}, + {"active_kv_tokens", this->auto_chat_engine->get_current_context_length()}, + {"max_kv_token_capacity", this->auto_chat_engine->get_max_length()}, + {"kv_token_occupancy_rate_percentage", (float)this->auto_chat_engine->get_current_context_length() / (float)this->auto_chat_engine->get_max_length() * 100}, + {"load_duration", static_cast(meta_info.load_duration) / 1'000'000'000}, + {"prefill_duration_ttft", static_cast(meta_info.prefill_duration) / 1'000'000'000}, + {"decoding_duration", static_cast(meta_info.decoding_duration) / 1'000'000'000}, + {"prefill_speed_tps", static_cast(meta_info.prompt_tokens - meta_info.cached_prompt_tokens) / static_cast(meta_info.prefill_duration) * 1'000'000'000}, + {"decoding_speed_tps", static_cast(meta_info.generated_tokens) / static_cast(meta_info.decoding_duration) * 1'000'000'000}, }; + if (include_usage) { + // OpenAI: every chunk carries a null usage, then one extra chunk + // with no choices carries the usage of the whole request + final_response["usage"] = nullptr; + } stream_callback("data: " + final_response.dump() + "\n\n", false); + if (include_usage) { + json usage_response = { + {"id", stream_id}, + {"object", "chat.completion.chunk"}, + {"created", created}, + {"model", model_name}, + {"system_fingerprint", system_fingerprint}, + {"choices", json::array()}, + {"usage", usage} + }; + stream_callback("data: " + usage_response.dump() + "\n\n", false); + } + // the server log shows the usage whether or not the client asked for it std::cout << "ChatCompletionChunk: " << final_response << std::endl; + std::cout << "Usage: " << usage << std::endl; // Send the [DONE] message stream_callback("data: [DONE]\n\n", true); } @@ -510,6 +557,8 @@ class streaming_buf_openai_chat : public std::streambuf { AutoModel* auto_chat_engine; ///@brief Stream callback StreamCallback stream_callback; + ///@brief Whether the client asked for usage, see stream_options.include_usage + bool include_usage; ///@brief Stream ID std::string stream_id; long created; @@ -524,8 +573,8 @@ class streaming_buf_openai_chat : public std::streambuf { class streaming_ostream_openai_chat : public std::ostream { public: streaming_ostream_openai_chat(const std::string& model, AutoModel* auto_chat_engine, streaming_buf_openai_chat::StreamCallback callback, - chat_parse_options options) - : std::ostream(&buf), buf(model, auto_chat_engine, callback, options) {} + chat_parse_options options, bool include_usage) + : std::ostream(&buf), buf(model, auto_chat_engine, callback, options, include_usage) {} ///@brief Call once the prompt is inserted, before generating void start(const chat_meta_info_t& meta_info) { diff --git a/src/test/openai_usage/test_usage_contract.py b/src/test/openai_usage/test_usage_contract.py index f13e3fe60..78f9d21a4 100755 --- a/src/test/openai_usage/test_usage_contract.py +++ b/src/test/openai_usage/test_usage_contract.py @@ -21,10 +21,10 @@ * ``usage.total_tokens == usage.prompt_tokens + usage.completion_tokens`` on every turn. -It deliberately does NOT assert anything about -``stream_options.include_usage`` or about ``usage`` being present (as -``null`` or otherwise) on intermediate streaming chunks -- that -behaviour belongs to a different, unmerged branch. +Streaming requests set ``stream_options.include_usage``, as an OpenAI +client must to get usage on a stream. The streaming test also checks +the shape OpenAI defines for it: every chunk carries ``usage``, null +on all but one final chunk that has no choices and the real usage. Usage ----- @@ -237,26 +237,44 @@ def extract_streamed_reply_and_usage( The assistant reply is reassembled by concatenating every ``choices[0].delta.content`` piece in order. The final usage is - taken from the last chunk that carries a non-null ``usage`` - object, per the OpenAI streaming contract. + taken from the last chunk, which per the OpenAI streaming contract + for ``stream_options.include_usage`` is the only one with a + non-null ``usage`` and has an empty ``choices`` list; every chunk + before it must carry ``usage: null``. """ + if not chunks: + raise AssertionError("the stream carried no chunks") content_parts = [] - final_usage = None - for chunk in chunks: + for index, chunk in enumerate(chunks): + if "usage" not in chunk: + raise AssertionError( + f"streamed chunk {index} has no 'usage' key although " + f"include_usage was set: {chunk!r}" + ) + is_last = index == len(chunks) - 1 + if not is_last and chunk["usage"] is not None: + raise AssertionError( + f"streamed chunk {index} carries a non-null 'usage' before " + f"the final usage chunk: {chunk!r}" + ) choices = chunk.get("choices") or [] if choices: delta = choices[0].get("delta") or {} piece = delta.get("content") if isinstance(piece, str): content_parts.append(piece) - usage = chunk.get("usage") - if isinstance(usage, dict): - final_usage = usage - if final_usage is None: + final_chunk = chunks[-1] + final_usage = final_chunk["usage"] + if not isinstance(final_usage, dict): raise AssertionError( - "no streamed chunk carried a non-null 'usage' object; " + "the final streamed chunk carries no 'usage' object; " f"chunks were: {chunks!r}" ) + if final_chunk.get("choices") != []: + raise AssertionError( + "the final usage chunk must have an empty 'choices' list: " + f"{final_chunk!r}" + ) return "".join(content_parts), final_usage @@ -393,6 +411,7 @@ def run_pinned_prefix_checks( } if stream: payload["stream"] = True + payload["stream_options"] = {"include_usage": True} body = post_json(url, payload, timeout) if stream: _, usage = extract_streamed_reply_and_usage(parse_sse_chunks(body)) @@ -469,6 +488,7 @@ def run_streaming_test(endpoint: str, model: str, timeout: float) -> None: "temperature": TEMPERATURE, "max_tokens": MAX_TOKENS, "stream": True, + "stream_options": {"include_usage": True}, } turn1_body = post_json(url, turn1_payload, timeout) turn1_chunks = parse_sse_chunks(turn1_body) From e74b13db2151d4b8ac1fee6e36d9f41dcd05019b Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Tue, 13 Oct 2026 15:50:45 -0700 Subject: [PATCH 07/18] Build xgrammar with /utf-8 under MSVC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xgrammar's sources hold UTF-8 literals, among them the SentencePiece "▁" that DetectMetadataFromHF compares tokenizer.json against. MSVC reads source in the system code page unless told otherwise, so those bytes only survived on machines whose code page happens to pass them through. Co-Authored-By: Claude --- src/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 847147adf..4743bf4e6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -298,6 +298,13 @@ target_include_directories(flm_xgrammar SYSTEM PUBLIC ${XGRAMMAR_DIR}/3rdparty/dlpack/include) target_compile_definitions(flm_xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) set_target_properties(flm_xgrammar PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(MSVC) + # Its sources hold UTF-8 literals (the SentencePiece "▁" that tokenizer + # metadata detection compares against, DeepSeek's "｜" markers). MSVC reads + # source in the system code page unless told otherwise, so without this the + # bytes survive only on machines whose code page happens to pass them through. + target_compile_options(flm_xgrammar PRIVATE /utf-8) +endif() # ——————————————————————————————————————————————— # Gather your sources From 403fc035a4c3d7d0fd4e409b07918e813492c3a9 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Tue, 13 Oct 2026 15:50:46 -0700 Subject: [PATCH 08/18] Keep reasoning out of the grammar when the model's thinking is off The constraint only dropped the optional reasoning block when the request said think=false or reasoning_effort=none. Gemma4 thinks only when asked, so with response_format set it was kept from its first choice of token and escaped into a <|channel>thought block, often spending the whole budget there and answering with empty content. AutoModel::thinking_enabled() reports the switch the chat template is rendered with; the six models that have one return it, and make_constraint turns allow_reasoning off when it is false. Co-Authored-By: Claude --- src/common/AutoModel/automodel.cpp | 9 ++++++++- src/include/AutoModel/automodel.hpp | 6 ++++++ src/include/AutoModel/modeling_gemma4_12b.hpp | 3 +++ src/include/AutoModel/modeling_gemma4e.hpp | 3 +++ src/include/AutoModel/modeling_qwen3.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_5vl.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_6_moe.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_8mtp.hpp | 4 ++++ 8 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 7b5f6b3bc..80c13e845 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -597,7 +597,14 @@ std::shared_ptr AutoModel::make_constraint(const chat_constra } return *this->grammar_engine_; }; - return chat_make_constraint(this->chat_format, request, vocab, engine); + // thinking turned off on the model, not just in the request: the grammar must + // not offer a reasoning block either, or a model kept from its first choice of + // token (Gemma4 before a JSON answer) escapes into one + chat_constraint_request effective = request; + if (!this->thinking_enabled()) { + effective.allow_reasoning = false; + } + return chat_make_constraint(this->chat_format, effective, vocab, engine); } std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index e15924bff..2c64efd21 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -497,6 +497,12 @@ class AutoModel { virtual bool check_using_checkpint() { return true; } + /// \brief Whether the chat template is rendered with thinking on + /// \note Only models with a thinking switch override this; the rest reason or + /// not as their template decides, which the constraint leaves alone. + virtual bool thinking_enabled() const { + return true; + } /// \brief Build the sampling constraint a request asks for /// \return null when the request needs no constraint diff --git a/src/include/AutoModel/modeling_gemma4_12b.hpp b/src/include/AutoModel/modeling_gemma4_12b.hpp index e7754a11e..a1232dbf8 100644 --- a/src/include/AutoModel/modeling_gemma4_12b.hpp +++ b/src/include/AutoModel/modeling_gemma4_12b.hpp @@ -107,6 +107,9 @@ class Gemma4_12B : public AutoModel { chat_template_type_t get_chat_template_type() override { return chat_template_type_t::gemma4; } + bool thinking_enabled() const override { + return this->enable_think; + } /// \brief Configure a parameter with type-erased value diff --git a/src/include/AutoModel/modeling_gemma4e.hpp b/src/include/AutoModel/modeling_gemma4e.hpp index 6ccff23c7..ac222de1b 100644 --- a/src/include/AutoModel/modeling_gemma4e.hpp +++ b/src/include/AutoModel/modeling_gemma4e.hpp @@ -130,6 +130,9 @@ class Gemma4e : public AutoModel { bool check_using_checkpint() { return this->enable_think; } + bool thinking_enabled() const override { + return this->enable_think; + } /// \brief Configure a parameter with type-erased value diff --git a/src/include/AutoModel/modeling_qwen3.hpp b/src/include/AutoModel/modeling_qwen3.hpp index 011691ab5..182d3e6b1 100644 --- a/src/include/AutoModel/modeling_qwen3.hpp +++ b/src/include/AutoModel/modeling_qwen3.hpp @@ -36,6 +36,9 @@ class Qwen3 : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + bool thinking_enabled() const override { + return this->enable_think; + } /// \brief Override configure_parameter to handle Qwen3-specific parameters bool configure_parameter(std::string parameter_name, const std::any& value) override { diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index 545920157..294412e4e 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -62,6 +62,9 @@ class Qwen3_5VL : public AutoModel { public: + bool thinking_enabled() const override { + return this->enable_think; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_6_moe.hpp b/src/include/AutoModel/modeling_qwen3_6_moe.hpp index 101325110..8225a4580 100644 --- a/src/include/AutoModel/modeling_qwen3_6_moe.hpp +++ b/src/include/AutoModel/modeling_qwen3_6_moe.hpp @@ -62,6 +62,9 @@ class Qwen3_6_MOE : public AutoModel { public: + bool thinking_enabled() const override { + return this->enable_think; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_8mtp.hpp b/src/include/AutoModel/modeling_qwen3_8mtp.hpp index ca117462c..9436f22ec 100644 --- a/src/include/AutoModel/modeling_qwen3_8mtp.hpp +++ b/src/include/AutoModel/modeling_qwen3_8mtp.hpp @@ -190,6 +190,10 @@ class Qwen3_8MTP : public AutoModel { public: + bool thinking_enabled() const override { + return this->enable_think; + } + /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter /// \param value the value to set (can be any type) From 4856ec8cdb95faea883f8f1aa006b0e280c63e97 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 14 Oct 2026 15:41:16 -0700 Subject: [PATCH 09/18] Move MiniCPM-V onto the shared chat library MiniCPM-V landed on main with its own reasoning and tool-call parsers and the single-token tool_choice mask. Drop both: its template is detected as qwen_xml with reasoning, and decode() samples through _sample() so tool_choice and response_format constraints hold. The think prefix that insert() holds back is reported to the constraint like Qwen3_5VL does. Co-Authored-By: Claude --- src/common/AutoModel/modeling_minicpm_v.cpp | 370 +------------------ src/create_new_model.md | 35 +- src/include/AutoModel/modeling_minicpm_v.hpp | 16 +- 3 files changed, 31 insertions(+), 390 deletions(-) diff --git a/src/common/AutoModel/modeling_minicpm_v.cpp b/src/common/AutoModel/modeling_minicpm_v.cpp index f1edf2153..b062539b8 100644 --- a/src/common/AutoModel/modeling_minicpm_v.cpp +++ b/src/common/AutoModel/modeling_minicpm_v.cpp @@ -317,7 +317,12 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std // The first prefix forward is left out of DECODING_TIME, as Qwen3_5VL does, // so `flm bench` measures decode speed the same way as for qwen3.5:0.8b. buffer y; + if (!this->enable_think) { + // fed for the KV cache only, it never reaches the output + this->_skip_insert_sample(meta_info); + } for (size_t i = 0; i < think_prefix.size(); i++) { + if (this->enable_think) this->_force_token(meta_info, think_prefix[i]); this->token_history.push_back(think_prefix[i]); if (i > 0) this->profiler_list[DECODING_TIME].start(); y = this->lm_engine->forward(think_prefix[i]); @@ -325,17 +330,14 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std this->total_tokens++; } if (this->enable_think) { - // Echo the opening tag so the stream parser enters reasoning mode. + // Echo the opening tag so the output parser enters reasoning mode. for (int id : think_prefix) { std::string token_str = this->tokenizer->run_time_decoder(id); result += token_str; os << token_str << std::flush; } } - // First real token: the model would open a tool call right here, so it - // needs the mask too. - this->_apply_tool_choice_mask(y, meta_info); - int sampled_token = this->sampler->sample(y); + int sampled_token = this->_sample(y, meta_info); while (true) { this->profiler_list[TKOEN_DECODE_TIME].start(); @@ -369,11 +371,6 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std } if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content - buffer_.clear(); - current_mode_ = StreamEventType::CONTENT; - tool_name_.clear(); - is_in_tool_block_ = false; break; } @@ -383,8 +380,7 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std this->total_tokens++; this->profiler_list[SAMPLING_TIME].start(); - this->_apply_tool_choice_mask(y, meta_info); - sampled_token = this->sampler->sample(y); + sampled_token = this->_sample(y, meta_info); this->profiler_list[SAMPLING_TIME].stop(1); } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; @@ -405,353 +401,3 @@ std::string MiniCPM_V::generate_with_prompt(chat_meta_info_t& meta_info, lm_unif // decode() knows to feed. return this->generate(meta_info, length_limit, os); } - -// Non-stream -NonStreamResult MiniCPM_V::parse_nstream_content(const std::string response_text) { - NonStreamResult result; - - const std::string think_start_tag = ""; - const std::string think_end_tag = ""; - std::string start_tag = ""; - std::string end_tag = ""; - std::string func_end_tag = ""; - std::string func_open = " fallback - size_t func_end_pos = answer_text.find(func_end_tag, block_content_start); - if (func_end_pos != std::string::npos) { - block_end = func_end_pos + func_end_tag.length(); - } else { - block_end = answer_text.length(); - } - search_from = block_end; - } - - std::string block = answer_text.substr(block_content_start, block_end - block_content_start); - - std::string tool_name; - size_t func_start = block.find(func_open); - if (func_start != std::string::npos) { - func_start += func_open.length(); - size_t func_name_end = block.find(">", func_start); - if (func_name_end != std::string::npos) { - tool_name = block.substr(func_start, func_name_end - func_start); - } - } - - nlohmann::json args = nlohmann::json::object(); - size_t pos = 0; - - while (true) { - size_t param_start = block.find(param_open, pos); - if (param_start == std::string::npos) break; - - param_start += param_open.length(); - size_t param_name_end = block.find(">", param_start); - if (param_name_end == std::string::npos) break; - - std::string param_name = block.substr(param_start, param_name_end - param_start); - size_t value_start = param_name_end + 1; - size_t value_end = block.find(param_close, value_start); - - size_t next_param_pos = block.find(param_open, value_start); - size_t func_boundary_pos = block.find(func_end_tag, value_start); - - auto use_earlier_boundary = [&value_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { - value_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - - if (value_end == std::string::npos) { - value_end = block.length(); - } - - std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); - - try { - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - pos = value_end; - if (block.compare(value_end, param_close.length(), param_close) == 0) { - pos += param_close.length(); - } - } - - result.tool_calls_list.emplace_back(tool_name, args.dump()); - } - - if (result.tool_calls_list.empty()) { - result.content = trim_tool_value(answer_text); - } else { - // Populate legacy single-tool fields from the first call for backward compatibility - result.tool_name = result.tool_calls_list[0].first; - result.tool_args = result.tool_calls_list[0].second; - // Extract content before the first - size_t first_tool = answer_text.find(start_tag); - if (first_tool != std::string::npos && first_tool > 0) { - result.content = trim_tool_value(answer_text.substr(0, first_tool)); - } - } - - return result; -} - -// Stream -StreamResult MiniCPM_V::parse_stream_content(const std::string content) { - return parse_stream_content_impl(content, false); -} - -StreamResult MiniCPM_V::parse_stream_content_final(const std::string content) { - return parse_stream_content_impl(content, true); -} - -StreamResult MiniCPM_V::parse_stream_content_impl(const std::string content, bool is_final) { - const std::string MARKER_THINK_START = ""; - const std::string MARKER_THINK_END = ""; - const std::string MARKER_TOOL_START = ""; - const std::string MARKER_TOOL_END = ""; - const std::string MARKER_FUNC_END = ""; - - - StreamResult result; - buffer_ += content; - - while (true) { - if (!is_in_tool_block_) { - size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); - if (stray_end_pos != std::string::npos) { - buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); - } - } - - if (!is_in_tool_block_) { - size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); - if (tool_start_pos != std::string::npos) { - if (tool_start_pos > 0) { - result.content = buffer_.substr(0, tool_start_pos); - result.type = current_mode_; - buffer_ = buffer_.substr(tool_start_pos); - return result; - } - - is_in_tool_block_ = true; - buffer_ = buffer_.substr(MARKER_TOOL_START.length()); - result.type = StreamEventType::WAITING; - return result; - } - } - - // tool calling process - if (is_in_tool_block_) { - size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); - size_t func_end_pos = buffer_.find(MARKER_FUNC_END); - - if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { - size_t actual_end_pos = buffer_.size(); - size_t skip_length = 0; - - if (tool_end_pos != std::string::npos) { - actual_end_pos = tool_end_pos; - skip_length = MARKER_TOOL_END.length(); - } - else if (func_end_pos != std::string::npos) { - actual_end_pos = func_end_pos; - skip_length = MARKER_FUNC_END.length(); - } - - std::string block = buffer_.substr(0, actual_end_pos + skip_length); - buffer_ = buffer_.substr(actual_end_pos + skip_length); - is_in_tool_block_ = false; - - try { - result.type = StreamEventType::TOOL_DONE; - result.tool_id = "call_" + std::to_string(std::time(nullptr)); - - // parse function name - std::string func_open = "", func_start); - if (func_end != std::string::npos) { - result.tool_name = block.substr(func_start, func_end - func_start); - } - } - - // parse parameters - nlohmann::json args = nlohmann::json::object(); - std::string param_open = "", p_start); - if (p_name_end == std::string::npos) break; - std::string param_name = block.substr(p_start, p_name_end - p_start); - - size_t val_start = p_name_end + 1; - if (val_start < block.size() && block[val_start] == '\n') val_start++; - - size_t param_close_pos = block.find(param_close, val_start); - size_t val_end = param_close_pos; - - size_t next_param_pos = block.find(param_open, val_start); - size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); - size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); - - auto use_earlier_boundary = [&val_end](size_t boundary_pos) { - if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { - val_end = boundary_pos; - } - }; - - use_earlier_boundary(next_param_pos); - use_earlier_boundary(func_boundary_pos); - use_earlier_boundary(tool_boundary_pos); - - if (val_end == std::string::npos && is_final) { - val_end = block.size(); - } - if (val_end == std::string::npos) break; - - std::string param_value = block.substr(val_start, val_end - val_start); - - // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter - while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { - param_value.pop_back(); - } - - try { - // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) - args[param_name] = nlohmann::json::parse(param_value); - } - catch (...) { - args[param_name] = param_value; - } - - search_pos = param_close_pos != std::string::npos && val_end == param_close_pos - ? val_end + param_close.length() - : val_end; - } - result.tool_args_str = args.dump(); - return result; - } - catch (...) { - result.type = StreamEventType::CONTENT; - result.content = "[Error parsing tool call]"; - return result; - } - } - else { - result.type = StreamEventType::WAITING; - return result; - } - } - - if (current_mode_ == StreamEventType::CONTENT) { - size_t think_start_pos = buffer_.find(MARKER_THINK_START); - if (think_start_pos != std::string::npos) { - if (think_start_pos > 0) { - result.content = buffer_.substr(0, think_start_pos); - result.type = StreamEventType::CONTENT; - buffer_ = buffer_.substr(think_start_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_START.length()); - current_mode_ = StreamEventType::REASONING; - continue; - } - } - else if (current_mode_ == StreamEventType::REASONING) { - size_t think_end_pos = buffer_.find(MARKER_THINK_END); - if (think_end_pos != std::string::npos) { - if (think_end_pos > 0) { - result.content = buffer_.substr(0, think_end_pos); - result.type = StreamEventType::REASONING; - buffer_ = buffer_.substr(think_end_pos); - return result; - } - buffer_ = buffer_.substr(MARKER_THINK_END.length()); - current_mode_ = StreamEventType::CONTENT; - continue; - } - } - - if (!buffer_.empty()) { - size_t last_lt = buffer_.rfind('<'); - // If '<' appears at the end (possibly an incomplete or tag) - if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { - if (last_lt > 0) { - // Only output the content before '<' - result.content = buffer_.substr(0, last_lt); - result.type = current_mode_; - buffer_ = buffer_.substr(last_lt); - return result; - } else { - // If '<' is the first character in the buffer, directly wait for the next chunk - result.type = StreamEventType::WAITING; - return result; - } - } - - result.content = buffer_; - result.type = current_mode_; - buffer_.clear(); - return result; - } - - break; - } - - result.type = current_mode_; - return result; -} diff --git a/src/create_new_model.md b/src/create_new_model.md index d92e3351d..7e34e9743 100644 --- a/src/create_new_model.md +++ b/src/create_new_model.md @@ -25,9 +25,10 @@ Every model has two layers: (`lib_npu.so` / `_npu.dll`). You normally do **not** write this; it comes from the kernel/IRON project. 2. **The AutoModel wrapper** — `class : public AutoModel`. This is what you - write. It owns the tokenizer, sampler, chat template, prompt cache and the - streaming / tool-call parsers, and drives the engine through - `prefill()` / `forward()` / `checkpoint()` / `restore()`. + write. It owns the tokenizer, sampler, chat template and prompt cache, and + drives the engine through `prefill()` / `forward()` / `checkpoint()` / `restore()`. + Reasoning and tool-call parsing is not per model: the shared chat library + (`common/chat/`) detects the format from the chat template. --- @@ -93,14 +94,11 @@ void load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) override; chat_template_type_t get_chat_template_type() override; // chat_ml | harmony | gemma4 bool configure_parameter(std::string name, const std::any& value) override; -NonStreamResult parse_nstream_content(const std::string response_text) override; -StreamResult parse_stream_content(const std::string content) override; -StreamResult parse_stream_content_final(const std::string content) override; +bool thinking_enabled() const override; // models with a thinking switch ``` -Also declare the private helpers the base class expects you to supply: -`setup_tokenizer(std::string model_path)` and, if you have a streaming parser, a -shared `parse_stream_content_impl(const std::string, bool is_final)`. +Also declare the private helper the base class expects you to supply: +`setup_tokenizer(std::string model_path)`. --- @@ -182,12 +180,21 @@ keys per message rather than failing. restore. `generate_with_prompt` is just `insert()` → optional `"\n"` banner → `generate()`. -### 2g. Stream & tool-call parsers +A model with its own decode loop (e.g. one that holds the think prefix back from +`insert()` and feeds it in `generate()`, like `modeling_qwen3_5vl.cpp` and +`modeling_minicpm_v.cpp`) must keep the decoding constraint in step: -Map the model's marker vocabulary to `StreamResult` / `NonStreamResult`. Gemma4, for -example, uses `<|channel>thought` / `` for reasoning and -`<|tool_call>` / `` for tool calls, with relaxed-JSON arguments delimited -by `<|"|>` that need a normalizing rewrite pass before `nlohmann::json::parse`. +- sample every token with `_sample(y, meta_info)`, never `sampler->sample()` directly; +- report each preamble token that is part of the output with `_force_token()`; +- call `_skip_insert_sample()` before feeding preamble tokens that never reach the + output (the empty think block when thinking is off). + +### 2g. Reasoning & tool-call format + +Nothing to write if the chat template uses a format `chat_format_detect()` in +`common/chat/chat_format.cpp` already knows (``, Hermes JSON or Qwen XML +``, LFM2, Gemma4, Harmony). The server log prints the detected one at load +(`Chat format: ...`); a new marker vocabulary is added there, not in the model. --- diff --git a/src/include/AutoModel/modeling_minicpm_v.hpp b/src/include/AutoModel/modeling_minicpm_v.hpp index 34c6b61d3..c7d32bf41 100644 --- a/src/include/AutoModel/modeling_minicpm_v.hpp +++ b/src/include/AutoModel/modeling_minicpm_v.hpp @@ -30,7 +30,6 @@ class MiniCPM_V : public AutoModel { bool enable_tool = false; static constexpr int think_start_id = 248068; static constexpr int think_end_id = 248069; - static constexpr int tool_start_token_id = 248058; void setup_tokenizer(std::string model_path); @@ -65,20 +64,9 @@ class MiniCPM_V : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; - NonStreamResult parse_nstream_content(const std::string response_text) override; - StreamResult parse_stream_content(const std::string content) override; - StreamResult parse_stream_content_final(const std::string content) override; -private: - StreamResult parse_stream_content_impl(const std::string content, bool is_final); - -public: - - /// \note MiniCPM-V opens every tool call with , so masking that - /// one token is enough to honour tool_choice=none. Gated on enable_tool - /// to match apply_chat_template, which only renders tools when it is set. - int get_tool_start_token_id() const override { - return this->enable_tool ? tool_start_token_id : -1; + bool thinking_enabled() const override { + return this->enable_think; } /// \brief Configure a parameter with type-erased value From 5f44227e5d3628de3ac26f95dbc1e2372ae60f4f Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 14 Oct 2026 16:23:25 -0700 Subject: [PATCH 10/18] Build the chat library into the test harnesses Every harness that compiles automodel.cpp stopped linking once it started parsing and constraining through the chat library. The xgrammar build moves into common/chat/chat_sources.cmake, which flm, add_npu_test and the rai frontend tests now share; the Linux Makefiles add the chat sources and link the main build's xgrammar. The rai Sampler stubs take the new mask argument. Co-Authored-By: Claude --- src/CMakeLists.txt | 23 ++--------- src/common/chat/chat_sources.cmake | 40 ++++++++++++++++++++ src/test/CMakeLists.txt | 7 +++- src/test/common.mk | 8 ++++ src/test/gemma4_12b/Makefile | 1 + src/test/gemma4_rai/CMakeLists.txt | 7 +++- src/test/gemma4_rai/test_gemma4_frontend.cpp | 2 +- src/test/gemma4e_flash/Makefile | 1 + src/test/gemma4e_npu/Makefile | 1 + src/test/gemma_npu/Makefile | 1 + src/test/gemma_text_npu/Makefile | 1 + src/test/gpt_oss_npu/Makefile | 1 + src/test/hunyuan_npu/Makefile | 1 + src/test/lfm2_npu/Makefile | 1 + src/test/llama_npu/Makefile | 1 + src/test/minicpm_v_4_7_npu/Makefile | 1 + src/test/nanbeige_npu/Makefile | 1 + src/test/phi4_npu/Makefile | 1 + src/test/phi4_rai/CMakeLists.txt | 12 ++++-- src/test/phi4_rai/test_phi4_frontend.cpp | 2 +- src/test/qwen2_npu/Makefile | 1 + src/test/qwen2vl_npu/Makefile | 1 + src/test/qwen3_5_omni_npu/Makefile | 1 + src/test/qwen3_5vl_npu/Makefile | 1 + src/test/qwen3_6_moe_npu/Makefile | 1 + src/test/qwen3_8mtp_npu/Makefile | 1 + src/test/qwen3_npu/Makefile | 1 + src/test/qwen3vl_flash/Makefile | 1 + src/test/qwen3vl_npu/Makefile | 1 + 29 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 src/common/chat/chat_sources.cmake diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 40734c4c3..404fdbb1a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -290,27 +290,10 @@ add_subdirectory(${CMAKE_SOURCE_DIR}/../third_party/tokenizers-cpp # ——————————————————————————————————————————————— # xgrammar: the grammar engine behind constrained decoding (tool_choice, -# response_format). Built from its sources as a plain static library rather than -# through its own CMakeLists, which would add install rules and -Werror. +# response_format), shared with the test harnesses. The chat sources themselves +# come in with the common/*/*.cpp glob below. # ——————————————————————————————————————————————— -set(XGRAMMAR_DIR ${CMAKE_SOURCE_DIR}/../third_party/xgrammar) -file(GLOB_RECURSE XGRAMMAR_SOURCES ${XGRAMMAR_DIR}/cpp/*.cc) -list(FILTER XGRAMMAR_SOURCES EXCLUDE REGEX "/cpp/tvm_ffi/.*\\.cc$") -add_library(flm_xgrammar STATIC ${XGRAMMAR_SOURCES}) -target_compile_features(flm_xgrammar PUBLIC cxx_std_17) -target_include_directories(flm_xgrammar PUBLIC ${XGRAMMAR_DIR}/include) -target_include_directories(flm_xgrammar SYSTEM PUBLIC - ${XGRAMMAR_DIR}/3rdparty/picojson - ${XGRAMMAR_DIR}/3rdparty/dlpack/include) -target_compile_definitions(flm_xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) -set_target_properties(flm_xgrammar PROPERTIES POSITION_INDEPENDENT_CODE ON) -if(MSVC) - # Its sources hold UTF-8 literals (the SentencePiece "▁" that tokenizer - # metadata detection compares against, DeepSeek's "｜" markers). MSVC reads - # source in the system code page unless told otherwise, so without this the - # bytes survive only on machines whose code page happens to pass them through. - target_compile_options(flm_xgrammar PRIVATE /utf-8) -endif() +include("${CMAKE_SOURCE_DIR}/common/chat/chat_sources.cmake") # ——————————————————————————————————————————————— # Gather your sources diff --git a/src/common/chat/chat_sources.cmake b/src/common/chat/chat_sources.cmake new file mode 100644 index 000000000..ea03a38a9 --- /dev/null +++ b/src/common/chat/chat_sources.cmake @@ -0,0 +1,40 @@ +# The shared chat library and the grammar engine under it, for every target that +# compiles automodel.cpp: flm itself and the standalone test harnesses. +# +# Sets FLM_CHAT_SOURCES and defines the flm_xgrammar static library once per +# build tree; link it alongside the sources. + +set(FLM_CHAT_SOURCES + "${CMAKE_CURRENT_LIST_DIR}/chat_args.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_constraint.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_format.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_grammar.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_msg.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_parser.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_stream.cpp") + +# ——————————————————————————————————————————————— +# xgrammar: the grammar engine behind constrained decoding (tool_choice, +# response_format). Built from its sources as a plain static library rather than +# through its own CMakeLists, which would add install rules and -Werror. +# ——————————————————————————————————————————————— +if(NOT TARGET flm_xgrammar) + get_filename_component(XGRAMMAR_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../third_party/xgrammar" ABSOLUTE) + file(GLOB_RECURSE XGRAMMAR_SOURCES ${XGRAMMAR_DIR}/cpp/*.cc) + list(FILTER XGRAMMAR_SOURCES EXCLUDE REGEX "/cpp/tvm_ffi/.*\\.cc$") + add_library(flm_xgrammar STATIC ${XGRAMMAR_SOURCES}) + target_compile_features(flm_xgrammar PUBLIC cxx_std_17) + target_include_directories(flm_xgrammar PUBLIC ${XGRAMMAR_DIR}/include) + target_include_directories(flm_xgrammar SYSTEM PUBLIC + ${XGRAMMAR_DIR}/3rdparty/picojson + ${XGRAMMAR_DIR}/3rdparty/dlpack/include) + target_compile_definitions(flm_xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) + set_target_properties(flm_xgrammar PROPERTIES POSITION_INDEPENDENT_CODE ON) + if(MSVC) + # Its sources hold UTF-8 literals (the SentencePiece "▁" that tokenizer + # metadata detection compares against, DeepSeek's "｜" markers). MSVC reads + # source in the system code page unless told otherwise, so without this the + # bytes survive only on machines whose code page happens to pass them through. + target_compile_options(flm_xgrammar PRIVATE /utf-8) + endif() +endif() diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 8d9258555..0d47f09fc 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -28,7 +28,9 @@ function(add_npu_test TARGET_NAME OUTPUT_SUBDIR) set(base_sources "") if(NPU_TEST_USE_AUTOMODEL) - list(APPEND base_sources ${CMAKE_SOURCE_DIR}/../../common/AutoModel/automodel.cpp) + # automodel.cpp parses output and constrains decoding with the chat library + include(${CMAKE_SOURCE_DIR}/../../common/chat/chat_sources.cmake) + list(APPEND base_sources ${CMAKE_SOURCE_DIR}/../../common/AutoModel/automodel.cpp ${FLM_CHAT_SOURCES}) endif() if(NPU_TEST_USE_TOKENIZER) list(APPEND base_sources ${CMAKE_SOURCE_DIR}/../../common/tokenizer/tokenizer.cpp) @@ -118,4 +120,7 @@ function(add_npu_test TARGET_NAME OUTPUT_SUBDIR) ntdll wsock32 ws2_32 Bcrypt iphlpapi userenv psapi ) + if(NPU_TEST_USE_AUTOMODEL) + target_link_libraries(${TARGET_NAME} PRIVATE flm_xgrammar) + endif() endfunction() diff --git a/src/test/common.mk b/src/test/common.mk index 12fddcd13..ffb266bc2 100644 --- a/src/test/common.mk +++ b/src/test/common.mk @@ -77,6 +77,14 @@ LDFLAGS += -ltokenizers_cpp -ltokenizers_c -lsentencepiece LDFLAGS += $(RUNTIME_LDFLAGS) DEPENDENCY_LDFLAGS += -lmha -ldequant -lgemm -llm_head -lq4_npu_eXpress +# automodel.cpp parses output and constrains decoding with the chat library: a +# harness that builds it adds $(CHAT_SOURCES), and gets the main build's xgrammar +CHAT_SOURCES := $(wildcard ../../common/chat/*.cpp) +XGRAMMAR_DIR := ../../../third_party/xgrammar +CXX_FLAGS += -I$(XGRAMMAR_DIR)/include -isystem $(XGRAMMAR_DIR)/3rdparty/dlpack/include +CXX_FLAGS += -DXGRAMMAR_ENABLE_CPPTRACE=0 +DEPENDENCY_LDFLAGS += $(if $(filter %/automodel.cpp,$(SOURCES)),-L../../build -lflm_xgrammar) + SOURCES += ../../common/utils.cpp diff --git a/src/test/gemma4_12b/Makefile b/src/test/gemma4_12b/Makefile index 522cf5c5a..03890f325 100644 --- a/src/test/gemma4_12b/Makefile +++ b/src/test/gemma4_12b/Makefile @@ -20,6 +20,7 @@ TYPE ?= 3 SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_gemma4_12b.cpp SOURCES += ../../common/AutoModel/modeling_gemma4_12b_image.cpp SOURCES += ../../common/AutoModel/modeling_gemma4_12b_audio.cpp diff --git a/src/test/gemma4_rai/CMakeLists.txt b/src/test/gemma4_rai/CMakeLists.txt index 2c224060c..318b7f816 100644 --- a/src/test/gemma4_rai/CMakeLists.txt +++ b/src/test/gemma4_rai/CMakeLists.txt @@ -77,11 +77,14 @@ if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/test_gemma4_frontend.cpp") HINTS "$ENV{CONDA_PREFIX}/Library/include" "$ENV{USERPROFILE}/anaconda3/Library/include") if(XRT_INCLUDE_DIR AND BOOST_INCLUDE_DIR) + # automodel.cpp parses output and constrains decoding with the chat library + include("${FLM_SOURCE_DIR}/common/chat/chat_sources.cmake") add_executable(test_gemma4_frontend test_gemma4_frontend.cpp "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" - "${FLM_SOURCE_DIR}/common/AutoModel/modeling_gemma4e.cpp") + "${FLM_SOURCE_DIR}/common/AutoModel/modeling_gemma4e.cpp" + ${FLM_CHAT_SOURCES}) target_include_directories(test_gemma4_frontend PRIVATE "${CMAKE_CURRENT_LIST_DIR}" "${FLM_SOURCE_DIR}/include" @@ -93,7 +96,7 @@ if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/test_gemma4_frontend.cpp") target_compile_options(test_gemma4_frontend PRIVATE $<$:/wd4005 /wd4244>) target_link_directories(test_gemma4_frontend PRIVATE "${XRT_INCLUDE_DIR}/../lib") - target_link_libraries(test_gemma4_frontend PRIVATE xrt_coreutil) + target_link_libraries(test_gemma4_frontend PRIVATE xrt_coreutil flm_xgrammar) add_test(NAME test_gemma4_frontend COMMAND test_gemma4_frontend) else() # LOUD AT CONFIGURE, AND RED AT CTEST -- the treatment C6 gave diff --git a/src/test/gemma4_rai/test_gemma4_frontend.cpp b/src/test/gemma4_rai/test_gemma4_frontend.cpp index 23cdc24af..172a95a11 100644 --- a/src/test/gemma4_rai/test_gemma4_frontend.cpp +++ b/src/test/gemma4_rai/test_gemma4_frontend.cpp @@ -257,7 +257,7 @@ Sampler::Sampler(int features, sampler_config& config) logits.resize(1); counters.resize(1); token_positions.resize(1, -1); } void Sampler::reset_penalties() {} -int Sampler::sample(buffer&) { +int Sampler::sample(buffer&, const std::vector*) { if (g_sample_index < g_samples.size()) return g_samples[g_sample_index++]; return 7; } diff --git a/src/test/gemma4e_flash/Makefile b/src/test/gemma4e_flash/Makefile index a67fd2f10..75aea3f93 100644 --- a/src/test/gemma4e_flash/Makefile +++ b/src/test/gemma4e_flash/Makefile @@ -18,6 +18,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_gemma4e.cpp SOURCES += ../../common/AutoModel/modeling_gemma4e_image.cpp SOURCES += ../../common/AutoModel/modeling_gemma4e_audio.cpp diff --git a/src/test/gemma4e_npu/Makefile b/src/test/gemma4e_npu/Makefile index 47190eb7a..1f77cd438 100644 --- a/src/test/gemma4e_npu/Makefile +++ b/src/test/gemma4e_npu/Makefile @@ -18,6 +18,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_gemma4e.cpp SOURCES += ../../common/AutoModel/modeling_gemma4e_image.cpp SOURCES += ../../common/AutoModel/modeling_gemma4e_audio.cpp diff --git a/src/test/gemma_npu/Makefile b/src/test/gemma_npu/Makefile index 0065c7a3b..df1bc31f6 100644 --- a/src/test/gemma_npu/Makefile +++ b/src/test/gemma_npu/Makefile @@ -19,6 +19,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_gemma3.cpp SOURCES += ../../common/AutoModel/modeling_gemma3_image.cpp SOURCES += ../../common/image/image_reader.cpp diff --git a/src/test/gemma_text_npu/Makefile b/src/test/gemma_text_npu/Makefile index bc7b4e26a..be08774a7 100644 --- a/src/test/gemma_text_npu/Makefile +++ b/src/test/gemma_text_npu/Makefile @@ -16,6 +16,7 @@ -include ../common.mk SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_gemma3_text.cpp) SOURCES += $(wildcard ../../common/tokenizer/tokenizer.cpp) SOURCES += $(wildcard ../../common/modules/sampler.cpp) diff --git a/src/test/gpt_oss_npu/Makefile b/src/test/gpt_oss_npu/Makefile index 59b5d035a..06519affa 100644 --- a/src/test/gpt_oss_npu/Makefile +++ b/src/test/gpt_oss_npu/Makefile @@ -17,6 +17,7 @@ SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_gpt_oss.cpp) SOURCES += $(wildcard ../../include/AutoModel/automodel.hpp) SOURCES += $(wildcard ../../include/AutoModel/modeling_gpt_oss.hpp) diff --git a/src/test/hunyuan_npu/Makefile b/src/test/hunyuan_npu/Makefile index 089111767..97f94bd4b 100644 --- a/src/test/hunyuan_npu/Makefile +++ b/src/test/hunyuan_npu/Makefile @@ -37,6 +37,7 @@ endif SOURCES += test.cpp SOURCES += bench_prefill.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_hunyuan.cpp SOURCES += ../../common/tokenizer/tokenizer.cpp SOURCES += ../../common/modules/sampler.cpp diff --git a/src/test/lfm2_npu/Makefile b/src/test/lfm2_npu/Makefile index b578ff628..53c6fd7b9 100644 --- a/src/test/lfm2_npu/Makefile +++ b/src/test/lfm2_npu/Makefile @@ -17,6 +17,7 @@ SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_lfm2.cpp) SOURCES += $(wildcard ../../include/AutoModel/automodel.hpp) SOURCES += $(wildcard ../../include/AutoModel/modeling_lfm2.hpp) diff --git a/src/test/llama_npu/Makefile b/src/test/llama_npu/Makefile index b2fbe0689..c9e496716 100644 --- a/src/test/llama_npu/Makefile +++ b/src/test/llama_npu/Makefile @@ -17,6 +17,7 @@ SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_llama3.cpp) SOURCES += $(wildcard ../../common/tokenizer/tokenizer.cpp) SOURCES += $(wildcard ../../common/modules/sampler.cpp) diff --git a/src/test/minicpm_v_4_7_npu/Makefile b/src/test/minicpm_v_4_7_npu/Makefile index dd6a66aba..2a4006180 100644 --- a/src/test/minicpm_v_4_7_npu/Makefile +++ b/src/test/minicpm_v_4_7_npu/Makefile @@ -15,6 +15,7 @@ LENGTH ?= 256 SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/model_backend.cpp SOURCES += ../../common/AutoModel/modeling_minicpm_v.cpp # Image support. modeling_minicpm_v_image.cpp holds the slicing, the PIL-exact diff --git a/src/test/nanbeige_npu/Makefile b/src/test/nanbeige_npu/Makefile index 0a2059614..43ed37664 100644 --- a/src/test/nanbeige_npu/Makefile +++ b/src/test/nanbeige_npu/Makefile @@ -17,6 +17,7 @@ SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_nanbeige.cpp) SOURCES += $(wildcard ../../common/tokenizer/tokenizer.cpp) SOURCES += $(wildcard ../../common/modules/sampler.cpp) diff --git a/src/test/phi4_npu/Makefile b/src/test/phi4_npu/Makefile index 641f9c83a..6f77fdef5 100644 --- a/src/test/phi4_npu/Makefile +++ b/src/test/phi4_npu/Makefile @@ -16,6 +16,7 @@ -include ../common.mk SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_phi4.cpp) SOURCES += $(wildcard ../../include/AutoModel/automodel.hpp) SOURCES += $(wildcard ../../include/AutoModel/modeling_phi4.hpp) diff --git a/src/test/phi4_rai/CMakeLists.txt b/src/test/phi4_rai/CMakeLists.txt index 9a84ae539..032c31f7a 100644 --- a/src/test/phi4_rai/CMakeLists.txt +++ b/src/test/phi4_rai/CMakeLists.txt @@ -121,10 +121,14 @@ target_link_libraries(test_phi4_engine PRIVATE xrt_coreutil) # The backend registry is part of the frontend now; builtin_backends.cpp is not, # because it names every prebuilt engine library. test_phi4_frontend.cpp supplies # its own empty register_builtin_backends and registers stubs instead. +# automodel.cpp parses output and constrains decoding with the chat library; +# every target built from these sources also links flm_xgrammar. +include("${FLM_SOURCE_DIR}/common/chat/chat_sources.cmake") set(PHI4_FRONTEND_SOURCES "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" - "${FLM_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp") + "${FLM_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp" + ${FLM_CHAT_SOURCES}) add_executable(test_phi4_frontend test_phi4_frontend.cpp fake_corelib.cpp @@ -149,7 +153,7 @@ target_compile_definitions(test_phi4_frontend PRIVATE target_compile_options(test_phi4_frontend PRIVATE $<$:/wd4005 /wd4244>) target_link_directories(test_phi4_frontend PRIVATE "${XRT_INCLUDE_DIR}/../lib") -target_link_libraries(test_phi4_frontend PRIVATE xrt_coreutil) +target_link_libraries(test_phi4_frontend PRIVATE xrt_coreutil flm_xgrammar) add_executable(test_model_downloader test_model_downloader.cpp @@ -193,7 +197,7 @@ target_compile_definitions(test_phi4_frontend_off PRIVATE target_compile_options(test_phi4_frontend_off PRIVATE $<$:/wd4005 /wd4244>) target_link_directories(test_phi4_frontend_off PRIVATE "${XRT_INCLUDE_DIR}/../lib") -target_link_libraries(test_phi4_frontend_off PRIVATE xrt_coreutil) +target_link_libraries(test_phi4_frontend_off PRIVATE xrt_coreutil flm_xgrammar) # Compile the actual production frontend translation units in both feature modes. # Empty declaration-only FFmpeg headers isolate this compile check from an @@ -237,6 +241,8 @@ function(add_frontend_compile_guard TARGET_NAME ENABLE_CORELIB) CMAKE_XCLBIN_PREFIX="${FLM_SOURCE_DIR}/xclbins") target_compile_options(${TARGET_NAME} PRIVATE $<$:/wd4005 /wd4244>) + # for xgrammar's headers, which chat_grammar.cpp includes + target_link_libraries(${TARGET_NAME} PRIVATE flm_xgrammar) if(ENABLE_CORELIB) target_sources(${TARGET_NAME} PRIVATE "${FLM_SOURCE_DIR}/common/models/phi4/rai/aie_next/phi4_rai_backend.cpp") diff --git a/src/test/phi4_rai/test_phi4_frontend.cpp b/src/test/phi4_rai/test_phi4_frontend.cpp index 83d0e1ef5..aa6ed5ad1 100644 --- a/src/test/phi4_rai/test_phi4_frontend.cpp +++ b/src/test/phi4_rai/test_phi4_frontend.cpp @@ -288,7 +288,7 @@ Sampler::Sampler(int features, sampler_config& config) logits.resize(1); counters.resize(1); token_positions.resize(1, -1); } void Sampler::reset_penalties() {} -int Sampler::sample(buffer& logits) { +int Sampler::sample(buffer& logits, const std::vector*) { // FakeEngine hands back logits it allocated, as the rai engines do. buffer's // copy is shallow and non-owning, so a buffer that no longer owns them by the // time they are sampled is reading memory its owner has already freed. diff --git a/src/test/qwen2_npu/Makefile b/src/test/qwen2_npu/Makefile index 3e24221f6..c0b8c6ae4 100644 --- a/src/test/qwen2_npu/Makefile +++ b/src/test/qwen2_npu/Makefile @@ -16,6 +16,7 @@ -include ../common.mk SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_qwen2.cpp) SOURCES += $(wildcard ../../common/tokenizer/tokenizer.cpp) SOURCES += $(wildcard ../../common/modules/sampler.cpp) diff --git a/src/test/qwen2vl_npu/Makefile b/src/test/qwen2vl_npu/Makefile index 8dc49765f..ba5a2e953 100644 --- a/src/test/qwen2vl_npu/Makefile +++ b/src/test/qwen2vl_npu/Makefile @@ -18,6 +18,7 @@ SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen2vl.cpp SOURCES += ../../common/AutoModel/modeling_qwen2vl_image.cpp SOURCES += ../../common/image_process_utils/imageproc.cpp diff --git a/src/test/qwen3_5_omni_npu/Makefile b/src/test/qwen3_5_omni_npu/Makefile index 1453e2764..cf6b65764 100644 --- a/src/test/qwen3_5_omni_npu/Makefile +++ b/src/test/qwen3_5_omni_npu/Makefile @@ -20,6 +20,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen3_5_omni.cpp SOURCES += ../../common/AutoModel/modeling_qwen3_5_omni_image.cpp SOURCES += ../../common/AutoModel/modeling_qwen3_5_omni_audio.cpp diff --git a/src/test/qwen3_5vl_npu/Makefile b/src/test/qwen3_5vl_npu/Makefile index a5045fd2a..7acd96a4b 100644 --- a/src/test/qwen3_5vl_npu/Makefile +++ b/src/test/qwen3_5vl_npu/Makefile @@ -18,6 +18,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen3_5vl.cpp SOURCES += ../../common/AutoModel/modeling_qwen3_5vl_image.cpp SOURCES += ../../common/image_process_utils/imageproc.cpp diff --git a/src/test/qwen3_6_moe_npu/Makefile b/src/test/qwen3_6_moe_npu/Makefile index 1c50e0cf2..2a341de5d 100644 --- a/src/test/qwen3_6_moe_npu/Makefile +++ b/src/test/qwen3_6_moe_npu/Makefile @@ -14,6 +14,7 @@ SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_qwen3_6_moe.cpp) SOURCES += $(wildcard ../../common/AutoModel/modeling_qwen3_6_moe_image.cpp) SOURCES += $(wildcard ../../common/image_process_utils/imageproc.cpp) diff --git a/src/test/qwen3_8mtp_npu/Makefile b/src/test/qwen3_8mtp_npu/Makefile index 7ac6fa856..f5520b46e 100644 --- a/src/test/qwen3_8mtp_npu/Makefile +++ b/src/test/qwen3_8mtp_npu/Makefile @@ -15,6 +15,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen3_8mtp.cpp SOURCES += ../../common/AutoModel/modeling_qwen3_8mtp_image.cpp SOURCES += ../../common/image_process_utils/imageproc.cpp diff --git a/src/test/qwen3_npu/Makefile b/src/test/qwen3_npu/Makefile index 39ab56c25..555c4e275 100644 --- a/src/test/qwen3_npu/Makefile +++ b/src/test/qwen3_npu/Makefile @@ -16,6 +16,7 @@ -include ../common.mk SOURCES += $(wildcard ../../common/AutoModel/automodel.cpp) +SOURCES += $(CHAT_SOURCES) SOURCES += $(wildcard ../../common/AutoModel/modeling_qwen3.cpp) SOURCES += $(wildcard ../../common/tokenizer/tokenizer.cpp) SOURCES += $(wildcard ../../common/modules/sampler.cpp) diff --git a/src/test/qwen3vl_flash/Makefile b/src/test/qwen3vl_flash/Makefile index faf9f7ebd..c339d6477 100644 --- a/src/test/qwen3vl_flash/Makefile +++ b/src/test/qwen3vl_flash/Makefile @@ -18,6 +18,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen3vl.cpp SOURCES += ../../common/AutoModel/modeling_qwen3vl_image.cpp SOURCES += ../../common/image_process_utils/imageproc.cpp diff --git a/src/test/qwen3vl_npu/Makefile b/src/test/qwen3vl_npu/Makefile index 4603dc369..1320c2505 100644 --- a/src/test/qwen3vl_npu/Makefile +++ b/src/test/qwen3vl_npu/Makefile @@ -18,6 +18,7 @@ SOURCES += test.cpp SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) SOURCES += ../../common/AutoModel/modeling_qwen3vl.cpp SOURCES += ../../common/AutoModel/modeling_qwen3vl_image.cpp SOURCES += ../../common/image_process_utils/imageproc.cpp From 594c8d495a9a891ddab3aa34f42cae78c0f7213a Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 09:31:13 -0600 Subject: [PATCH 11/18] docs(server): document reasoning, tool calling, and structured output Add guides for reasoning and tool calls, structured output, and the chat output internals, and update the OpenAPI and tool calling pages. Co-Authored-By: Claude --- .../server/chat_output_internals.md | 208 +++++++++ docs/docs/instructions/server/openapi.md | 2 + .../server/reasoning_and_tools.md | 409 +++++++++++++++++ .../instructions/server/structured_output.md | 426 ++++++++++++++++++ docs/docs/instructions/server/tool_calling.md | 69 +-- 5 files changed, 1086 insertions(+), 28 deletions(-) create mode 100644 docs/docs/instructions/server/chat_output_internals.md create mode 100644 docs/docs/instructions/server/reasoning_and_tools.md create mode 100644 docs/docs/instructions/server/structured_output.md diff --git a/docs/docs/instructions/server/chat_output_internals.md b/docs/docs/instructions/server/chat_output_internals.md new file mode 100644 index 000000000..b5e55de8c --- /dev/null +++ b/docs/docs/instructions/server/chat_output_internals.md @@ -0,0 +1,208 @@ +--- +layout: docs +title: "Under the Hood: Output Parsing & Constrained Decoding" +nav_order: 11 +parent: Local Server (Server Mode) +--- + +# 🔬 Under the Hood: Output Parsing & Constrained Decoding + +This page is for contributors. It explains how `/v1/chat/completions` turns raw model text into `reasoning_content` / `content` / `tool_calls`, and how `tool_choice` and `response_format` restrict decoding. For the user-facing behaviour, see [Reasoning, Tool Calls & Streaming](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/) and [Structured Output & tool_choice](https://fastflowlm.com/docs/instructions/server/structured_output/). + +All of it lives in one shared library, `src/common/chat/` with headers in `src/include/chat/`. Models don't carry their own parsers any more. + +--- + +## 🗺️ The Big Picture + +```text + request response + │ ▲ + ▼ │ + chat_constraint_request_from_openai() ┌──────── SSE chunks / JSON body + tool_choice, parallel_tool_calls, │ ▲ + response_format, think/reasoning_effort │ chat_compute_deltas() (stream only) + │ │ ▲ + ▼ │ chat_parse(spec, prefill + text) + AutoModel::make_constraint() │ ▲ + └─ chat_make_constraint(spec, …) │ │ generated text + ├─ chat_structural_tag() ──► chat_grammar_engine (xgrammar) ──► decode_constraint + └─ token_ban_constraint (tool_choice=none) │ + ▼ + decode loop: _sample(): fill_mask ─► Sampler::sample(logits, mask) + accept(token) +``` + +Both halves are driven by one `chat_format_spec`, detected once per model. **The grammar that constrains decoding and the parser that reads the output are built from the same spec, so they can't disagree about the syntax.** + +| File | Role | +|---|---| +| `chat_format.{hpp,cpp}` | `chat_format_spec`, detection from the template, prefill computation | +| `chat_parser.{hpp,cpp}` | `chat_parse()`: text → `chat_msg` for every format | +| `chat_args.{hpp,cpp}` | Prefix-stable conversion of XML / Python / Gemma4 arguments to JSON | +| `chat_msg.{hpp,cpp}` | `chat_msg`, `chat_msg_delta`, `chat_compute_deltas()` | +| `chat_stream.{hpp,cpp}` | Per-request streaming state, tool call ids | +| `chat_constraint.{hpp,cpp}` | OpenAI request → `chat_constraint_request` → `decode_constraint` | +| `chat_grammar.{hpp,cpp}` | xgrammar structural tags per format, `chat_grammar_engine` | +| `decode_constraint.hpp` | The interface the decode loop talks to, plus `token_ban_constraint` | + +--- + +## 1️⃣ Format Detection + +`AutoModel::_shared_setup_tokenizer()` calls `_detect_chat_format()`, which runs `chat_format_detect(template_src, is_token)`. The detector checks markers from most specific to least, and looks at **both** the Jinja template and the tokenizer. Some templates never render the markers the model emits: LFM2.5-Thinking prints tool calls as plain content, and the R1 distills never mention ``. + +| Check (in order) | Result | +|---|---| +| `<|channel|>` and `<|message|>` known | `HARMONY` | +| `<|tool_call>` and `<|channel>` known | `GEMMA4` | +| `<|tool_call_start|>` known | `LFM2_PYTHONIC` | +| Template contains `` | `HERMES_JSON` (the token alone isn't trusted: some vocabularies carry it without training) | +| otherwise | `CONTENT_ONLY`; `reasoning_forced` for DeepSeek-R1 style templates | + +"Known" means present in the template or a single token in the vocabulary. `` / `` turn on reasoning for the tag-based kinds. + +`extra_stops` holds markers that end the turn without being in the model's eos list (`<|call|>` for gpt-oss, `<|tool_response>` for Gemma4). They're added to `eos_token_ids` at load time. + +The server log prints the result: `Chat format: hermes_json with reasoning`. + +### Prefill + +The generation prompt often opens a block the model then continues. Qwen3 with thinking on ends the prompt with `\n`, and Harmony ends it with `<|start|>assistant`. `_begin_turn()` decodes the last 32 prompt tokens into `meta_info.prompt_tail`, and `chat_format_prefill(spec, prompt_tail)` returns the part the parser and grammar must see first: + +- tag formats: `reasoning_start` if the tail opened a reasoning block that it didn't close, else `""` (or `reasoning_start` when `reasoning_forced`); +- Harmony: everything after the last `<|start|>`. + +The parser reads `prefill + generated_text`. The constraint gets the same prefill in `start()`. + +--- + +## 2️⃣ The Parser + +```cpp +chat_msg chat_parse(const chat_format_spec& spec, const std::string& text, + bool is_partial, const chat_parse_options& options); +``` + +One function serves both streaming and non-streaming. The design depends on one invariant: + +> **Prefix property.** For a partial text `t` and any continuation `t + u`, every field of `chat_parse(t, partial=true)` is a prefix of the same field of `chat_parse(t + u, …)`. + +That's what lets the stream send plain suffixes as deltas. Three techniques keep it: + +1. **Partial markers are held back.** `cut_partial_marker()` drops a tail that could be the start of a marker (`…<|channel|>…<|message|>…<|end|>` messages: `analysis` becomes reasoning, `to=functions.x` becomes a call, and everything else becomes content. + +`chat_parse_options.parse_tool_calls` is false when the request has no tools or `tool_choice` is `"none"`. Tool markup then stays in content. + +--- + +## 3️⃣ Streaming + +`streaming_buf_openai_chat` (in `src/server/streaming_ostream_openai.hpp`) owns a `chat_stream`. On every flush of complete UTF-8: + +```cpp +std::vector chat_stream::update(const std::string& text, bool is_final); +``` + +1. appends the text and **re-parses the whole turn** (a scan of a few KB per token, which is noise next to a decode step); +2. assigns a stable `call_<24 chars>` id to each new tool call; +3. returns `chat_compute_deltas(prev, cur)`: the new reasoning suffix, the new content suffix, and per tool call either a first delta (id + name + arguments so far) or an arguments suffix. + +If a field ever fails to extend its previous value, that's a parser bug. `chat_compute_deltas` logs `chat parser rewrote already streamed …` and skips the field rather than sending a contradiction. + +`finalize()` runs a last `update("", true)` so held-back text is released. It promotes `finish_reason` to `tool_calls` only on a clean EOT stop, then sends the usage chunk when `stream_options.include_usage` was set, then `[DONE]`. + +The non-streaming path (`RestHandler::build_nstream_response`) calls `chat_parse(…, is_partial=false, …)` once, so the two modes agree by construction. + +--- + +## 4️⃣ Constrained Decoding + +### From request to constraint + +`chat_constraint_request_from_openai()` validates and normalizes `tool_choice`, `parallel_tool_calls`, `response_format` and thinking (`think: false` or `reasoning_effort: "none"` → `allow_reasoning = false`). Errors become HTTP 400. + +`AutoModel::make_constraint()` also clears `allow_reasoning` when the *model* has thinking off (`thinking_enabled()`), builds a `chat_vocab`, and calls `chat_make_constraint()`, which returns one of: + +- an **xgrammar constraint** when `chat_structural_tag()` returns a grammar; +- a **`token_ban_constraint`** for `tool_choice: "none"` when the tool opener is a single token; +- **null**: nothing to constrain, and the decode loop pays nothing. + +### Structural tags + +`chat_structural_tag(spec, request)` emits an [xgrammar](https://github.com/mlc-ai/xgrammar) structural tag: + +| Request | Grammar shape (tag formats) | +|---|---| +| `auto` + tools | `[reasoning?] triggered_tags(trigger = tool opener, tags = one per tool)`: free text, but every call is fully constrained | +| `required` | same, with `at_least_one` | +| function form | `[reasoning?] ws ` | +| `parallel_tool_calls: false` | `stop_after_first` on the triggered tags | +| `response_format` | `[reasoning?] ws json_schema(schema)`: `{"type": "object"}` for `json_object` | + +`[reasoning?]` is an optional `reasoning_start any_text reasoning_end ws` block. It's there only when reasoning is allowed or the model forces it. It's optional so that a prefill that already opened it is accepted. + +Per-tool tags follow xgrammar's builtin styles where one exists, and are hand-written otherwise: + +| Format | Tag | +|---|---| +| `HERMES_JSON` | `\n{"name": "f", "arguments": ` + JSON schema + `}\n` (xgrammar `qwen_3`) | +| `QWEN_XML` | `\n\n` + schema in `qwen_xml` style + `\n\n`; booleans widened to `True/False/true/false` | +| `GEMMA4` | `<|tool_call>call:f` + EBNF from `gemma4_arguments_grammar()` + `` | +| `LFM2_PYTHONIC` | one tag `<|tool_call_start|>[` + EBNF `python_calls_grammar()` + `]<|tool_call_end|>` holding all calls | +| `HARMONY` | `harmony_format()`: analysis / final / `commentary to=functions.f` messages joined by `<|start|>assistant` (xgrammar `harmony` style, with both spacings gpt-oss uses) | + +The EBNF argument grammars (`argument_list_rules()`) put declared properties in one fixed order, require `required` ones, make optional ones skippable, and type values from the schema (string enums spelled out, array `items` followed). They bound numbers to 16 digits and whitespace to a few characters, because a constrained model otherwise tends to pad forever. + +### The engine + +`chat_grammar_engine` is built lazily, once per model (`Grammar engine ready in … ms`). It reads `tokenizer.json` for the token strings, and builds xgrammar's `TokenizerInfo` and `GrammarCompiler`. Stop tokens that the grammar matches *as text* (`<|call|>`, `<|return|>`, `<|end|>` for Harmony) are kept out of xgrammar's stop list. + +`make()` compiles the request's tag into an `xgrammar_constraint`: + +- `start(prefill)` feeds the prefill (Harmony drops the leading `assistant` role) into the matcher; +- `fill_mask()` fills a `uint32` bitset over the vocabulary; +- `accept(token)` advances; `rollback(n)` undoes; +- `accept_forced(token)` handles a token the model is *made* to emit. If thinking was off in the grammar but a thinking-only model is forced to open ``, the constraint switches to a precompiled grammar that has reasoning and replays its history into it; +- if the matcher ever rejects a token, `give_up()` logs `Constrained decoding disabled for this request: …` and stops masking. A desynchronised grammar would only produce garbage. + +### The decode loop + +`AutoModel::_sample(logits, meta_info)` is the only way a token gets sampled: + +1. `constraint->fill_mask(constraint_mask_)`; +2. `sampler->sample(logits, &mask)`: `Sampler::sampler_allowed_apply()` sets masked logits to `-inf` on the sampler's **own fp32 copy**, after repetition penalties and before top-k/top-p. The engine's device-mapped logits buffer is never written. Greedy sampling copies too when a mask is present; +3. if the sample still lands outside the mask (only with broken logits, e.g. all NaN), the best allowed token is used instead; +4. `constraint->accept(token)`. + +`insert()` samples the first token through `_sample()` and sets `meta_info.insert_sample_pending`. Models whose `generate()` feeds preamble tokens before decoding have to keep the constraint in step: + +| Helper | When | +|---|---| +| `_force_token(meta_info, tok)` | A preamble token that **is** part of the output (e.g. a forced `\n`). It rolls back the unused insert sample and calls `accept_forced()`. | +| `_skip_insert_sample(meta_info)` | Preamble tokens that are **not** output (the empty think block fed when thinking is off). It rolls back the unused insert sample only. | + +See `modeling_qwen3_5vl.cpp` and `modeling_minicpm_v.cpp` for examples, and `src/create_new_model.md` §2f–2g. + +--- + +## ➕ Adding a New Format + +1. Add a `chat_format_kind` and its name in `FORMAT_NAMES` (`chat_format.cpp`). +2. Give it markers in `chat_format_make()`, and a detection rule in `chat_format_detect()` placed by specificity. +3. Parse its tool block in `parse_tool_block()`. Keep the prefix property: hold partial markers back, and stream strings but hold scalars. If the arguments aren't JSON, add a dialect to `chat_args`. +4. Add its tags in `tool_tags()` (`chat_grammar.cpp`). Prefer an xgrammar builtin style. Otherwise write an EBNF grammar on `argument_list_rules()`. +5. If the grammar matches any eos token as text, list it in `text_markers()`. +6. Verify with `chat_grammar_engine::accepts(spec, request, prefill, text, error)`. It feeds a whole output through the grammar and reports the byte where it fails, so you can check that real model outputs are accepted. +7. Run the live API tests in `src/test/openai_api_test/` against a model of that family. + +--- + +## 🧪 Tests + +`src/test/openai_api_test/` holds stdlib-only Python scripts that drive a running server. They check the response contract, stream/non-stream agreement, the thinking switch, every `tool_choice` mode, `response_format` schema conformance, and the 400 errors. See its `README.md`. diff --git a/docs/docs/instructions/server/openapi.md b/docs/docs/instructions/server/openapi.md index ec4cf0330..6a55472a8 100644 --- a/docs/docs/instructions/server/openapi.md +++ b/docs/docs/instructions/server/openapi.md @@ -21,6 +21,8 @@ parent: Local Server (Server Mode) - [Example: Multi-Modal Input](#️-example-multi-modal-input) - [Example: Flash Models (Single-Turn, Pinned System Prompt)](#-example-flash-models-single-turn-pinned-system-prompt) +> 📘 **Related pages:** [Tool Calling](https://fastflowlm.com/docs/instructions/server/tool_calling/) · [Reasoning, Tool Calls & Streaming](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/) (`reasoning_content`, streamed tool call deltas, the `think` switch) · [Structured Output & tool_choice](https://fastflowlm.com/docs/instructions/server/structured_output/) (`tool_choice`, `parallel_tool_calls`, `response_format` JSON / JSON schema) + --- # 📡 How Does the OpenAI Standard Work? diff --git a/docs/docs/instructions/server/reasoning_and_tools.md b/docs/docs/instructions/server/reasoning_and_tools.md new file mode 100644 index 000000000..62e38363c --- /dev/null +++ b/docs/docs/instructions/server/reasoning_and_tools.md @@ -0,0 +1,409 @@ +--- +layout: docs +title: Reasoning, Tool Calls & Streaming +nav_order: 9 +parent: Local Server (Server Mode) +--- + +# 🧠 Reasoning, Tool Calls & Streaming Output + +Every model writes its answer in its own markup. Qwen3 wraps its thinking in `…` and tool calls in `…`. Gemma4 thinks in a `<|channel>thought` channel and writes calls as `call:f{…}`. gpt-oss uses Harmony channels. LFM2 writes Python calls. + +FLM's `/v1/chat/completions` endpoint removes that markup for you. Whatever the model, you get the same OpenAI-shaped message: + +| Field | What it holds | +|---|---| +| `reasoning_content` | The model's thinking, without the tags. Left out when the model didn't reason. | +| `content` | The answer the user should see. `null` when the turn is only tool calls. | +| `tool_calls` | The function calls the model asked for. Each one has `arguments` as a **JSON string**, whatever syntax the model used. | + +This page covers how that output looks with and without streaming, how to turn thinking on and off, and how to read it from the OpenAI Python SDK. + +> 📘 To *force* a tool call, ban one, or make the model answer in JSON that matches a schema, see [Structured Output & tool_choice](https://fastflowlm.com/docs/instructions/server/structured_output/). + +--- + +# 📑 Table of Contents + +- [Supported Model Formats](#-supported-model-formats) +- [Non-Streaming Responses](#-non-streaming-responses) +- [Streaming Responses](#-streaming-responses) +- [Turning Thinking On and Off](#-turning-thinking-on-and-off) +- [Use Case: Show the Model's Thinking Live](#-use-case-show-the-models-thinking-live) +- [Use Case: A Streaming Tool-Calling Agent Loop](#-use-case-a-streaming-tool-calling-agent-loop) +- [Use Case: Parallel Tool Calls](#-use-case-parallel-tool-calls) +- [Rules Worth Knowing](#-rules-worth-knowing) +- [Troubleshooting](#-troubleshooting) + +--- + +## 🗂️ Supported Model Formats + +FLM detects the format once, when the model loads, from its chat template and tokenizer. The server log names the format it detected: + +```text +[FLM] Chat format: hermes_json with reasoning +``` + +| Format (log name) | Model families | Reasoning | Tool call syntax the model writes | +|---|---|---|---| +| `hermes_json` | Qwen3, Qwen2.5, Nanbeige | `…` | `{"name": …, "arguments": {…}}` | +| `qwen_xml` | Qwen3.5 and later | `…` | `v` | +| `lfm2_pythonic` | LFM2, LFM2.5 | `…` (thinking variants) | `<|tool_call_start|>[f(a=1, b="x")]<|tool_call_end|>` | +| `gemma4` | Gemma4 | `<|channel>thought … ` | `<|tool_call>call:f{a:<|"|>x<|"|>}` | +| `harmony` | gpt-oss | `analysis` channel | `commentary to=functions.f` channel | +| `content_only` | everything else | `…` if the model uses it (e.g. DeepSeek-R1) | none: the model has no tool call syntax | + +Your code doesn't depend on any of this. The table explains why a model can or can't call tools, and what its raw output looks like if you ever read the server log. + +> 💡 Check the [model card](https://fastflowlm.com/docs/models/) for which models are labelled `tool-calling` and `reasoning`. + +--- + +## 📦 Non-Streaming Responses + +A request with tools: + +```python +from openai import OpenAI +import json + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["city"], + }, + }, +}] + +resp = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "What's the weather in Paris?"}], + tools=tools, +) + +choice = resp.choices[0] +msg = choice.message +print("finish_reason:", choice.finish_reason) # "tool_calls" +print("reasoning:", getattr(msg, "reasoning_content", None)) # the text, tags removed +print("content:", msg.content) # None: the turn is only a call +for call in msg.tool_calls or []: + print(call.id, call.function.name, json.loads(call.function.arguments)) +``` + +The raw JSON body looks like this: + +```json +{ + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "reasoning_content": "The user wants the weather in Paris, I should call get_weather…", + "content": null, + "tool_calls": [{ + "index": 0, + "id": "call_QbsRMJjkvHumJZVPdR0Gkqsm", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"} + }] + }, + "finish_reason": "tool_calls" + }], + "usage": {"prompt_tokens": 180, "completion_tokens": 64, "total_tokens": 244, "...": "..."} +} +``` + +How to read it: + +- **`reasoning_content`** is a FLM extension that other OpenAI-compatible servers also use. The OpenAI SDK keeps unknown fields, so read it with `getattr(msg, "reasoning_content", None)` or `msg.model_extra`. +- **`content` is `null`** when the model only called tools, as with OpenAI. When the model writes text *and* calls tools, both are set. +- **`arguments` is always a JSON string**, even when the model wrote XML (Qwen3.5), Python (`f(a=1)`, LFM2) or Gemma4's relaxed syntax. FLM converts it. When the model's syntax doesn't say whether a value is a string or a number (Qwen XML), FLM uses the type declared in your tool's `parameters`. +- **`id`** is generated by FLM as `call_` plus 24 random characters. Send it back as `tool_call_id` in the `tool` message. +- **`finish_reason`** is: + - `"tool_calls"` when the model stopped normally after calling at least one tool, + - `"stop"` when it stopped normally without calls, + - `"length"` when it hit `max_tokens` or the context limit. A truncated generation is never reported as `"tool_calls"`, even if a call was started. Treat the call's arguments as incomplete. + +--- + +## 🌊 Streaming Responses + +With `stream=True` the same message arrives in small deltas. FLM runs the **same parser** for both modes. When the stream ends, the deltas add up to exactly the message a non-streaming request would have returned. + +Each chunk's `delta` carries **one** of these: + +| Delta | Meaning | +|---|---| +| `{"reasoning_content": "…"}` | More thinking text. Append it. | +| `{"content": "…"}` | More answer text. Append it. | +| `{"tool_calls": [{"index": 0, "id": "call_…", "type": "function", "function": {"name": "get_weather", "arguments": ""}}]}` | **A new call starts.** Only the first delta of a call has `id`, `type` and `name`. | +| `{"tool_calls": [{"index": 0, "function": {"arguments": "{\"city\": \"Par"}}]}` | More argument text for call `index`. Append it to that call's `arguments`. | + +Then a final chunk with `"delta": {"content": null}` and the `finish_reason`, then `data: [DONE]`. + +Here is a real stream from `qwen3:0.6b` (abridged) that calls `get_weather` twice: + +```text +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_803aPw…","type":"function","function":{"name":"get_weather","arguments":""}}]}, …}]} +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"function":{"arguments":"{\""}}]}, …}]} +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"function":{"arguments":"city"}}]}, …}]} +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"function":{"arguments":"\": \"Paris\"}"}}]}, …}]} +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":1,"id":"call_gDVP0q…","type":"function","function":{"name":"get_weather","arguments":""}}]}, …}]} +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"index":1,"function":{"arguments":"{\"city\": \"Tokyo\"}"}}]}, …}]} +data: {"choices":[{"delta":{"content":null},"finish_reason":"tool_calls"}], …} +data: [DONE] +``` + +### ✅ Accumulating a stream correctly + +Merge tool call deltas **by `index`**. Don't append every delta as a new call. Later deltas have no `id` and no `name`. + +```python +def collect_stream(stream): + """Fold a chat.completion.chunk stream into one assistant message.""" + reasoning, content = [], [] + calls = {} # index -> {"id", "type", "function": {"name", "arguments"}} + finish_reason = None + usage = None + + for chunk in stream: + if chunk.usage: # only with stream_options={"include_usage": True} + usage = chunk.usage + if not chunk.choices: # the usage chunk has no choices + continue + choice = chunk.choices[0] + delta = choice.delta + if choice.finish_reason: + finish_reason = choice.finish_reason + + text = getattr(delta, "reasoning_content", None) + if text: + reasoning.append(text) + if delta.content: + content.append(delta.content) + for tc in delta.tool_calls or []: + call = calls.setdefault(tc.index, { + "id": None, "type": "function", + "function": {"name": "", "arguments": ""}, + }) + if tc.id: + call["id"] = tc.id + if tc.function and tc.function.name: + call["function"]["name"] = tc.function.name + if tc.function and tc.function.arguments: + call["function"]["arguments"] += tc.function.arguments + + message = {"role": "assistant", "content": "".join(content) or None} + if calls: + message["tool_calls"] = [calls[i] for i in sorted(calls)] + return message, "".join(reasoning), finish_reason, usage +``` + +`message` can go straight back into `messages` for the next turn. Leave the reasoning out: FLM strips `reasoning_content` from earlier assistant turns before it builds the prompt anyway. + +### 📊 Usage in streams + +As with OpenAI, a stream reports token usage only when you ask for it: + +```python +stream = client.chat.completions.create(..., stream=True, + stream_options={"include_usage": True}) +``` + +Every chunk then has `"usage": null`, and one extra chunk before `[DONE]` has `"choices": []` and the full usage, including FLM's timing fields (`prefill_speed_tps`, `decoding_speed_tps`, …). Check `chunk.choices` before you index it. + +--- + +## 🔀 Turning Thinking On and Off + +Reasoning models can be switched per request. These are FLM extensions, so the OpenAI SDK sends them through `extra_body`: + +```python +# Turn thinking off: answers come faster, reasoning_content is absent +client.chat.completions.create(model="qwen3:4b", messages=msgs, extra_body={"think": False}) + +# Turn it back on +client.chat.completions.create(model="qwen3:4b", messages=msgs, extra_body={"think": True}) + +# The OpenAI-style knob works too: "none" turns thinking off, +# "low" / "medium" / "high" turn it on (gpt-oss also uses it as its effort level) +client.chat.completions.create(model="gpt-oss:20b", messages=msgs, reasoning_effort="low") +``` + +> ⚠️ **The setting sticks.** `think` and `reasoning_effort` are applied to the loaded model and stay in effect for later requests that don't mention them. If your application mixes thinking and non-thinking calls, send the setting on **every** request. + +What changes when thinking is off: + +- `reasoning_content` is left out of the response. +- With [constrained decoding](https://fastflowlm.com/docs/instructions/server/structured_output/) active (`tool_choice`, `response_format`), the grammar leaves no room for a reasoning block either, so the model can't slip into one. +- Models that can't switch thinking off (for example `qwen3-tk`, `lfm2.5-tk`, `deepseek-r1`) keep reasoning. Their thinking still arrives in `reasoning_content`, never in `content`. + +--- + +## 💭 Use Case: Show the Model's Thinking Live + +A chat UI usually shows the thinking dimmed or folded away, and the answer normally: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") + +DIM, RESET = "\033[2m", "\033[0m" +in_thinking = False + +stream = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "A bat and a ball cost $1.10 in total. " + "The bat costs $1.00 more than the ball. " + "How much is the ball?"}], + stream=True, + extra_body={"think": True}, +) +for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + thinking = getattr(delta, "reasoning_content", None) + if thinking: + if not in_thinking: + print(f"{DIM}🤔 ", end="") + in_thinking = True + print(thinking, end="", flush=True) + if delta.content: + if in_thinking: + print(f"{RESET}\n\n", end="") + in_thinking = False + print(delta.content, end="", flush=True) +print(RESET) +``` + +You never see `` or `` in either field, even half-written. FLM holds back a few characters whenever the text might be the start of a tag, until it can tell. + +--- + +## 🔁 Use Case: A Streaming Tool-Calling Agent Loop + +This loop keeps calling the model until it answers without asking for a tool. It uses `collect_stream()` from [above](#-accumulating-a-stream-correctly). + +```python +import json +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") +MODEL = "qwen3:4b" + +tools = [ + {"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"]}}}, + {"type": "function", "function": { + "name": "convert_currency", + "description": "Convert an amount between two currencies", + "parameters": {"type": "object", + "properties": {"amount": {"type": "number"}, + "from": {"type": "string"}, + "to": {"type": "string"}}, + "required": ["amount", "from", "to"]}}}, +] + +def get_weather(city): + return {"city": city, "temperature_c": 18, "sky": "cloudy"} + +def convert_currency(amount, **kw): + return {"amount": round(amount * 1.08, 2), "currency": kw["to"]} + +IMPLEMENTATIONS = {"get_weather": get_weather, "convert_currency": convert_currency} + +messages = [{"role": "user", + "content": "I'm flying to Paris with 200 USD. What's the weather there, " + "and how many euros is that?"}] + +for step in range(5): # never loop forever + stream = client.chat.completions.create( + model=MODEL, messages=messages, tools=tools, stream=True, + extra_body={"think": False}, + ) + message, _reasoning, finish_reason, _ = collect_stream(stream) + messages.append(message) + + if finish_reason != "tool_calls": + print(message["content"]) + break + + for call in message["tool_calls"]: + fn = IMPLEMENTATIONS[call["function"]["name"]] + args = json.loads(call["function"]["arguments"]) + result = fn(**args) + print(f"🔧 {call['function']['name']}({args}) -> {result}") + messages.append({ + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps(result), + }) +``` + +Things this loop relies on: + +1. **`finish_reason == "tool_calls"`** is the signal to run tools. It's only set when the model stopped normally, so a `"length"` cut-off falls through instead of running a half-written call. +2. **The assistant message goes back with its `tool_calls`**, and each result goes back as a `tool` message with the matching `tool_call_id`. FLM rebuilds the model's native markup from these for the next prompt. +3. **`json.loads(arguments)` is safe** once the stream has ended. While streaming, `arguments` is a growing prefix of the final JSON. Don't parse it before the call is complete. + +--- + +## 🧵 Use Case: Parallel Tool Calls + +Ask for two things and a capable model calls the same tool twice. Each call has its own `index` and `id`: + +```python +resp = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "Weather in Paris and in Tokyo?"}], + tools=tools, +) +for call in resp.choices[0].message.tool_calls or []: + print(call.index, call.id, call.function.arguments) +# 0 call_Gce8tV… {"city": "Paris"} +# 1 call_sCOOJo… {"city": "Tokyo"} +``` + +Answer **every** call with its own `tool` message before the next request. To get at most one call per turn, send `parallel_tool_calls=False` (see [Structured Output & tool_choice](https://fastflowlm.com/docs/instructions/server/structured_output/#-parallel_tool_calls)). + +--- + +## 📏 Rules Worth Knowing + +- **Tools are only parsed when the request has `tools`.** Without them, any tool markup the model writes stays in `content` as plain text. +- **`tool_choice: "none"` hides the tools from the model** entirely. They're not put in the prompt, so the model answers in prose. +- **Text and calls can interleave.** A model may write a sentence, call a tool, then write more. Both parts are returned: the text pieces are joined in `content` and the calls are in `tool_calls`. When separate text pieces are joined, they're separated by a blank line (`\n\n`). +- **Whitespace at the edges is trimmed.** `content` and `reasoning_content` never start or end with blank lines. While streaming, trailing whitespace is held back until more text follows it. +- **Text after the model hands over to a tool is dropped.** Gemma4 marks the end of its turn with `<|tool_response>`, and gpt-oss with `<|call|>`. Generation stops there. +- **Only `/v1/chat/completions` does this parsing.** `/v1/completions` returns the raw text. + +--- + +## 🩺 Troubleshooting + +| Symptom | Likely cause | +|---|---| +| Tool markup like `` shows up in `content` | The request had no `tools`, or the model's format is `content_only` (check the `Chat format:` log line). | +| My client creates one tool call per streamed chunk | Merge deltas by `index`, as in [`collect_stream()`](#-accumulating-a-stream-correctly). | +| `json.loads(arguments)` fails | You parsed while still streaming, or `finish_reason` is `"length"`: raise `max_tokens`. | +| `reasoning_content` is empty but I asked for thinking | A previous request sent `think: false` and the setting stuck. Send `think: true` explicitly. | +| Long pause before the first content token | The model is thinking. Stream it and read `reasoning_content`, or turn thinking off. | +| `usage` is missing in streams | Pass `stream_options={"include_usage": True}`. | diff --git a/docs/docs/instructions/server/structured_output.md b/docs/docs/instructions/server/structured_output.md new file mode 100644 index 000000000..0ad094ce0 --- /dev/null +++ b/docs/docs/instructions/server/structured_output.md @@ -0,0 +1,426 @@ +--- +layout: docs +title: Structured Output & tool_choice +nav_order: 10 +parent: Local Server (Server Mode) +--- + +# 🧩 Structured Output & `tool_choice` (Constrained Decoding) + +Small local models are good at following a format *most* of the time. Constrained decoding makes it *all* of the time. Before every token is sampled, FLM works out which tokens can still lead to valid output and rules out all the others. A model can't write a misspelled tool name, an argument of the wrong type, a missing required field, or a stray Markdown fence around your JSON, because those tokens are never allowed. + +You control it with the standard OpenAI request fields: + +| Field | Values | What FLM guarantees | +|---|---|---| +| `tool_choice` | `"auto"` (default), `"none"`, `"required"`, `{"type": "function", "function": {"name": …}}` | Whether a call happens, which function, and that its arguments match the tool's `parameters` schema | +| `parallel_tool_calls` | `true` (default), `false` | At most one call per turn when `false` | +| `response_format` | `{"type": "text"}`, `{"type": "json_object"}`, `{"type": "json_schema", "json_schema": {"schema": …}}` | `content` is valid JSON, matching the schema if one is given | + +It works the same with and without `stream`, and with or without thinking. The grammar engine is [xgrammar](https://github.com/mlc-ai/xgrammar). + +> 📘 How reasoning, content and tool calls come back in the response is covered in [Reasoning, Tool Calls & Streaming](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/). + +--- + +# 📑 Table of Contents + +- [What the Constraint Guarantees, and What It Doesn't](#-what-the-constraint-guarantees-and-what-it-doesnt) +- [tool_choice](#-tool_choice) +- [parallel_tool_calls](#-parallel_tool_calls) +- [Tool Schemas and `strict`](#-tool-schemas-and-strict) +- [response_format](#-response_format) +- [Use Cases](#-use-cases) + - [Extract Data into a Pydantic Model](#-extract-data-into-a-pydantic-model) + - [Classify with an Enum](#️-classify-with-an-enum) + - [Quick JSON Mode](#-quick-json-mode) + - [Force One Specific Function](#-force-one-specific-function) + - [Route a Request to One of Several Tools](#-route-a-request-to-one-of-several-tools) + - [Answer Without Tools on the Last Turn](#-answer-without-tools-on-the-last-turn) + - [Think First, Then Answer in JSON](#-think-first-then-answer-in-json) + - [LangChain](#-langchain) + - [curl](#-curl) +- [Errors](#-errors) +- [Performance](#-performance) +- [Troubleshooting](#-troubleshooting) + +--- + +## ✅ What the Constraint Guarantees, and What It Doesn't + +**Guaranteed** (when the request ends with `finish_reason` `"stop"` or `"tool_calls"`): + +- Tool names are one of the tools you declared. +- Tool arguments are well-formed and follow the tool's `parameters` schema: types, `required` fields, `enum` values, and array `items`. +- `response_format` content parses as JSON and follows the schema. +- When thinking is off, no reasoning block is generated. + +**Not guaranteed:** + +- **Values that make sense.** The model can still put `"Paris"` where you wanted `"Tokyo"`. The constraint shapes the output; it doesn't check facts. +- **Complete output on `"length"`.** If `max_tokens` runs out mid-JSON, the JSON is cut off. Give structured requests enough room, and more when thinking is on. +- **Silence around calls under `"auto"` / `"required"`.** Those modes allow text before, between and after calls (see below). Only the forced-function form allows nothing but the call. + +> 💡 Describe the format in your prompt as well. The constraint stops invalid output, but the model picks better values when it knows what you want. + +--- + +## 🎛️ `tool_choice` + +| Value | What the model may write | +|---|---| +| `"auto"` *(default)* | Free text, or tool calls, or both. Once the model **starts** a call, the rest of that call is constrained: the name must be a declared tool and the arguments must match its schema. | +| `"none"` | Text only. The tools are **left out of the prompt**, so the model doesn't know they exist, and the call opener token is banned. | +| `"required"` | **At least one** tool call. Text is still allowed around the calls. Pair with `parallel_tool_calls=False` for exactly one. | +| `{"type": "function", "function": {"name": "f"}}` | **Exactly one** call to `f` and nothing else in `content` (the model may still reason first when thinking is on). | + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") + +resp = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "Weather in Paris?"}], + tools=tools, + tool_choice="required", # or "auto", "none", + # or {"type": "function", "function": {"name": "get_weather"}} + extra_body={"think": False}, +) +``` + +Notes: + +- `"required"` and the function form need a model whose format has a tool call syntax (every format except `content_only`, see [the format table](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/#️-supported-model-formats)). On other models they're rejected with HTTP 400 rather than silently ignored. +- `"auto"` on a model without a tool call syntax is accepted, but nothing is constrained or parsed. +- With `"required"`, small models sometimes keep writing after their call, for example a made-up answer to their own question. Look at `tool_calls`. `content` may hold this extra text. + +--- + +## 🧵 `parallel_tool_calls` + +| Value | Effect | +|---|---| +| `true` *(default)* | The model may make several calls in one turn. | +| `false` | Generation stops after the first complete call. | + +`parallel_tool_calls=False` with `tool_choice="required"` means **exactly one call**: a clean way to make a model pick one action. + +--- + +## 📐 Tool Schemas and `strict` + +The tool's `function.parameters` is the schema the arguments must follow. + +- **Schemas are enforced by default.** In FLM every tool behaves as if `"strict": true`. Set `"strict": false` on a tool to only require well-formed JSON arguments. +- A tool with **no `parameters`** gets any well-formed JSON arguments. +- An array without `items` accepts any items. +- **Qwen3.5+ booleans** (`qwen_xml` format): the model's template writes Python-style `True`/`False`. Both spellings are accepted and come back as JSON `true`/`false`. +- **Gemma4 and LFM2** use FLM's own argument grammars, written for each model's native call syntax, instead of a full JSON-schema engine. Top-level arguments get their declared types, `required` fields, string `enum`s and array `items` enforced. Arguments are written in one fixed order (alphabetical for Gemma4, declaration order for LFM2). **Nested objects** are only held to valid syntax, not to their sub-schema. Numbers are limited to 16 digits so a model can't pad one forever. + +--- + +## 📄 `response_format` + +### JSON mode + +```python +resp = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "Give me a JSON object with keys color and hex for red."}], + response_format={"type": "json_object"}, +) +print(resp.choices[0].message.content) # {"color": "red", "hex": "#FF0000"} +``` + +`content` is a JSON **object**: any keys, any values. + +### JSON schema + +```python +resp = client.chat.completions.create( + model="qwen3:4b", + messages=[{"role": "user", "content": "Extract: Alice is 31 and lives in Berlin."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "person", # accepted, not used + "strict": True, # accepted; the schema is always enforced + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "city": {"type": "string"}, + }, + "required": ["name", "age", "city"], + "additionalProperties": False, + }, + }, + }, + extra_body={"think": False}, +) +print(resp.choices[0].message.content) # {"name": "Alice", "age": 31, "city": "Berlin"} +``` + +Details: + +- The schema supports what xgrammar supports. That covers types, `properties`, `required`, `enum`, `const`, `items`, `anyOf`, `$defs`/`$ref` (so Pydantic's nested models work), string `pattern`s and number bounds. A schema xgrammar can't compile is rejected with a 400 that names the problem. +- **Whitespace is bounded.** Up to 8 blanks before the JSON and a limited amount of indentation inside it. This stops a constrained model from padding with blank lines forever. +- **Thinking still works.** With thinking on, the model reasons first, freely, into `reasoning_content`. Only `content` is constrained. +- **`response_format` and tools.** With `tool_choice="auto"`, `response_format` wins: the answer is JSON and the model can't call tools. With `"required"` or a forced function, the call wins and `response_format` is ignored. Use separate requests if you need both. +- Generation ends when the JSON is complete: `finish_reason` is `"stop"`. + +--- + +## 💡 Use Cases + +All examples assume: + +```python +import json +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") +MODEL = "qwen3:4b" +``` + +### 🧾 Extract Data into a Pydantic Model + +Let Pydantic write the schema and validate the result: + +```python +from pydantic import BaseModel + +class Item(BaseModel): + name: str + qty: int + +class Order(BaseModel): + customer: str + items: list[Item] + +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Bob wants 2 apples and 3 pears."}], + response_format={ + "type": "json_schema", + "json_schema": {"name": "Order", "schema": Order.model_json_schema()}, + }, + extra_body={"think": False}, +) +order = Order.model_validate_json(resp.choices[0].message.content) +print(order) +# customer='Bob' items=[Item(name='apples', qty=2), Item(name='pears', qty=3)] +``` + +> Newer OpenAI SDKs also have `client.chat.completions.parse(response_format=Order, …)`. It sends the same `json_schema` request, so it works with FLM too. + +### 🏷️ Classify with an Enum + +The model can only write one of the labels: + +```python +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Classify the sentiment: I loved this movie!"}], + response_format={"type": "json_schema", "json_schema": {"name": "sentiment", "schema": { + "type": "object", + "properties": {"sentiment": {"type": "string", + "enum": ["positive", "neutral", "negative"]}}, + "required": ["sentiment"], + "additionalProperties": False, + }}}, + max_tokens=30, + extra_body={"think": False}, +) +label = json.loads(resp.choices[0].message.content)["sentiment"] # "positive" +``` + +### 🧱 Quick JSON Mode + +When you only need *some* JSON object: + +```python +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "system", "content": "Reply with a JSON object with keys 'title' and 'tags'."}, + {"role": "user", "content": "Summarize: NPUs make local LLMs fast and efficient."}], + response_format={"type": "json_object"}, +) +data = json.loads(resp.choices[0].message.content) +``` + +### 🎯 Force One Specific Function + +Name the function, and the response is exactly one call to it with valid arguments and no text. This is a common way to do *extraction through a tool definition*: + +```python +tools = [{"type": "function", "function": { + "name": "create_event", + "description": "Add an event to the calendar", + "parameters": {"type": "object", "properties": { + "title": {"type": "string"}, + "date": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$"}, + "attendees": {"type": "array", "items": {"type": "string"}}, + }, "required": ["title", "date"]}, +}}] + +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Lunch with Ana and Raj on 2026-11-03."}], + tools=tools, + tool_choice={"type": "function", "function": {"name": "create_event"}}, + extra_body={"think": False}, +) +call = resp.choices[0].message.tool_calls[0] +print(call.function.name, json.loads(call.function.arguments)) +# e.g. create_event {'title': 'Lunch with Ana and Raj', 'date': '2026-11-03', 'attendees': ['Ana', 'Raj']} +``` + +### 🔀 Route a Request to One of Several Tools + +`"required"` plus `parallel_tool_calls=False` makes the model pick **exactly one** of your tools. Use it as an intent router: + +```python +router_tools = [ + {"type": "function", "function": {"name": "search_docs", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}}}, + {"type": "function", "function": {"name": "open_ticket", + "parameters": {"type": "object", "properties": { + "summary": {"type": "string"}, + "priority": {"type": "string", "enum": ["low", "medium", "high"]}}, + "required": ["summary", "priority"]}}}, + {"type": "function", "function": {"name": "small_talk", + "parameters": {"type": "object", "properties": {}}}}, +] + +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "The server is down and customers can't log in!"}], + tools=router_tools, + tool_choice="required", + parallel_tool_calls=False, + extra_body={"think": False}, +) +call = resp.choices[0].message.tool_calls[0] +print(call.function.name, call.function.arguments) +# e.g. open_ticket {"summary": "Server down, customers cannot log in", "priority": "high"} +``` + +### 🔚 Answer Without Tools on the Last Turn + +In an agent loop, once you have enough tool results, `tool_choice="none"` makes the model write its final answer instead of calling yet another tool: + +```python +messages.append({"role": "user", "content": "Thanks, now summarize what you found."}) +final = client.chat.completions.create( + model=MODEL, messages=messages, tools=tools, tool_choice="none", +) +print(final.choices[0].message.content) +``` + +You can keep sending the same `tools` list. FLM leaves it out of the prompt for this request. + +### 🤔 Think First, Then Answer in JSON + +On reasoning models, thinking before a structured answer often improves the values. The thinking is free text and only the answer is constrained: + +```python +resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "A train leaves at 14:35 and arrives at 17:10. " + "How long is the trip? Answer as JSON."}], + response_format={"type": "json_schema", "json_schema": {"name": "duration", "schema": { + "type": "object", + "properties": {"hours": {"type": "integer"}, "minutes": {"type": "integer"}}, + "required": ["hours", "minutes"], "additionalProperties": False}}}, + max_tokens=2048, # leave room for the thinking + extra_body={"think": True}, +) +msg = resp.choices[0].message +print("thought:", getattr(msg, "reasoning_content", "")[:200], "…") +print("answer:", json.loads(msg.content)) # {'hours': 2, 'minutes': 35} +``` + +### 🦜 LangChain + +`ChatOpenAI` sends the same fields, so LangChain's structured output helpers work against FLM: + +```python +from langchain_openai import ChatOpenAI +from pydantic import BaseModel + +class Person(BaseModel): + name: str + age: int + +llm = ChatOpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm", model="qwen3:4b", + extra_body={"think": False}) + +# via response_format json_schema +person = llm.with_structured_output(Person, method="json_schema").invoke("Alice is 31.") + +# via a forced tool call +person = llm.with_structured_output(Person, method="function_calling").invoke("Alice is 31.") +``` + +### 🐚 curl + +```bash +curl http://127.0.0.1:52625/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen3:4b", + "think": false, + "messages": [{"role": "user", "content": "Extract: Alice is 31 and lives in Berlin."}], + "response_format": {"type": "json_schema", "json_schema": {"name": "person", "schema": { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}, "city": {"type": "string"}}, + "required": ["name", "age", "city"], "additionalProperties": false}}} + }' +``` + +--- + +## ❌ Errors + +A request FLM can't serve as asked gets **HTTP 400** with an OpenAI-style error body, instead of being quietly downgraded: + +```json +{"error": {"message": "tool_choice names function \"nope\" which is not in tools", + "type": "invalid_request_error", "code": 400}} +``` + +In the OpenAI Python SDK this raises `openai.BadRequestError`. + +| Request | Message | +|---|---| +| `tool_choice: "sometimes"` | `tool_choice must be "auto", "none", "required" or a function, got "sometimes"` | +| `tool_choice: "required"` with no `tools` | `tool_choice requires a tool call but the request has no tools` | +| Function form naming a tool that isn't in `tools` | `tool_choice names function "nope" which is not in tools` | +| `response_format: {"type": "xml"}` | `response_format type must be text, json_object or json_schema` | +| `json_schema` without `schema` | `response_format json_schema needs json_schema.schema` | +| A schema xgrammar can't compile (e.g. a dangling `$ref`) | `cannot build the decoding grammar: …` with xgrammar's reason | +| `"required"` / function form on a model without tool syntax | `this model's chat format (content_only) has no tool call syntax to constrain` | + +--- + +## ⚡ Performance + +- **First constrained request:** FLM builds the grammar engine for the loaded model. It indexes the whole vocabulary once, and the log shows `Grammar engine ready in … ms` (about 0.3 s for Qwen3). Later requests reuse it. +- **Per request:** the request's grammar is compiled. That's quick for typical tool lists and schemas and grows with schema size. +- **Per token:** the allowed-token mask is computed on the CPU before each sample, which adds a small, fixed cost per token. +- **Unconstrained requests pay nothing.** A request with no `tools` and no `response_format`, or with `tool_choice="auto"` on a model without a tool call syntax, never touches the grammar engine. + +--- + +## 🩺 Troubleshooting + +| Symptom | What to do | +|---|---| +| `content` is cut-off JSON and `finish_reason` is `"length"` | Raise `max_tokens`. With thinking on, budget for the reasoning too. | +| Values are valid but wrong | Improve the prompt and field `description`s, use a bigger model, or let it think first. | +| Server log says `Constrained decoding disabled for this request: …` | The grammar lost track of the output (rare, e.g. a template quirk) and FLM let the model finish unconstrained rather than produce garbage. Validate on your side too, and please report it with the log line. | +| 400 `has no tool call syntax to constrain` | The model has no tool format. Use a model labelled `tool-calling`, or use `response_format` instead. | +| `"required"` returned a call **and** some text | Expected: `"required"` only guarantees *at least one* call. Use the function form, or ignore `content`. | diff --git a/docs/docs/instructions/server/tool_calling.md b/docs/docs/instructions/server/tool_calling.md index 232a07591..39e7c0f11 100644 --- a/docs/docs/instructions/server/tool_calling.md +++ b/docs/docs/instructions/server/tool_calling.md @@ -69,6 +69,10 @@ Tool calling has been supported since `v0.9.26`. Please refer to the [model card](https://fastflowlm.com/docs/models/) to see whether tool calling is supported for each model. +> 📘 Going further: +> - [Reasoning, Tool Calls & Streaming](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/) explains exactly how tool calls and reasoning come back, streamed and not, plus an agent loop example. +> - [Structured Output & tool_choice](https://fastflowlm.com/docs/instructions/server/structured_output/) shows how to force a call (`tool_choice="required"` or a named function), forbid one (`"none"`), limit a turn to one call (`parallel_tool_calls=False`), and get schema-valid JSON answers (`response_format`). + ## 🐍 How to use tool calling with FLM via Python script This section walks through an end‑to‑end tool‑calling flow in both streaming and non‑streaming modes. @@ -117,7 +121,9 @@ messages = [ ] ## Ask model; it should emit a tool call -tool_calls = [] +## A streamed call arrives in pieces: its first delta carries id and name, +## later deltas carry more of the arguments. Merge them by index. +tool_calls = {} assistant_content = "" for chunk in client.chat.completions.create( model="qwen3-it:4b", @@ -125,41 +131,49 @@ for chunk in client.chat.completions.create( tools=tools, stream=True, ): + if not chunk.choices: + continue delta = chunk.choices[0].delta # Collect assistant text (if any) - if hasattr(delta, "content") and delta.content is not None: + if delta.content: assistant_content += delta.content - # Collect tool call tokens - if hasattr(delta, "tool_calls") and delta.tool_calls is not None: - for tc in delta.tool_calls: - tool_calls.append(tc) + # Collect tool call pieces + for tc in delta.tool_calls or []: + call = tool_calls.setdefault(tc.index, { + "id": None, "type": "function", + "function": {"name": "", "arguments": ""}, + }) + if tc.id: + call["id"] = tc.id + if tc.function and tc.function.name: + call["function"]["name"] = tc.function.name + if tc.function and tc.function.arguments: + call["function"]["arguments"] += tc.function.arguments + +tool_calls = [tool_calls[i] for i in sorted(tool_calls)] # Step 2: ## Add assistant message (with tool calls) to messages -messages.append( - { - "role": "assistant", - "content": assistant_content, # may be None - "tool_calls": tool_calls # important: keep tool_calls - } -) +assistant_msg = {"role": "assistant", "content": assistant_content or None} +if tool_calls: + assistant_msg["tool_calls"] = tool_calls # important: keep tool_calls +messages.append(assistant_msg) ## Execute tool calls and add tool call outputs back -if tool_calls: - for tc in tool_calls: - if tc.function.name == "get_temperature": - args = json.loads(tc.function.arguments) - temp = get_temperature(args["city"]) - messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": json.dumps({"temperature": temp}), - } - ) +for tc in tool_calls: + if tc["function"]["name"] == "get_temperature": + args = json.loads(tc["function"]["arguments"]) + temp = get_temperature(args["city"]) + messages.append( + { + "role": "tool", + "tool_call_id": tc["id"], + "content": json.dumps({"temperature": temp}), + } + ) print("Messages so far:") print(messages) @@ -172,9 +186,8 @@ for chunk in client.chat.completions.create( tools=tools, stream=True, ): - delta = chunk.choices[0].delta - if hasattr(delta, "content") and delta.content is not None: - print(delta.content, end="", flush=True) + if chunk.choices and chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) ``` From bddd0a66759f76dbb3f8a6e5accbf9874514dbce Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 09:31:13 -0600 Subject: [PATCH 12/18] test(server): add OpenAI API conformance tests Cover response shape, tool calls, response_format, reasoning, and error handling against a running FLM server. Co-Authored-By: Claude --- src/test/openai_api_test/README.md | 64 ++ src/test/openai_api_test/fixtures.py | 129 ++++ src/test/openai_api_test/flm_api.py | 561 ++++++++++++++++++ src/test/openai_api_test/run_all.py | 36 ++ src/test/openai_api_test/test_errors.py | 99 ++++ src/test/openai_api_test/test_reasoning.py | 122 ++++ .../openai_api_test/test_response_format.py | 163 +++++ .../openai_api_test/test_response_shape.py | 97 +++ src/test/openai_api_test/test_tool_calls.py | 242 ++++++++ 9 files changed, 1513 insertions(+) create mode 100644 src/test/openai_api_test/README.md create mode 100644 src/test/openai_api_test/fixtures.py create mode 100644 src/test/openai_api_test/flm_api.py create mode 100644 src/test/openai_api_test/run_all.py create mode 100644 src/test/openai_api_test/test_errors.py create mode 100644 src/test/openai_api_test/test_reasoning.py create mode 100644 src/test/openai_api_test/test_response_format.py create mode 100644 src/test/openai_api_test/test_response_shape.py create mode 100644 src/test/openai_api_test/test_tool_calls.py diff --git a/src/test/openai_api_test/README.md b/src/test/openai_api_test/README.md new file mode 100644 index 000000000..842ff9928 --- /dev/null +++ b/src/test/openai_api_test/README.md @@ -0,0 +1,64 @@ +# OpenAI API tests + +Live tests for `/v1/chat/completions`: output parsing (reasoning, content, tool calls), streaming, and constrained decoding (`tool_choice`, `parallel_tool_calls`, `response_format`). They drive a FastFlowLM server that is already running and need only the Python 3 standard library. + +User docs for what's tested here: + +- `docs/docs/instructions/server/reasoning_and_tools.md` +- `docs/docs/instructions/server/structured_output.md` +- `docs/docs/instructions/server/chat_output_internals.md` (how it's built) + +## Running + +```bash +flm serve qwen3:4b # in another terminal +python3 run_all.py qwen3:4b # everything +python3 test_tool_calls.py qwen3:4b # one module +python3 run_all.py qwen3:4b -k schema # only cases whose name contains "schema" +``` + +| Option | Meaning | +|---|---| +| `--endpoint URL` | Server base URL, default `http://127.0.0.1:52625` | +| `--timeout T` | Per-request timeout in seconds, default 300 | +| `--always-thinks` | The model reasons even with `think: false` (`qwen3-tk`, `lfm2.5-tk`, `deepseek-r1`, `gpt-oss`). Don't require reasoning to disappear. | +| `-k NAME` | Run only matching cases | +| `-v` | Print tracebacks for failures | + +Each case prints `PASS`, `FAIL` or `SKIP`. The exit code is non-zero if anything failed. A full run takes about a minute on `qwen3:0.6b`. + +## What's covered + +| Module | Checks | +|---|---| +| `test_response_shape.py` | Message and chunk shape, one delta kind per chunk, `finish_reason`, `usage` only with `stream_options.include_usage`, stream folds to the non-stream message | +| `test_reasoning.py` | `think: true` → `reasoning_content`, `think: false` / `reasoning_effort: "none"` → none, reasoning streamed before content, switch is per request, reasoning in history is accepted | +| `test_tool_calls.py` | `auto` / `required` / forced function / `none`, `parallel_tool_calls: false`, ids, arguments are JSON matching the tool schema (typed: integer, number, boolean, enum, arrays), `strict: false`, streamed call deltas, a tool round trip | +| `test_response_format.py` | `json_object`, `json_schema` (flat, enum, Pydantic `$defs`/`$ref`, `const`/`anyOf`/bounds/`pattern`), streamed, with and without thinking, schema vs. `auto` tools, `text` | +| `test_errors.py` | Every malformed `tool_choice` / `response_format` gets HTTP 400 with an OpenAI error body, streaming too, and the server keeps serving | + +Some cases skip by design: + +- **Tool cases** skip on models whose chat format has no tool call syntax (the server answers `has no tool call syntax`). +- **Thinking-on cases** skip on models that never reason. +- **Multi-turn cases** skip on single-turn (flash) models. +- **Stream/non-stream parity** compares greedy outputs. Greedy decoding on the NPU isn't always bit-reproducible between identical requests (mostly over long outputs), so the check retries and only fails if the non-streaming output reproduced itself while the stream never matched it. Otherwise it skips. + +## Notes + +- Requests pin `think`, `top_k: 1` and `temperature: 0` explicitly. FLM applies sampling and thinking settings to the loaded model and **keeps them for later requests**, so after a run the server is left greedy with thinking as the last case set it. Send your own values, or restart the server, before using it interactively. +- The tests check guarantees, not model quality. Under `auto` a model may or may not call a tool, and the tests only check that what it returns is well formed. Under `required`, a forced function, or `response_format`, the grammar makes the shape a guarantee, so any violation is a server bug. +- Shared helpers live in `flm_api.py`: the HTTP/SSE client, `fold_stream()` (merges tool call deltas by `index`, as clients should), the contract checks, and a small JSON Schema validator. Tools and schemas are in `fixtures.py`. + +## Adding a case + +```python +@case +def my_case(api: Api) -> None: + """One line saying what is guaranteed.""" + choice = api.ok(api.body(user("..."), tools=WEATHER_TOOLS, tool_choice="required")) + flm_api.check_message_contract(choice) + ... +``` + +Functions marked `@case` are collected in definition order. Raise `AssertionError` (or use `expect()`) to fail, and `Skip` when the case doesn't apply to the model. diff --git a/src/test/openai_api_test/fixtures.py b/src/test/openai_api_test/fixtures.py new file mode 100644 index 000000000..a24bb713f --- /dev/null +++ b/src/test/openai_api_test/fixtures.py @@ -0,0 +1,129 @@ +"""Tools, schemas and prompts shared by the OpenAI API tests.""" + +from __future__ import annotations + +from typing import Any + + +def user(text: str) -> list[dict[str, Any]]: + """A one-message conversation.""" + return [{"role": "user", "content": text}] + + +def function_choice(name: str) -> dict[str, Any]: + """tool_choice forcing one function.""" + return {"type": "function", "function": {"name": name}} + + +GET_WEATHER = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name, e.g. Paris"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["city"], + }, + }, +} + +GET_TIME = { + "type": "function", + "function": { + "name": "get_time", + "description": "Get the current local time in a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} + +WEATHER_TOOLS = [GET_WEATHER, GET_TIME] + +# Every argument type the grammars handle: integer, number, boolean, string +# enum, array with typed items, and an optional field. +BOOK_FLIGHT = { + "type": "function", + "function": { + "name": "book_flight", + "description": "Book a flight", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string"}, + "destination": {"type": "string"}, + "passengers": {"type": "integer"}, + "max_price": {"type": "number"}, + "refundable": {"type": "boolean"}, + "cabin": {"type": "string", "enum": ["economy", "business", "first"]}, + "meals": {"type": "array", "items": {"type": "string", "enum": ["veg", "vegan", "standard"]}}, + }, + "required": ["origin", "destination", "passengers", "refundable", "cabin"], + }, + }, +} + +PERSON_SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "city": {"type": "string"}, + }, + "required": ["name", "age", "city"], + "additionalProperties": False, +} + +SENTIMENT_SCHEMA = { + "type": "object", + "properties": {"sentiment": {"type": "string", "enum": ["positive", "neutral", "negative"]}}, + "required": ["sentiment"], + "additionalProperties": False, +} + +# What Pydantic emits for a model with a nested list of models. +ORDER_SCHEMA = { + "$defs": { + "Item": { + "title": "Item", + "type": "object", + "properties": { + "name": {"title": "Name", "type": "string"}, + "qty": {"title": "Qty", "type": "integer"}, + }, + "required": ["name", "qty"], + "additionalProperties": False, + }, + }, + "title": "Order", + "type": "object", + "properties": { + "customer": {"title": "Customer", "type": "string"}, + "items": {"title": "Items", "type": "array", "items": {"$ref": "#/$defs/Item"}}, + }, + "required": ["customer", "items"], + "additionalProperties": False, +} + +REVIEW_SCHEMA = { + "type": "object", + "properties": { + "kind": {"const": "review"}, + "score": {"type": "integer", "minimum": 1, "maximum": 5}, + "vegetarian": {"anyOf": [{"type": "boolean"}, {"type": "null"}]}, + "date": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$"}, + }, + "required": ["kind", "score", "vegetarian", "date"], + "additionalProperties": False, +} + + +def json_schema_format(name: str, schema: dict[str, Any]) -> dict[str, Any]: + """response_format for a JSON schema.""" + return {"type": "json_schema", "json_schema": {"name": name, "strict": True, "schema": schema}} diff --git a/src/test/openai_api_test/flm_api.py b/src/test/openai_api_test/flm_api.py new file mode 100644 index 000000000..1cff0a2e0 --- /dev/null +++ b/src/test/openai_api_test/flm_api.py @@ -0,0 +1,561 @@ +"""Shared helpers for the OpenAI API tests. + +Everything here uses only the Python 3 standard library, so the tests run +anywhere a FastFlowLM server is reachable: + +* ``Api``: posts chat completion requests and reads SSE streams. +* ``fold_stream``: folds a stream of chunks into one assistant message, + the way an OpenAI client should (tool calls merged by ``index``). +* ``validate``: a small JSON Schema checker covering the keywords the + tests use. +* ``case`` / ``run_cases`` / ``main``: a minimal runner printing + PASS / FAIL / SKIP per case and exiting non-zero on any failure. + +Every request the tests send pins ``think``, ``top_k`` and ``temperature`` +explicitly: FastFlowLM applies those to the loaded model and keeps them for +later requests, so a test must never rely on what an earlier one left set. +""" + +# Every check here reports on the server under test, not on a caller +# passing bad arguments, so a wrong type in the response is a test +# failure rather than a TypeError. +# ruff: noqa: TRY004 + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import traceback +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Callable + +DEFAULT_ENDPOINT = "http://127.0.0.1:52625" +DEFAULT_TIMEOUT = 300.0 + +# Raw markup any supported format may write; none of it may reach the client. +MARKERS = ( + "", "", + "", "", "", "<|tool_call_end|>", + "<|tool_call>", "", "<|channel>", "", "<|tool_response>", + "<|channel|>", "<|message|>", "<|start|>", "<|end|>", "<|call|>", "<|return|>", +) + +TOOL_CALL_ID = re.compile(r"^call_[A-Za-z0-9]{24}$") + + +class Skip(Exception): + """The case does not apply to the model under test.""" + + +# ---------------------------------------------------------------- HTTP + + +@dataclass +class Response: + """A non-streaming response: HTTP status and decoded JSON body.""" + + status: int + body: dict[str, Any] + + def error_message(self) -> str: + err = self.body.get("error") + if isinstance(err, dict): + return str(err.get("message", "")) + return str(err or "") + + +@dataclass +class Stream: + """A streaming response: every ``data:`` payload before ``[DONE]``.""" + + status: int + chunks: list[dict[str, Any]] = field(default_factory=list) + done: bool = False # saw "data: [DONE]" + error: dict[str, Any] | None = None # body of a non-200 reply + + +class Api: + """Talks to one FastFlowLM server about one model.""" + + def __init__(self, endpoint: str, model: str, timeout: float, always_thinks: bool = False) -> None: + self.url = endpoint.rstrip("/") + "/v1/chat/completions" + self.model = model + self.timeout = timeout + # the model reasons whatever think says (qwen3-tk, deepseek-r1, gpt-oss, ...) + self.always_thinks = always_thinks + + def body(self, messages: list[dict[str, Any]], *, think: bool = False, + max_tokens: int = 256, **fields: Any) -> dict[str, Any]: + """Build a request body with sampling pinned (see the module note).""" + body: dict[str, Any] = { + "model": self.model, + "messages": messages, + "max_tokens": max_tokens, + "think": think, + "top_k": 1, + "temperature": 0.0, + } + body.update(fields) + return body + + def _open(self, body: dict[str, Any]): + req = urllib.request.Request( + self.url, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + return urllib.request.urlopen(req, timeout=self.timeout) + + def post(self, body: dict[str, Any]) -> Response: + """Send a non-streaming request; HTTP errors are returned, not raised.""" + body = dict(body, stream=False) + try: + with self._open(body) as resp: + return Response(resp.status, json.loads(resp.read().decode("utf-8"))) + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", "replace") + try: + return Response(e.code, json.loads(raw)) + except json.JSONDecodeError: + return Response(e.code, {"error": {"message": raw}}) + + def stream(self, body: dict[str, Any]) -> Stream: + """Send a streaming request and collect its SSE payloads.""" + body = dict(body, stream=True) + try: + resp = self._open(body) + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", "replace") + try: + error = json.loads(raw) + except json.JSONDecodeError: + error = {"error": {"message": raw}} + return Stream(e.code, error=error) + out = Stream(resp.status) + with resp: + for raw_line in resp: + line = raw_line.decode("utf-8").strip() + if not line.startswith("data:"): + continue + payload = line[len("data:"):].strip() + if payload == "[DONE]": + out.done = True + break + out.chunks.append(json.loads(payload)) + return out + + def ok(self, body: dict[str, Any]) -> dict[str, Any]: + """Non-streaming request that must succeed; returns ``choices[0]``.""" + resp = self.post(body) + if resp.status != 200: + raise_for_capability(resp.error_message()) + raise AssertionError(f"HTTP {resp.status}: {resp.error_message()}") + expect(isinstance(resp.body.get("choices"), list) and len(resp.body["choices"]) == 1, + f"expected one choice, got {resp.body.get('choices')!r}") + return resp.body["choices"][0] + + def ok_stream(self, body: dict[str, Any]) -> Stream: + """Streaming request that must succeed and end with ``[DONE]``.""" + s = self.stream(body) + if s.status != 200: + message = str((s.error or {}).get("error", {}).get("message", s.error)) + raise_for_capability(message) + raise AssertionError(f"HTTP {s.status}: {message}") + expect(s.done, "stream ended without data: [DONE]") + return s + + +def raise_for_capability(message: str) -> None: + """Turn a 'this model cannot do that' rejection into a skip.""" + if "has no tool call syntax" in message: + raise Skip("the model's chat format has no tool call syntax") + if "only supports single-turn" in message: + raise Skip("the model only supports single-turn requests") + + +# ---------------------------------------------------------------- stream folding + + +@dataclass +class Folded: + """An assistant turn rebuilt from stream chunks.""" + + reasoning: str + content: str + tool_calls: list[dict[str, Any]] + finish_reason: str | None + usage: dict[str, Any] | None + # kind of each delta in order: "reasoning", "content" or "tool:" + order: list[str] + + +def fold_stream(chunks: list[dict[str, Any]]) -> Folded: + """Fold chunks into one message, merging tool call deltas by index.""" + reasoning: list[str] = [] + content: list[str] = [] + calls: dict[int, dict[str, Any]] = {} + finish_reason = None + usage = None + order: list[str] = [] + for chunk in chunks: + if chunk.get("usage"): + usage = chunk["usage"] + for choice in chunk.get("choices") or []: + if choice.get("finish_reason"): + finish_reason = choice["finish_reason"] + delta = choice.get("delta") or {} + if delta.get("reasoning_content"): + reasoning.append(delta["reasoning_content"]) + order.append("reasoning") + if delta.get("content"): + content.append(delta["content"]) + order.append("content") + for tc in delta.get("tool_calls") or []: + call = calls.setdefault(tc["index"], { + "id": None, "type": None, "function": {"name": "", "arguments": ""}, + }) + if tc.get("id"): + call["id"] = tc["id"] + if tc.get("type"): + call["type"] = tc["type"] + fn = tc.get("function") or {} + if fn.get("name"): + call["function"]["name"] = fn["name"] + call["function"]["arguments"] += fn.get("arguments") or "" + order.append(f"tool:{tc['index']}") + return Folded( + reasoning="".join(reasoning), + content="".join(content), + tool_calls=[calls[i] for i in sorted(calls)], + finish_reason=finish_reason, + usage=usage, + order=order, + ) + + +# ---------------------------------------------------------------- checks + + +def expect(condition: bool, message: str) -> None: + """Assert with a message that survives ``python -O``.""" + if not condition: + raise AssertionError(message) + + +def assert_no_markers(text: str | None, where: str) -> None: + """No raw template markup may leak into what the client sees.""" + for marker in MARKERS: + expect(marker not in (text or ""), f"raw marker {marker!r} leaked into {where}: {text!r}") + + +def parse_arguments(call: dict[str, Any]) -> Any: + """Tool arguments must be a JSON string; return the decoded value.""" + args = call["function"]["arguments"] + expect(isinstance(args, str), f"arguments must be a JSON string, got {type(args).__name__}") + try: + return json.loads(args) + except json.JSONDecodeError as e: + raise AssertionError(f"arguments are not valid JSON ({e}): {args!r}") from None + + +def tool_schema(tools: list[dict[str, Any]], name: str) -> dict[str, Any]: + for tool in tools: + if tool["function"]["name"] == name: + return tool["function"].get("parameters", {}) + raise AssertionError(f"model called {name!r}, which is not one of the request's tools") + + +def check_tool_calls(calls: list[dict[str, Any]], tools: list[dict[str, Any]]) -> None: + """Every call is well formed, uses a declared tool and matches its schema.""" + ids = set() + for i, call in enumerate(calls): + expect(call.get("type") == "function", f"call {i}: type is {call.get('type')!r}") + expect(isinstance(call.get("id"), str) and TOOL_CALL_ID.match(call["id"]) is not None, + f"call {i}: id {call.get('id')!r} is not call_ + 24 alphanumerics") + expect(call["id"] not in ids, f"call {i}: id {call['id']} is not unique") + ids.add(call["id"]) + args = parse_arguments(call) + validate(args, tool_schema(tools, call["function"]["name"]), f"call {i} arguments") + + +def check_message_contract(choice: dict[str, Any]) -> dict[str, Any]: + """The non-streaming message shape every response must have.""" + msg = choice.get("message") + expect(isinstance(msg, dict), f"choice has no message: {choice!r}") + expect(msg.get("role") == "assistant", f"role is {msg.get('role')!r}") + expect(choice.get("finish_reason") in ("stop", "length", "tool_calls"), + f"unexpected finish_reason {choice.get('finish_reason')!r}") + if "reasoning_content" in msg: + expect(isinstance(msg["reasoning_content"], str) and msg["reasoning_content"] != "", + "reasoning_content, when present, must be a non-empty string") + assert_no_markers(msg.get("content"), "content") + assert_no_markers(msg.get("reasoning_content"), "reasoning_content") + if msg.get("tool_calls"): + expect(msg.get("content") is None or isinstance(msg["content"], str), + "content must be a string or null") + for i, call in enumerate(msg["tool_calls"]): + expect(call.get("index") == i, f"tool call {i} has index {call.get('index')!r}") + else: + expect(isinstance(msg.get("content"), str), "content must be a string when there are no tool calls") + expect(choice.get("finish_reason") != "tool_calls", "finish_reason is tool_calls without tool calls") + return msg + + +def check_stream_contract(s: Stream, *, include_usage: bool) -> Folded: + """The chunk-level shape every stream must have; returns the folded turn.""" + expect(len(s.chunks) > 0, "stream has no chunks") + ids = {c.get("id") for c in s.chunks} + expect(len(ids) == 1, f"chunks carry different ids: {ids}") + seen_tool_ids: dict[int, str] = {} + finish_chunks = 0 + for n, chunk in enumerate(s.chunks): + expect(chunk.get("object") == "chat.completion.chunk", f"chunk {n}: object is {chunk.get('object')!r}") + if include_usage: + expect("usage" in chunk, f"chunk {n}: include_usage asked for a usage key on every chunk") + else: + expect("usage" not in chunk, f"chunk {n}: usage sent although include_usage was not asked") + choices = chunk.get("choices") + expect(isinstance(choices, list), f"chunk {n}: choices is {choices!r}") + if not choices: + expect(include_usage and n == len(s.chunks) - 1, + f"chunk {n}: only the last chunk, with include_usage, may have no choices") + expect(isinstance(chunk.get("usage"), dict), "the usage chunk has no usage") + continue + choice = choices[0] + delta = choice.get("delta") or {} + if choice.get("finish_reason") is not None: + finish_chunks += 1 + continue + parts = [k for k in ("reasoning_content", "content", "tool_calls") if delta.get(k)] + expect(len(parts) == 1, f"chunk {n}: a delta must carry exactly one part, got {parts}: {delta}") + for tc in delta.get("tool_calls") or []: + index = tc.get("index") + expect(isinstance(index, int), f"chunk {n}: tool call delta without an index") + if index not in seen_tool_ids: + expect(index == len(seen_tool_ids), f"chunk {n}: tool call index {index} skips ahead") + expect(bool(tc.get("id")) and tc.get("type") == "function" and bool(tc["function"].get("name")), + f"chunk {n}: the first delta of a call must carry id, type and name: {tc}") + seen_tool_ids[index] = tc["id"] + else: + expect("id" not in tc and "name" not in (tc.get("function") or {}), + f"chunk {n}: a later delta of call {index} repeats its id or name: {tc}") + expect(finish_chunks == 1, f"expected one chunk with a finish_reason, got {finish_chunks}") + folded = fold_stream(s.chunks) + assert_no_markers(folded.content, "streamed content") + assert_no_markers(folded.reasoning, "streamed reasoning_content") + if include_usage: + expect(folded.usage is not None, "include_usage was asked but no usage arrived") + return folded + + +def assert_parity(api: Api, body: dict[str, Any], key: Callable[[dict[str, Any]], Any], + attempts: int = 3) -> Folded: + """The stream folds to the non-streaming message, compared through key. + + key maps a message dict (content, reasoning_content, tool_calls) to the + value to compare. Greedy decoding on the NPU is not always bit-reproducible + between identical requests, mostly over long outputs, so each mode is tried + up to ``attempts`` times: any matching pair passes (a parser disagreement + would be systematic), and no match is a failure only when the + non-streaming request reproduced itself; otherwise the case is skipped. + """ + + def from_message(msg: dict[str, Any]) -> dict[str, Any]: + return { + "content": msg.get("content") or None, + "reasoning_content": msg.get("reasoning_content", ""), + "tool_calls": msg.get("tool_calls") or [], + } + + plain: list[Any] = [] + streamed: list[Any] = [] + for _ in range(attempts): + plain.append(key(from_message(api.ok(body)["message"]))) + folded = check_stream_contract(api.ok_stream(body), include_usage=False) + streamed.append(key({ + "content": folded.content or None, + "reasoning_content": folded.reasoning, + "tool_calls": folded.tool_calls, + })) + if any(s == p for s in streamed for p in plain): + return folded + if any(p != plain[0] for p in plain): + raise Skip(f"greedy decoding did not reproduce over {attempts} identical requests; " + "cannot compare stream and non-stream") + raise AssertionError(f"stream and non-stream differ:\n stream: {streamed[0]!r}\n" + f" non-stream: {plain[0]!r}") + + +# ---------------------------------------------------------------- JSON Schema + + +def _type_ok(value: Any, kind: str) -> bool: + if kind == "object": + return isinstance(value, dict) + if kind == "array": + return isinstance(value, list) + if kind == "string": + return isinstance(value, str) + if kind == "boolean": + return isinstance(value, bool) + if kind == "null": + return value is None + if kind == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if kind == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + return True + + +def validate(value: Any, schema: Any, where: str = "value", root: Any = None) -> None: + """Check value against the JSON Schema keywords these tests use. + + Covers type, enum, const, properties, required, additionalProperties, + items, anyOf, minimum/maximum, pattern and local $ref. + """ + root = schema if root is None else root + if schema is True or not isinstance(schema, dict): + return + if "$ref" in schema: + ref = schema["$ref"] + expect(ref.startswith("#/"), f"{where}: only local $ref is supported, got {ref}") + target = root + for part in ref[2:].split("/"): + target = target[part] + validate(value, target, where, root) + return + if "anyOf" in schema: + errors = [] + for option in schema["anyOf"]: + try: + validate(value, option, where, root) + break + except AssertionError as e: + errors.append(str(e)) + else: + raise AssertionError(f"{where}: matches no anyOf option: {errors}") + kind = schema.get("type") + if isinstance(kind, list): + expect(any(_type_ok(value, k) for k in kind), f"{where}: {value!r} is not any of {kind}") + elif isinstance(kind, str): + expect(_type_ok(value, kind), f"{where}: {value!r} is not of type {kind}") + if "enum" in schema: + expect(value in schema["enum"], f"{where}: {value!r} is not one of {schema['enum']}") + if "const" in schema: + expect(value == schema["const"], f"{where}: {value!r} != const {schema['const']!r}") + if isinstance(value, (int, float)) and not isinstance(value, bool): + if "minimum" in schema: + expect(value >= schema["minimum"], f"{where}: {value} < minimum {schema['minimum']}") + if "maximum" in schema: + expect(value <= schema["maximum"], f"{where}: {value} > maximum {schema['maximum']}") + if isinstance(value, str) and "pattern" in schema: + expect(re.search(schema["pattern"], value) is not None, + f"{where}: {value!r} does not match {schema['pattern']}") + if isinstance(value, dict): + props = schema.get("properties", {}) + for name in schema.get("required", []): + expect(name in value, f"{where}: required key {name!r} is missing from {value!r}") + for name, item in value.items(): + if name in props: + validate(item, props[name], f"{where}.{name}", root) + elif schema.get("additionalProperties") is False: + raise AssertionError(f"{where}: unexpected key {name!r}") + if isinstance(value, list) and "items" in schema: + for i, item in enumerate(value): + validate(item, schema["items"], f"{where}[{i}]", root) + + +# ---------------------------------------------------------------- runner + +Case = Callable[[Api], None] + + +def case(func: Case) -> Case: + """Mark a test function; its docstring's first line is its description.""" + func.is_case = True # type: ignore[attr-defined] + return func + + +def collect(module_globals: dict[str, Any]) -> list[Case]: + """The cases of a module, in definition order.""" + return [f for f in module_globals.values() if callable(f) and getattr(f, "is_case", False)] + + +@dataclass +class Tally: + passed: int = 0 + failed: int = 0 + skipped: int = 0 + + def add(self, other: Tally) -> None: + self.passed += other.passed + self.failed += other.failed + self.skipped += other.skipped + + +def run_cases(cases: list[Case], api: Api, *, only: str | None = None, verbose: bool = False) -> Tally: + """Run cases and print one line per case.""" + tally = Tally() + for func in cases: + module = os.path.splitext(os.path.basename(func.__globals__.get("__file__", func.__module__)))[0] + name = f"{module}.{func.__name__}" + if only and only not in name: + continue + start = time.monotonic() + try: + func(api) + except Skip as e: + tally.skipped += 1 + print(f"SKIP {name}: {e}") + continue + except AssertionError as e: + tally.failed += 1 + print(f"FAIL {name} ({time.monotonic() - start:.1f}s): {e}") + if verbose: + traceback.print_exc() + continue + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + tally.failed += 1 + print(f"FAIL {name}: cannot reach the server: {e}") + continue + tally.passed += 1 + print(f"PASS {name} ({time.monotonic() - start:.1f}s)") + return tally + + +def parse_args(description: str) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=description) + parser.add_argument("model", help="model tag currently loaded by the FastFlowLM server") + parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT, help="server base URL (default: %(default)s)") + parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT, + help="per-request timeout in seconds (default: %(default)s)") + parser.add_argument("--always-thinks", action="store_true", + help="the model cannot turn thinking off (e.g. qwen3-tk, deepseek-r1, gpt-oss): " + "do not require think=false to drop reasoning_content") + parser.add_argument("-k", dest="only", help="run only cases whose name contains this text") + parser.add_argument("-v", "--verbose", action="store_true", help="print tracebacks of failures") + return parser.parse_args() + + +def make_api(args: argparse.Namespace) -> Api: + return Api(args.endpoint, args.model, args.timeout, always_thinks=args.always_thinks) + + +def summarize(tally: Tally) -> int: + print(f"\n{tally.passed} passed, {tally.failed} failed, {tally.skipped} skipped") + return 1 if tally.failed else 0 + + +def main(cases: list[Case], description: str) -> None: + """Entry point for running one test module on its own.""" + args = parse_args(description) + api = make_api(args) + sys.exit(summarize(run_cases(cases, api, only=args.only, verbose=args.verbose))) diff --git a/src/test/openai_api_test/run_all.py b/src/test/openai_api_test/run_all.py new file mode 100644 index 000000000..f1cb9b6f7 --- /dev/null +++ b/src/test/openai_api_test/run_all.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Run every OpenAI API test module against one running server. + +Usage:: + + python3 run_all.py [--always-thinks] [--endpoint URL] [--timeout T] [-k NAME] [-v] + +Exits 0 when nothing failed (skips are fine), 1 otherwise. +""" + +from __future__ import annotations + +import sys + +import flm_api +import test_errors +import test_reasoning +import test_response_format +import test_response_shape +import test_tool_calls + +MODULES = [test_response_shape, test_reasoning, test_tool_calls, test_response_format, test_errors] + + +def main() -> None: + args = flm_api.parse_args(__doc__.splitlines()[0]) + api = flm_api.make_api(args) + total = flm_api.Tally() + for module in MODULES: + print(f"\n== {module.__name__}") + total.add(flm_api.run_cases(module.CASES, api, only=args.only, verbose=args.verbose)) + sys.exit(flm_api.summarize(total)) + + +if __name__ == "__main__": + main() diff --git a/src/test/openai_api_test/test_errors.py b/src/test/openai_api_test/test_errors.py new file mode 100644 index 000000000..ecedaba45 --- /dev/null +++ b/src/test/openai_api_test/test_errors.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Requests the server must refuse with HTTP 400 instead of serving loosely. + +Each malformed tool_choice / response_format is rejected before any +generation starts, with an OpenAI style error body +(``{"error": {"message", "type": "invalid_request_error", "code": 400}}``), +for streaming requests too. A good request afterwards must still work: +a rejection must not leave the server in a bad state. + +Usage:: + + python3 test_errors.py [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +from typing import Any + +import flm_api +from flm_api import Api, case, expect +from fixtures import WEATHER_TOOLS, function_choice, json_schema_format, user + + +def expect_400(api: Api, fields: dict[str, Any], message_part: str) -> None: + """Non-streaming and streaming both refuse, naming the problem.""" + body = api.body(user("hi"), max_tokens=8, **fields) + resp = api.post(body) + expect(resp.status == 400, f"expected HTTP 400, got {resp.status}: {resp.body}") + err = resp.body.get("error") + expect(isinstance(err, dict), f"error body is not an object: {resp.body}") + expect(err.get("type") == "invalid_request_error", f"error type is {err.get('type')!r}") + expect(message_part in str(err.get("message")), f"message {err.get('message')!r} lacks {message_part!r}") + + s = api.stream(body) + expect(s.status == 400, f"streaming: expected HTTP 400, got {s.status}") + expect(not s.chunks, "streaming: chunks were sent before the rejection") + + +@case +def bad_tool_choice_string(api: Api) -> None: + """An unknown tool_choice string.""" + expect_400(api, {"tools": WEATHER_TOOLS, "tool_choice": "sometimes"}, "tool_choice must be") + + +@case +def bad_tool_choice_object(api: Api) -> None: + """A tool_choice object without function.name.""" + expect_400(api, {"tools": WEATHER_TOOLS, "tool_choice": {"type": "function"}}, "tool_choice must be") + + +@case +def required_without_tools(api: Api) -> None: + """required with no tools to call.""" + expect_400(api, {"tool_choice": "required"}, "has no tools") + + +@case +def function_without_tools(api: Api) -> None: + """A forced function with no tools at all.""" + expect_400(api, {"tool_choice": function_choice("get_weather")}, "has no tools") + + +@case +def unknown_function(api: Api) -> None: + """A forced function that is not among the tools.""" + expect_400(api, {"tools": WEATHER_TOOLS, "tool_choice": function_choice("launch_rocket")}, + "not in tools") + + +@case +def bad_response_format_type(api: Api) -> None: + """An unknown response_format type.""" + expect_400(api, {"response_format": {"type": "xml"}}, "response_format type must be") + + +@case +def json_schema_without_schema(api: Api) -> None: + """json_schema without its schema.""" + expect_400(api, {"response_format": {"type": "json_schema", "json_schema": {"name": "x"}}}, + "needs json_schema.schema") + + +@case +def uncompilable_schema(api: Api) -> None: + """A schema the grammar compiler cannot build: a dangling $ref.""" + schema = {"type": "object", "properties": {"a": {"$ref": "#/$defs/Missing"}}, "required": ["a"]} + expect_400(api, {"response_format": json_schema_format("bad", schema)}, "cannot build the decoding grammar") + + +@case +def server_still_serves(api: Api) -> None: + """After the rejections a normal request still succeeds.""" + flm_api.check_message_contract(api.ok(api.body(user("Say hi."), max_tokens=16))) + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_reasoning.py b/src/test/openai_api_test/test_reasoning.py new file mode 100644 index 000000000..aacaa0058 --- /dev/null +++ b/src/test/openai_api_test/test_reasoning.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Reasoning output and the per-request thinking switch. + +* With ``think: true`` a reasoning model's thinking comes back in + ``reasoning_content``, never as ```` markup in ``content``. +* With ``think: false`` (or ``reasoning_effort: "none"``) there is no + ``reasoning_content``, unless the model cannot stop thinking: pass + ``--always-thinks`` for those. +* A stream sends every reasoning delta before the first content delta and + folds to exactly the non-streaming message, so no half-written tag ever + went out as text. + +Models that never reason skip the thinking-on cases. + +Usage:: + + python3 test_reasoning.py [--always-thinks] [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +import flm_api +from flm_api import Api, Skip, case, expect +from fixtures import user + +# Short to think about, so a small budget still reaches the answer. +PROMPT = "Is 7 a prime number? Answer with yes or no." +THINK_BUDGET = 1536 + + +def require_reasoning(msg: dict) -> None: + if not msg.get("reasoning_content"): + raise Skip("no reasoning_content with think=true: the model does not reason") + + +@case +def think_on_nonstream(api: Api) -> None: + """think=true puts the thinking in reasoning_content and only the answer in content.""" + choice = api.ok(api.body(user(PROMPT), think=True, max_tokens=THINK_BUDGET)) + msg = flm_api.check_message_contract(choice) + require_reasoning(msg) + if choice["finish_reason"] == "stop": + expect(msg["content"].strip() != "", "the model stopped after thinking without an answer") + + +@case +def think_on_stream(api: Api) -> None: + """A stream sends all reasoning before any content, and folds to the non-streaming message.""" + body = api.body(user(PROMPT), think=True, max_tokens=THINK_BUDGET) + folded = flm_api.check_stream_contract(api.ok_stream(body), include_usage=False) + if folded.reasoning == "": + raise Skip("no reasoning_content with think=true: the model does not reason") + if "content" in folded.order: + first_content = folded.order.index("content") + expect("reasoning" not in folded.order[first_content:], + "a reasoning delta arrived after content had started") + # long outputs do not reproduce bit for bit between requests: compare a short one + short = api.body(user(PROMPT), think=True, max_tokens=64) + flm_api.assert_parity(api, short, lambda m: (m["reasoning_content"], m["content"])) + + +@case +def think_off_nonstream(api: Api) -> None: + """think=false returns no reasoning_content.""" + choice = api.ok(api.body(user(PROMPT), think=False, max_tokens=128)) + msg = flm_api.check_message_contract(choice) + if api.always_thinks: + return + expect("reasoning_content" not in msg, + f"think=false still returned reasoning_content (pass --always-thinks for models that " + f"cannot stop thinking): {msg.get('reasoning_content', '')[:120]!r}") + expect(msg["content"].strip() != "", "content is empty") + + +@case +def think_off_stream(api: Api) -> None: + """think=false streams no reasoning_content deltas.""" + s = api.ok_stream(api.body(user(PROMPT), think=False, max_tokens=128)) + folded = flm_api.check_stream_contract(s, include_usage=False) + if not api.always_thinks: + expect(folded.reasoning == "", f"think=false streamed reasoning: {folded.reasoning[:120]!r}") + + +@case +def reasoning_effort_none(api: Api) -> None: + """reasoning_effort "none" turns thinking off like think=false.""" + body = api.body(user(PROMPT), max_tokens=128, reasoning_effort="none") + del body["think"] + msg = flm_api.check_message_contract(api.ok(body)) + if not api.always_thinks: + expect("reasoning_content" not in msg, "reasoning_effort=none still returned reasoning_content") + + +@case +def think_switch_is_per_request(api: Api) -> None: + """Off, on, off again: each request gets what it asked for.""" + off = api.ok(api.body(user(PROMPT), think=False, max_tokens=128))["message"] + on = api.ok(api.body(user(PROMPT), think=True, max_tokens=THINK_BUDGET))["message"] + require_reasoning(on) + off_again = api.ok(api.body(user(PROMPT), think=False, max_tokens=128))["message"] + if not api.always_thinks: + expect("reasoning_content" not in off and "reasoning_content" not in off_again, + "think=false returned reasoning_content after a think=true request") + + +@case +def history_with_reasoning(api: Api) -> None: + """A previous turn's reasoning_content sent back in history is accepted (and dropped).""" + first = api.ok(api.body(user("Pick a random fruit. One word."), think=True, max_tokens=THINK_BUDGET))["message"] + messages = user("Pick a random fruit. One word.") + [ + {"role": "assistant", "content": first.get("content") or "apple", + "reasoning_content": first.get("reasoning_content", "")}, + {"role": "user", "content": "What color is it usually? One word."}, + ] + choice = api.ok(api.body(messages, think=False, max_tokens=64)) + flm_api.check_message_contract(choice) + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_response_format.py b/src/test/openai_api_test/test_response_format.py new file mode 100644 index 000000000..d19885967 --- /dev/null +++ b/src/test/openai_api_test/test_response_format.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""response_format: JSON mode and JSON schema, streamed and not. + +The grammar guarantees the shape of ``content`` whatever the model, so +every finished answer (``finish_reason`` ``stop``) must parse as JSON and +validate against the request's schema. Covered: + +* ``json_object`` gives a JSON object; +* ``json_schema`` with flat properties, a string enum, Pydantic style + ``$defs``/``$ref`` nesting, and const / anyOf / bounds / pattern; +* the same through a stream; +* with thinking on, reasoning stays free text and only content is JSON; +* with thinking off, there is no reasoning at all; +* ``text`` is the same as sending nothing. + +Usage:: + + python3 test_response_format.py [--always-thinks] [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +import json +from typing import Any + +import flm_api +from flm_api import Api, case, expect +from fixtures import ( + ORDER_SCHEMA, + PERSON_SCHEMA, + REVIEW_SCHEMA, + SENTIMENT_SCHEMA, + WEATHER_TOOLS, + json_schema_format, + user, +) + + +def expect_json(content: str | None, finish_reason: str | None, schema: Any = None) -> Any: + """content is complete JSON matching schema; returns the decoded value.""" + expect(finish_reason == "stop", f"finish_reason is {finish_reason!r}, the JSON may be cut off") + expect(isinstance(content, str), f"content is {content!r}") + try: + value = json.loads(content) + except json.JSONDecodeError as e: + raise AssertionError(f"content is not valid JSON ({e}): {content!r}") from None + if schema is not None: + flm_api.validate(value, schema, "content") + return value + + +def nonstream_json(api: Api, body: dict, schema: Any) -> tuple[dict, Any]: + choice = api.ok(body) + msg = flm_api.check_message_contract(choice) + expect("tool_calls" not in msg, "a response_format answer has tool_calls") + return msg, expect_json(msg["content"], choice["finish_reason"], schema) + + +# ---------------------------------------------------------------- json_object + + +@case +def json_object_nonstream(api: Api) -> None: + """json_object returns a JSON object.""" + body = api.body(user("Describe the color red as JSON with keys name and hex."), + response_format={"type": "json_object"}) + _, value = nonstream_json(api, body, {"type": "object"}) + expect(isinstance(value, dict), f"json_object returned {type(value).__name__}") + + +@case +def json_object_stream(api: Api) -> None: + """json_object, streamed: the folded content is a JSON object.""" + body = api.body(user("Describe the color red as JSON with keys name and hex."), + response_format={"type": "json_object"}) + folded = flm_api.check_stream_contract(api.ok_stream(body), include_usage=False) + expect_json(folded.content, folded.finish_reason, {"type": "object"}) + + +# ---------------------------------------------------------------- json_schema + + +@case +def schema_flat(api: Api) -> None: + """A flat schema with required string and integer fields.""" + body = api.body(user("Extract the person: Alice is 31 years old and lives in Berlin."), + response_format=json_schema_format("person", PERSON_SCHEMA)) + nonstream_json(api, body, PERSON_SCHEMA) + + +@case +def schema_enum(api: Api) -> None: + """A string enum only lets the listed labels through.""" + body = api.body(user("Classify the sentiment of: 'I loved this movie!'"), + response_format=json_schema_format("sentiment", SENTIMENT_SCHEMA), max_tokens=48) + nonstream_json(api, body, SENTIMENT_SCHEMA) + + +@case +def schema_nested_refs(api: Api) -> None: + """Pydantic style $defs/$ref nesting, a list of objects.""" + body = api.body(user("Bob orders 2 apples and 3 pears. Extract the order."), + response_format=json_schema_format("Order", ORDER_SCHEMA)) + nonstream_json(api, body, ORDER_SCHEMA) + + +@case +def schema_keywords(api: Api) -> None: + """const, anyOf, integer bounds and a string pattern.""" + body = api.body(user("Review a margherita pizza eaten on 2026-03-14: score it 1-5 and say if it is vegetarian."), + response_format=json_schema_format("review", REVIEW_SCHEMA)) + nonstream_json(api, body, REVIEW_SCHEMA) + + +@case +def schema_stream(api: Api) -> None: + """json_schema, streamed: the folded content validates.""" + body = api.body(user("Bob orders 2 apples and 3 pears. Extract the order."), + response_format=json_schema_format("Order", ORDER_SCHEMA)) + folded = flm_api.check_stream_contract(api.ok_stream(body), include_usage=False) + expect(folded.tool_calls == [], "a response_format answer streamed tool calls") + expect_json(folded.content, folded.finish_reason, ORDER_SCHEMA) + + +@case +def schema_with_thinking(api: Api) -> None: + """Thinking stays free text; only content is held to the schema.""" + body = api.body(user("Extract the person: Alice is 31 years old and lives in Berlin."), + think=True, max_tokens=2048, + response_format=json_schema_format("person", PERSON_SCHEMA)) + msg, _ = nonstream_json(api, body, PERSON_SCHEMA) + flm_api.assert_no_markers(msg.get("reasoning_content"), "reasoning_content") + + +@case +def schema_without_thinking(api: Api) -> None: + """With thinking off the grammar leaves no room for a reasoning block.""" + body = api.body(user("Extract the person: Alice is 31 years old and lives in Berlin."), + think=False, response_format=json_schema_format("person", PERSON_SCHEMA)) + msg, _ = nonstream_json(api, body, PERSON_SCHEMA) + if not api.always_thinks: + expect("reasoning_content" not in msg, "think=false with a schema still produced reasoning_content") + + +@case +def schema_beats_auto_tools(api: Api) -> None: + """With tool_choice auto, response_format wins: JSON content and no calls.""" + body = api.body(user("Extract the person: Alice is 31 years old and lives in Berlin."), + tools=WEATHER_TOOLS, response_format=json_schema_format("person", PERSON_SCHEMA)) + nonstream_json(api, body, PERSON_SCHEMA) + + +@case +def text_format(api: Api) -> None: + """response_format text is plain, unconstrained output.""" + choice = api.ok(api.body(user("Say hello."), response_format={"type": "text"}, max_tokens=32)) + flm_api.check_message_contract(choice) + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_response_shape.py b/src/test/openai_api_test/test_response_shape.py new file mode 100644 index 000000000..9c52416ec --- /dev/null +++ b/src/test/openai_api_test/test_response_shape.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Response shape of /v1/chat/completions, streamed and not. + +Checks the contract every client relies on, whatever the model: + +* a non-streaming message has the OpenAI fields, no raw template markup, + and a finish_reason that agrees with its tool calls; +* every stream chunk carries exactly one kind of delta, one stream id, + one finish chunk, then ``data: [DONE]``; +* usage is only streamed when ``stream_options.include_usage`` asks, as + one last chunk with no choices; +* streaming and non-streaming produce the same message: the server + parses both with one parser, so with greedy sampling they must agree + (skipped when greedy decoding itself does not reproduce). + +Usage:: + + python3 test_response_shape.py [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +import flm_api +from flm_api import Api, case, expect +from fixtures import WEATHER_TOOLS, function_choice, user + +PROMPT = "In one short sentence, what is the capital of France?" + + +@case +def nonstream_message(api: Api) -> None: + """A plain answer has role, string content and a usage block that adds up.""" + resp = api.post(api.body(user(PROMPT), max_tokens=64)) + expect(resp.status == 200, f"HTTP {resp.status}: {resp.error_message()}") + expect(resp.body.get("object") == "chat.completion", f"object is {resp.body.get('object')!r}") + choice = resp.body["choices"][0] + msg = flm_api.check_message_contract(choice) + expect(msg["content"].strip() != "", "content is empty") + expect("tool_calls" not in msg, "no tools were offered, yet the message has tool_calls") + usage = resp.body.get("usage") or {} + for key in ("prompt_tokens", "completion_tokens", "total_tokens"): + expect(isinstance(usage.get(key), int), f"usage.{key} missing: {usage}") + expect(usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"], + f"usage does not add up: {usage}") + + +@case +def stream_without_usage(api: Api) -> None: + """Without include_usage no chunk carries a usage key.""" + s = api.ok_stream(api.body(user(PROMPT), max_tokens=64)) + folded = flm_api.check_stream_contract(s, include_usage=False) + expect(folded.content.strip() != "", "streamed content is empty") + expect(folded.finish_reason in ("stop", "length"), f"finish_reason {folded.finish_reason!r}") + + +@case +def stream_with_usage(api: Api) -> None: + """include_usage puts usage: null on every chunk and the totals on a last, choice-less chunk.""" + s = api.ok_stream(api.body(user(PROMPT), max_tokens=64, stream_options={"include_usage": True})) + folded = flm_api.check_stream_contract(s, include_usage=True) + last = s.chunks[-1] + expect(last["choices"] == [], "the usage chunk must be the last one, with no choices") + for chunk in s.chunks[:-1]: + expect(chunk["usage"] is None, "usage must be null on every chunk but the last") + expect(folded.usage["completion_tokens"] > 0, f"usage reports no completion tokens: {folded.usage}") + + +@case +def length_finish(api: Api) -> None: + """Running out of max_tokens reports finish_reason "length" in both modes.""" + body = api.body(user("Count from 1 to 100, separated by commas."), max_tokens=8) + choice = api.ok(body) + expect(choice["finish_reason"] == "length", f"non-stream finish_reason is {choice['finish_reason']!r}") + folded = flm_api.check_stream_contract(api.ok_stream(body), include_usage=False) + expect(folded.finish_reason == "length", f"stream finish_reason is {folded.finish_reason!r}") + + +@case +def stream_matches_nonstream(api: Api) -> None: + """Greedy streaming and non-streaming answers are the same text.""" + body = api.body(user("Name three primary colors, comma separated."), max_tokens=48) + flm_api.assert_parity(api, body, lambda m: (m["reasoning_content"], m["content"])) + + +@case +def stream_matches_nonstream_tools(api: Api) -> None: + """With a forced call, both modes return the same call name and arguments.""" + body = api.body(user("What's the weather in Paris?"), tools=WEATHER_TOOLS, + tool_choice=function_choice("get_weather")) + flm_api.assert_parity( + api, body, lambda m: [(c["function"]["name"], c["function"]["arguments"]) for c in m["tool_calls"]]) + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_tool_calls.py b/src/test/openai_api_test/test_tool_calls.py new file mode 100644 index 000000000..dc62c1cbc --- /dev/null +++ b/src/test/openai_api_test/test_tool_calls.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Tool calling: parsing, streaming deltas and every tool_choice mode. + +What the server guarantees, and so what is checked, per mode: + +* ``auto`` calls, if any, name a declared tool and follow its schema +* ``required`` at least one such call +* function exactly one call, to that function, and no content +* ``none`` no calls at all; the tools are not even shown to the model +* ``parallel_tool_calls: false`` at most one call + +Every call must have an OpenAI-style id, a JSON-string ``arguments`` that +validates against the tool's ``parameters``, and ``finish_reason`` +``tool_calls``. Streamed calls must open with id/type/name and only +append arguments after that. + +Whether a model *chooses* to call a tool under ``auto`` is up to the model, +so those cases check the shape only. Models whose chat format has no tool +call syntax skip everything that forces a call. + +Usage:: + + python3 test_tool_calls.py [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +import json + +import flm_api +from flm_api import Api, Skip, case, expect +from fixtures import BOOK_FLIGHT, GET_WEATHER, WEATHER_TOOLS, function_choice, user + +WEATHER_PROMPT = "What's the weather in Paris right now?" +TWO_CITIES_PROMPT = "What's the weather in Paris and in Tokyo?" + + +def expect_calls(choice: dict, tools: list[dict], *, at_least: int = 1) -> list[dict]: + """A finished turn with valid calls; returns them.""" + msg = flm_api.check_message_contract(choice) + expect(choice["finish_reason"] != "length", "ran out of max_tokens inside a tool call") + calls = msg.get("tool_calls") or [] + expect(len(calls) >= at_least, f"expected at least {at_least} tool call(s), got {len(calls)}: {msg}") + flm_api.check_tool_calls(calls, tools) + expect(choice["finish_reason"] == "tool_calls", f"finish_reason is {choice['finish_reason']!r}") + return calls + + +def expect_streamed_calls(api: Api, body: dict, tools: list[dict], *, at_least: int = 1) -> list[dict]: + """The streamed version of expect_calls.""" + folded = flm_api.check_stream_contract(api.ok_stream(body), include_usage=False) + expect(folded.finish_reason != "length", "ran out of max_tokens inside a tool call") + expect(len(folded.tool_calls) >= at_least, + f"expected at least {at_least} streamed tool call(s), got {len(folded.tool_calls)}") + flm_api.check_tool_calls(folded.tool_calls, tools) + expect(folded.finish_reason == "tool_calls", f"finish_reason is {folded.finish_reason!r}") + return folded.tool_calls + + +# ---------------------------------------------------------------- auto + + +@case +def auto_nonstream(api: Api) -> None: + """tool_choice auto: any calls the model makes are valid.""" + choice = api.ok(api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS)) + msg = flm_api.check_message_contract(choice) + if msg.get("tool_calls"): + expect_calls(choice, WEATHER_TOOLS) + else: + expect(choice["finish_reason"] in ("stop", "length"), f"finish_reason {choice['finish_reason']!r}") + + +@case +def auto_stream(api: Api) -> None: + """tool_choice auto, streamed: any calls are well formed and valid.""" + folded = flm_api.check_stream_contract( + api.ok_stream(api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS)), include_usage=False) + if folded.tool_calls and folded.finish_reason != "length": + flm_api.check_tool_calls(folded.tool_calls, WEATHER_TOOLS) + expect(folded.finish_reason == "tool_calls", f"finish_reason {folded.finish_reason!r}") + + +@case +def no_tools_no_calls(api: Api) -> None: + """Without tools in the request nothing is parsed as a tool call.""" + msg = flm_api.check_message_contract(api.ok(api.body(user(WEATHER_PROMPT), max_tokens=96))) + expect("tool_calls" not in msg, f"a request without tools returned tool_calls: {msg}") + + +# ---------------------------------------------------------------- required + + +@case +def required_nonstream(api: Api) -> None: + """tool_choice required: at least one valid call.""" + expect_calls(api.ok(api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice="required")), + WEATHER_TOOLS) + + +@case +def required_stream(api: Api) -> None: + """tool_choice required, streamed: at least one valid call.""" + expect_streamed_calls(api, api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice="required"), + WEATHER_TOOLS) + + +@case +def required_even_when_off_topic(api: Api) -> None: + """required forces a call even when the prompt does not need a tool.""" + expect_calls(api.ok(api.body(user("Tell me a joke about cats."), tools=WEATHER_TOOLS, tool_choice="required")), + WEATHER_TOOLS) + + +@case +def required_single_call(api: Api) -> None: + """required + parallel_tool_calls=false: exactly one call, even when two are asked for.""" + body = api.body(user(TWO_CITIES_PROMPT), tools=WEATHER_TOOLS, tool_choice="required", + parallel_tool_calls=False) + calls = expect_calls(api.ok(body), WEATHER_TOOLS) + expect(len(calls) == 1, f"parallel_tool_calls=false allowed {len(calls)} calls") + calls = expect_streamed_calls(api, body, WEATHER_TOOLS) + expect(len(calls) == 1, f"parallel_tool_calls=false allowed {len(calls)} streamed calls") + + +# ---------------------------------------------------------------- forced function + + +@case +def function_nonstream(api: Api) -> None: + """Forcing a function returns exactly one call to it and no content.""" + body = api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice=function_choice("get_time")) + choice = api.ok(body) + calls = expect_calls(choice, WEATHER_TOOLS) + expect(len(calls) == 1, f"a forced function produced {len(calls)} calls") + expect(calls[0]["function"]["name"] == "get_time", f"forced get_time, got {calls[0]['function']['name']}") + expect(choice["message"].get("content") is None, + f"a forced call must leave content null, got {choice['message'].get('content')!r}") + + +@case +def function_stream(api: Api) -> None: + """Forcing a function, streamed: one call to it, arguments sent as growing JSON.""" + body = api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice=function_choice("get_time")) + s = api.ok_stream(body) + folded = flm_api.check_stream_contract(s, include_usage=False) + expect(folded.content == "", f"a forced call streamed content: {folded.content!r}") + expect([c["function"]["name"] for c in folded.tool_calls] == ["get_time"], + f"expected one get_time call, got {folded.tool_calls}") + flm_api.check_tool_calls(folded.tool_calls, WEATHER_TOOLS) + # each prefix of the streamed arguments is a prefix of the final JSON + pieces = [tc["function"].get("arguments", "") + for chunk in s.chunks for choice in chunk.get("choices") or [] + for tc in (choice.get("delta") or {}).get("tool_calls") or []] + expect("".join(pieces) == folded.tool_calls[0]["function"]["arguments"], "argument pieces do not add up") + + +@case +def function_typed_arguments(api: Api) -> None: + """Integers, numbers, booleans, enums and typed arrays all follow the schema.""" + prompt = ("Book 2 passengers from Paris to Rome in business class, refundable, " + "at most 450.50 euros, one vegan and one standard meal.") + choice = api.ok(api.body(user(prompt), tools=[BOOK_FLIGHT], tool_choice=function_choice("book_flight"))) + calls = expect_calls(choice, [BOOK_FLIGHT]) + args = json.loads(calls[0]["function"]["arguments"]) + expect(isinstance(args["passengers"], int) and not isinstance(args["passengers"], bool), + f"passengers is not an integer: {args['passengers']!r}") + expect(isinstance(args["refundable"], bool), f"refundable is not a boolean: {args['refundable']!r}") + + +@case +def function_with_thinking(api: Api) -> None: + """A forced call still works when the model thinks first.""" + body = api.body(user(WEATHER_PROMPT), think=True, max_tokens=1536, tools=WEATHER_TOOLS, + tool_choice=function_choice("get_weather")) + calls = expect_calls(api.ok(body), WEATHER_TOOLS) + expect(calls[0]["function"]["name"] == "get_weather", "wrong function called") + + +@case +def non_strict_tool(api: Api) -> None: + """strict=false only requires well-formed JSON arguments.""" + loose = json.loads(json.dumps(GET_WEATHER)) + loose["function"]["strict"] = False + loose["function"]["parameters"]["properties"]["city"] = {"type": "integer"} + choice = api.ok(api.body(user(WEATHER_PROMPT), tools=[loose], tool_choice="required")) + msg = flm_api.check_message_contract(choice) + calls = msg.get("tool_calls") or [] + expect(len(calls) >= 1, "required produced no call") + flm_api.parse_arguments(calls[0]) # valid JSON; the schema is not enforced + + +# ---------------------------------------------------------------- none + + +@case +def none_nonstream(api: Api) -> None: + """tool_choice none: plain text, no calls, no tool markup.""" + choice = api.ok(api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice="none", max_tokens=96)) + msg = flm_api.check_message_contract(choice) + expect("tool_calls" not in msg, f"tool_choice none returned tool calls: {msg}") + expect(choice["finish_reason"] in ("stop", "length"), f"finish_reason {choice['finish_reason']!r}") + + +@case +def none_stream(api: Api) -> None: + """tool_choice none, streamed: no tool call deltas.""" + s = api.ok_stream(api.body(user(WEATHER_PROMPT), tools=WEATHER_TOOLS, tool_choice="none", max_tokens=96)) + folded = flm_api.check_stream_contract(s, include_usage=False) + expect(folded.tool_calls == [], f"tool_choice none streamed tool calls: {folded.tool_calls}") + + +# ---------------------------------------------------------------- multi turn + + +@case +def round_trip(api: Api) -> None: + """A call, its tool result, then a final answer that uses the result.""" + messages = user(WEATHER_PROMPT) + first = api.ok(api.body(messages, tools=WEATHER_TOOLS, tool_choice=function_choice("get_weather"))) + call = expect_calls(first, WEATHER_TOOLS)[0] + messages += [ + {"role": "assistant", "content": None, "tool_calls": [ + {"id": call["id"], "type": "function", "function": call["function"]}]}, + {"role": "tool", "tool_call_id": call["id"], + "content": json.dumps({"temperature_c": 23, "sky": "sunny"})}, + ] + resp = api.post(api.body(messages, tools=WEATHER_TOOLS, max_tokens=128)) + if resp.status != 200: + flm_api.raise_for_capability(resp.error_message()) + raise AssertionError(f"HTTP {resp.status}: {resp.error_message()}") + choice = resp.body["choices"][0] + msg = flm_api.check_message_contract(choice) + if msg.get("tool_calls"): + raise Skip("the model called another tool instead of answering; nothing to check") + expect("23" in (msg["content"] or ""), f"the answer does not use the tool result: {msg['content']!r}") + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) From 3fad463e94b911227a71b5a3220f3cdf7145c833 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:04:10 -0700 Subject: [PATCH 13/18] docs: update sidebar --- docs/_config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_config.yml b/docs/_config.yml index d9f1eb302..2c998ed01 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -87,6 +87,12 @@ docs_nav: url: "/docs/instructions/server/webui/" - label: Tool Calling url: "/docs/instructions/server/tool_calling/" + - label: Reasoning & Tool Calls + url: "/docs/instructions/server/reasoning_and_tools/" + - label: Structured Output + url: "/docs/instructions/server/structured_output/" + - label: Output Parsing Internals + url: "/docs/instructions/server/chat_output_internals/" - label: LangChain RAG url: "/docs/instructions/server/rag_LangChain/" - label: LangChain Web Search From 444a1e4f4cc63259321aa759c75df0c6b4a4c063 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:28:30 -0700 Subject: [PATCH 14/18] fix(chat): keep the grammar when the prompt opens a reasoning block When a request turns thinking off but the template still opens (a model without a thinking switch), the prefill was rejected and the constraint gave up for the whole request, leaving response_format and tool_choice unenforced. start() now falls back to the precompiled grammar with reasoning, as accept_forced() already did; both share use_reasoning_grammar(). Co-Authored-By: Claude --- .../server/chat_output_internals.md | 3 +- src/common/chat/chat_grammar.cpp | 41 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/docs/docs/instructions/server/chat_output_internals.md b/docs/docs/instructions/server/chat_output_internals.md index b5e55de8c..b7d9f466c 100644 --- a/docs/docs/instructions/server/chat_output_internals.md +++ b/docs/docs/instructions/server/chat_output_internals.md @@ -168,7 +168,8 @@ The EBNF argument grammars (`argument_list_rules()`) put declared properties in - `start(prefill)` feeds the prefill (Harmony drops the leading `assistant` role) into the matcher; - `fill_mask()` fills a `uint32` bitset over the vocabulary; - `accept(token)` advances; `rollback(n)` undoes; -- `accept_forced(token)` handles a token the model is *made* to emit. If thinking was off in the grammar but a thinking-only model is forced to open ``, the constraint switches to a precompiled grammar that has reasoning and replays its history into it; +- `accept_forced(token)` handles a token the model is *made* to emit; +- if thinking was off in the grammar but the model reasons anyway, because the template opened `` in the prefill or the model is forced to emit it, `start()` / `accept_forced()` switch to a precompiled grammar that has reasoning and replay the prefill and history into it; - if the matcher ever rejects a token, `give_up()` logs `Constrained decoding disabled for this request: …` and stops masking. A desynchronised grammar would only produce garbage. ### The decode loop diff --git a/src/common/chat/chat_grammar.cpp b/src/common/chat/chat_grammar.cpp index 4d0219ce0..77f4bdfb0 100644 --- a/src/common/chat/chat_grammar.cpp +++ b/src/common/chat/chat_grammar.cpp @@ -466,7 +466,12 @@ class xgrammar_constraint : public decode_constraint { text = text.substr(skip_.size()); } prefill_ = text; - if (!text.empty() && !matcher_.AcceptString(text)) { + if (text.empty() || matcher_.AcceptString(text)) { + return; + } + // the template opened a reasoning block although the request turned thinking + // off (a model without a thinking switch): the answer still has to follow it + if (!use_reasoning_grammar()) { give_up("the grammar does not accept the prompt's prefill \"" + text + "\""); } } @@ -507,23 +512,11 @@ class xgrammar_constraint : public decode_constraint { history_.push_back(token); return; } - if (!with_reasoning_) { - give_up("the grammar rejected forced token " + std::to_string(token)); - return; - } - // the model is made to reason although the request turned thinking off: - // carry on with the grammar that has the reasoning block, from the same point - xgrammar::GrammarMatcher reasoning(*with_reasoning_, std::nullopt, false, MAX_ROLLBACK); - with_reasoning_.reset(); - bool ok = prefill_.empty() || reasoning.AcceptString(prefill_); - for (size_t i = 0; ok && i < history_.size(); i++) { - ok = reasoning.AcceptToken(history_[i]); - } - if (!ok || !reasoning.AcceptToken(token)) { + // the model is made to reason although the request turned thinking off + if (!use_reasoning_grammar() || !matcher_.AcceptToken(token)) { give_up("the grammar rejected forced token " + std::to_string(token)); return; } - matcher_ = std::move(reasoning); history_.push_back(token); } @@ -537,6 +530,24 @@ class xgrammar_constraint : public decode_constraint { private: static constexpr int MAX_ROLLBACK = 8; + /// \brief Carry on with the grammar that has the reasoning block, from the same point + /// \return false when there is none, or it does not accept what came so far either + bool use_reasoning_grammar() { + if (!with_reasoning_) { + return false; + } + xgrammar::GrammarMatcher reasoning(*with_reasoning_, std::nullopt, false, MAX_ROLLBACK); + with_reasoning_.reset(); + bool ok = prefill_.empty() || reasoning.AcceptString(prefill_); + for (size_t i = 0; ok && i < history_.size(); i++) { + ok = reasoning.AcceptToken(history_[i]); + } + if (ok) { + matcher_ = std::move(reasoning); + } + return ok; + } + void give_up(const std::string& why) { // a desynchronised grammar would only produce garbage; let the model finish free header_print("WARNING", "Constrained decoding disabled for this request: " << why); From f1d2a1170428565cee48f5283ba3816725d4c455 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:29:06 -0700 Subject: [PATCH 15/18] fix(qwen3.5-omni): report thinking as off generate() always feeds an empty think block, but the model inherited thinking_enabled() == true, so a constrained request still offered an optional reasoning block before the JSON or tool call and the answer could escape into reasoning_content. Co-Authored-By: Claude --- src/create_new_model.md | 2 +- src/include/AutoModel/modeling_qwen3_5_omni.hpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/create_new_model.md b/src/create_new_model.md index 4c8241568..2a45abcdc 100644 --- a/src/create_new_model.md +++ b/src/create_new_model.md @@ -94,7 +94,7 @@ void load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) override; chat_template_type_t get_chat_template_type() override; // chat_ml | harmony | gemma4 bool configure_parameter(std::string name, const std::any& value) override; -bool thinking_enabled() const override; // models with a thinking switch +bool thinking_enabled() const override; // models with a thinking switch, or always off ``` Also declare the private helper the base class expects you to supply: diff --git a/src/include/AutoModel/modeling_qwen3_5_omni.hpp b/src/include/AutoModel/modeling_qwen3_5_omni.hpp index 6e10d7907..abd07121b 100644 --- a/src/include/AutoModel/modeling_qwen3_5_omni.hpp +++ b/src/include/AutoModel/modeling_qwen3_5_omni.hpp @@ -61,6 +61,11 @@ class Qwen3_5_Omni : public AutoModel { std::string show_profile() override; std::pair> get_history() override; + /// \note generate() always feeds an empty think block, so thinking is never on + bool thinking_enabled() const override { + return false; + } + /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter /// \param value the value to set (can be any type) From 3b52b393114ce3885e1ee565fc63149459ba5da1 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:29:07 -0700 Subject: [PATCH 16/18] fix(server): reject forced tool calls on models not shown tools Models with enable_tool off (Qwen3 up to 1.7B, Qwen3.5 0.8B) leave the tools out of the prompt, yet tool_choice "required" or a named function built a grammar that forced a call, with arguments made up for a tool the model never saw. A new tools_enabled() hook lets make_constraint() answer those with a 400 and drop the tool grammar under "auto". The API tests skip tool cases on the new error. Co-Authored-By: Claude --- docs/docs/instructions/server/chat_output_internals.md | 2 +- docs/docs/instructions/server/structured_output.md | 4 +++- src/common/AutoModel/automodel.cpp | 8 ++++++++ src/create_new_model.md | 1 + src/include/AutoModel/automodel.hpp | 6 ++++++ src/include/AutoModel/modeling_minicpm_v.hpp | 3 +++ src/include/AutoModel/modeling_qwen3.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_5vl.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_6_moe.hpp | 3 +++ src/include/AutoModel/modeling_qwen3_8mtp.hpp | 3 +++ src/test/openai_api_test/README.md | 2 +- src/test/openai_api_test/flm_api.py | 2 ++ 12 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/docs/instructions/server/chat_output_internals.md b/docs/docs/instructions/server/chat_output_internals.md index b7d9f466c..98b67750e 100644 --- a/docs/docs/instructions/server/chat_output_internals.md +++ b/docs/docs/instructions/server/chat_output_internals.md @@ -127,7 +127,7 @@ The non-streaming path (`RestHandler::build_nstream_response`) calls `chat_parse `chat_constraint_request_from_openai()` validates and normalizes `tool_choice`, `parallel_tool_calls`, `response_format` and thinking (`think: false` or `reasoning_effort: "none"` → `allow_reasoning = false`). Errors become HTTP 400. -`AutoModel::make_constraint()` also clears `allow_reasoning` when the *model* has thinking off (`thinking_enabled()`), builds a `chat_vocab`, and calls `chat_make_constraint()`, which returns one of: +`AutoModel::make_constraint()` also clears `allow_reasoning` when the *model* has thinking off (`thinking_enabled()`). When the model leaves tools out of its prompt (`tools_enabled()` false, e.g. Qwen3 up to 1.7B), it rejects `required` and the function form with a 400 and drops the tools for `auto`. It then builds a `chat_vocab` and calls `chat_make_constraint()`, which returns one of: - an **xgrammar constraint** when `chat_structural_tag()` returns a grammar; - a **`token_ban_constraint`** for `tool_choice: "none"` when the tool opener is a single token; diff --git a/docs/docs/instructions/server/structured_output.md b/docs/docs/instructions/server/structured_output.md index 0ad094ce0..aabffe802 100644 --- a/docs/docs/instructions/server/structured_output.md +++ b/docs/docs/instructions/server/structured_output.md @@ -92,7 +92,8 @@ resp = client.chat.completions.create( Notes: - `"required"` and the function form need a model whose format has a tool call syntax (every format except `content_only`, see [the format table](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/#️-supported-model-formats)). On other models they're rejected with HTTP 400 rather than silently ignored. -- `"auto"` on a model without a tool call syntax is accepted, but nothing is constrained or parsed. +- They're also rejected on models too small to be shown tools (`qwen3:0.6b`, `qwen3:1.7b`, `qwen3.5:0.8b`). These leave `tools` out of the prompt, so a forced call would have arguments made up for a tool the model never saw. +- `"auto"` on a model without a tool call syntax, or one that isn't shown tools, is accepted, but nothing is constrained. - With `"required"`, small models sometimes keep writing after their call, for example a made-up answer to their own question. Look at `tool_calls`. `content` may hold this extra text. --- @@ -403,6 +404,7 @@ In the OpenAI Python SDK this raises `openai.BadRequestError`. | `json_schema` without `schema` | `response_format json_schema needs json_schema.schema` | | A schema xgrammar can't compile (e.g. a dangling `$ref`) | `cannot build the decoding grammar: …` with xgrammar's reason | | `"required"` / function form on a model without tool syntax | `this model's chat format (content_only) has no tool call syntax to constrain` | +| `"required"` / function form on a model too small to be shown tools | `this model does not support tool calling; tool_choice must be "auto" or "none"` | --- diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 23f3d45b4..e5cc893f6 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace { @@ -622,6 +623,13 @@ std::shared_ptr AutoModel::make_constraint(const chat_constra if (!this->thinking_enabled()) { effective.allow_reasoning = false; } + // a model never shown the tools would make up the arguments of a call it is forced into + if (!this->tools_enabled()) { + if (request.tool_choice == chat_tool_choice::REQUIRED || request.tool_choice == chat_tool_choice::FUNCTION) { + throw std::invalid_argument("this model does not support tool calling; tool_choice must be \"auto\" or \"none\""); + } + effective.tools = nullptr; + } return chat_make_constraint(this->chat_format, effective, vocab, engine); } diff --git a/src/create_new_model.md b/src/create_new_model.md index 2a45abcdc..b1ab2e002 100644 --- a/src/create_new_model.md +++ b/src/create_new_model.md @@ -95,6 +95,7 @@ void load_model(std::string model_path, json model_info, chat_template_type_t get_chat_template_type() override; // chat_ml | harmony | gemma4 bool configure_parameter(std::string name, const std::any& value) override; bool thinking_enabled() const override; // models with a thinking switch, or always off +bool tools_enabled() const override; // models that may leave tools out of the prompt ``` Also declare the private helper the base class expects you to supply: diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index b520f354d..a2921a192 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -504,9 +504,15 @@ class AutoModel { virtual bool thinking_enabled() const { return true; } + /// \brief Whether the chat template is shown the request's tools + /// \note Models too small for tool calling leave them out of the prompt + virtual bool tools_enabled() const { + return true; + } /// \brief Build the sampling constraint a request asks for /// \return null when the request needs no constraint + /// \throws std::invalid_argument when the model cannot do what the request asks std::shared_ptr make_constraint(const chat_constraint_request& request); /// \brief Insert the tokens /// \param tokens the tokens diff --git a/src/include/AutoModel/modeling_minicpm_v.hpp b/src/include/AutoModel/modeling_minicpm_v.hpp index c7d32bf41..9a5b2f03f 100644 --- a/src/include/AutoModel/modeling_minicpm_v.hpp +++ b/src/include/AutoModel/modeling_minicpm_v.hpp @@ -68,6 +68,9 @@ class MiniCPM_V : public AutoModel { bool thinking_enabled() const override { return this->enable_think; } + bool tools_enabled() const override { + return this->enable_tool; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3.hpp b/src/include/AutoModel/modeling_qwen3.hpp index 182d3e6b1..77eec81a7 100644 --- a/src/include/AutoModel/modeling_qwen3.hpp +++ b/src/include/AutoModel/modeling_qwen3.hpp @@ -39,6 +39,9 @@ class Qwen3 : public AutoModel { bool thinking_enabled() const override { return this->enable_think; } + bool tools_enabled() const override { + return this->enable_tool; + } /// \brief Override configure_parameter to handle Qwen3-specific parameters bool configure_parameter(std::string parameter_name, const std::any& value) override { diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index 294412e4e..c7d55af10 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -65,6 +65,9 @@ class Qwen3_5VL : public AutoModel { bool thinking_enabled() const override { return this->enable_think; } + bool tools_enabled() const override { + return this->enable_tool; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_6_moe.hpp b/src/include/AutoModel/modeling_qwen3_6_moe.hpp index 8225a4580..ee57fe5ce 100644 --- a/src/include/AutoModel/modeling_qwen3_6_moe.hpp +++ b/src/include/AutoModel/modeling_qwen3_6_moe.hpp @@ -65,6 +65,9 @@ class Qwen3_6_MOE : public AutoModel { bool thinking_enabled() const override { return this->enable_think; } + bool tools_enabled() const override { + return this->enable_tool; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_8mtp.hpp b/src/include/AutoModel/modeling_qwen3_8mtp.hpp index 9436f22ec..fab6dd505 100644 --- a/src/include/AutoModel/modeling_qwen3_8mtp.hpp +++ b/src/include/AutoModel/modeling_qwen3_8mtp.hpp @@ -193,6 +193,9 @@ class Qwen3_8MTP : public AutoModel { bool thinking_enabled() const override { return this->enable_think; } + bool tools_enabled() const override { + return this->enable_tool; + } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/test/openai_api_test/README.md b/src/test/openai_api_test/README.md index 842ff9928..76c4202a0 100644 --- a/src/test/openai_api_test/README.md +++ b/src/test/openai_api_test/README.md @@ -39,7 +39,7 @@ Each case prints `PASS`, `FAIL` or `SKIP`. The exit code is non-zero if anything Some cases skip by design: -- **Tool cases** skip on models whose chat format has no tool call syntax (the server answers `has no tool call syntax`). +- **Tool cases** skip on models whose chat format has no tool call syntax (the server answers `has no tool call syntax`), and on models too small to be shown tools (`does not support tool calling`). - **Thinking-on cases** skip on models that never reason. - **Multi-turn cases** skip on single-turn (flash) models. - **Stream/non-stream parity** compares greedy outputs. Greedy decoding on the NPU isn't always bit-reproducible between identical requests (mostly over long outputs), so the check retries and only fails if the non-streaming output reproduced itself while the stream never matched it. Otherwise it skips. diff --git a/src/test/openai_api_test/flm_api.py b/src/test/openai_api_test/flm_api.py index 1cff0a2e0..19795e297 100644 --- a/src/test/openai_api_test/flm_api.py +++ b/src/test/openai_api_test/flm_api.py @@ -177,6 +177,8 @@ def raise_for_capability(message: str) -> None: """Turn a 'this model cannot do that' rejection into a skip.""" if "has no tool call syntax" in message: raise Skip("the model's chat format has no tool call syntax") + if "does not support tool calling" in message: + raise Skip("the model is not shown tools") if "only supports single-turn" in message: raise Skip("the model only supports single-turn requests") From 84dffcf95444721031260181f408eb9f96e08dc2 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:29:09 -0700 Subject: [PATCH 17/18] fix(server): treat null request fields as unset Some OpenAI clients send null for every option they leave unset. parallel_tool_calls: null became a 500 and tool_choice: null a 400; null sampling options, think, reasoning_effort, max_tokens, stream and tools were already 500s. The constraint parser and the OpenAI chat handler, including configure_chat_engine_parameters(), now read null as missing. Co-Authored-By: Claude --- .../server/chat_output_internals.md | 2 +- .../instructions/server/structured_output.md | 2 + src/common/chat/chat_constraint.cpp | 39 +++++++++++---- src/server/rest_handler.cpp | 50 ++++++++++++------- src/test/openai_api_test/test_errors.py | 9 ++++ 5 files changed, 73 insertions(+), 29 deletions(-) diff --git a/docs/docs/instructions/server/chat_output_internals.md b/docs/docs/instructions/server/chat_output_internals.md index 98b67750e..bb3a8d086 100644 --- a/docs/docs/instructions/server/chat_output_internals.md +++ b/docs/docs/instructions/server/chat_output_internals.md @@ -125,7 +125,7 @@ The non-streaming path (`RestHandler::build_nstream_response`) calls `chat_parse ### From request to constraint -`chat_constraint_request_from_openai()` validates and normalizes `tool_choice`, `parallel_tool_calls`, `response_format` and thinking (`think: false` or `reasoning_effort: "none"` → `allow_reasoning = false`). Errors become HTTP 400. +`chat_constraint_request_from_openai()` validates and normalizes `tool_choice`, `parallel_tool_calls`, `response_format` and thinking (`think: false` or `reasoning_effort: "none"` → `allow_reasoning = false`). A field set to `null` reads as missing. Errors become HTTP 400. `AutoModel::make_constraint()` also clears `allow_reasoning` when the *model* has thinking off (`thinking_enabled()`). When the model leaves tools out of its prompt (`tools_enabled()` false, e.g. Qwen3 up to 1.7B), it rejects `required` and the function form with a 400 and drops the tools for `auto`. It then builds a `chat_vocab` and calls `chat_make_constraint()`, which returns one of: diff --git a/docs/docs/instructions/server/structured_output.md b/docs/docs/instructions/server/structured_output.md index aabffe802..8b0699827 100644 --- a/docs/docs/instructions/server/structured_output.md +++ b/docs/docs/instructions/server/structured_output.md @@ -406,6 +406,8 @@ In the OpenAI Python SDK this raises `openai.BadRequestError`. | `"required"` / function form on a model without tool syntax | `this model's chat format (content_only) has no tool call syntax to constrain` | | `"required"` / function form on a model too small to be shown tools | `this model does not support tool calling; tool_choice must be "auto" or "none"` | +Options sent as `null`, as some clients do for the ones they leave unset, are treated as unset, not as errors. + --- ## ⚡ Performance diff --git a/src/common/chat/chat_constraint.cpp b/src/common/chat/chat_constraint.cpp index 514592c7a..68eb9b648 100644 --- a/src/common/chat/chat_constraint.cpp +++ b/src/common/chat/chat_constraint.cpp @@ -6,21 +6,38 @@ using json = nlohmann::ordered_json; +namespace { + +/// \brief A field of a request object, null when it is missing +/// \note Some clients send null for every option they leave unset, so null +/// has to read as missing; json::value() throws on it instead. +const json& field(const json& object, const char* key) { + static const json missing; + auto it = object.find(key); + return it == object.end() ? missing : *it; +} + +} // namespace + bool chat_constraint_request_from_openai(const json& request, const json& tools, chat_constraint_request& out, std::string& error) { out = chat_constraint_request(); out.tools = &tools; - out.parallel_tool_calls = request.value("parallel_tool_calls", true); + const json& parallel = field(request, "parallel_tool_calls"); + if (!parallel.is_null() && !parallel.is_boolean()) { + error = "parallel_tool_calls must be a boolean"; + return false; + } + out.parallel_tool_calls = parallel.is_null() || parallel.get(); // thinking turned off: some models still get an empty think block fed in, but // the grammar must not offer room to reason - if (request.contains("think") && request["think"].is_boolean() && !request["think"].get()) { - out.allow_reasoning = false; - } - if (request.value("reasoning_effort", "") == "none") { + const json& think = field(request, "think"); + if ((think.is_boolean() && !think.get()) || field(request, "reasoning_effort") == "none") { out.allow_reasoning = false; } - const json choice = request.value("tool_choice", json("auto")); + const json& choice_field = field(request, "tool_choice"); + const json choice = choice_field.is_null() ? json("auto") : choice_field; if (choice.is_string()) { const std::string mode = choice.get(); if (mode == "auto") { @@ -33,8 +50,8 @@ bool chat_constraint_request_from_openai(const json& request, const json& tools, error = "tool_choice must be \"auto\", \"none\", \"required\" or a function, got \"" + mode + "\""; return false; } - } else if (choice.is_object() && choice.value("type", "") == "function" && choice.contains("function") && - choice["function"].is_object() && choice["function"].contains("name")) { + } else if (choice.is_object() && field(choice, "type") == "function" && + field(field(choice, "function"), "name").is_string()) { out.tool_choice = chat_tool_choice::FUNCTION; out.function_name = choice["function"]["name"].get(); } else { @@ -51,7 +68,8 @@ bool chat_constraint_request_from_openai(const json& request, const json& tools, if (out.tool_choice == chat_tool_choice::FUNCTION) { bool found = false; for (const auto& tool : tools) { - if (tool.contains("function") && tool["function"].value("name", "") == out.function_name) { + const json& name = field(field(tool, "function"), "name"); + if (name.is_string() && name.get() == out.function_name) { found = true; } } @@ -63,7 +81,8 @@ bool chat_constraint_request_from_openai(const json& request, const json& tools, if (request.contains("response_format") && request["response_format"].is_object()) { const json& format = request["response_format"]; - const std::string type = format.value("type", "text"); + const json& type_field = field(format, "type"); + const std::string type = type_field.is_null() ? "text" : type_field.is_string() ? type_field.get() : ""; if (type == "json_object") { out.json_object = true; } else if (type == "json_schema") { diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 192dc7c67..477fa5b80 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -33,6 +33,20 @@ json ExceptionResponse(const std::exception& error) { return ModelErrorResponse(*model_error); return {{"error", error.what()}}; } + +/// \brief Whether a request sets a field +/// \note Some clients send null for every option they leave unset, so null +/// counts as not set; json::value() and conversions throw on it instead. +bool has_value(const json& request, const char* key) { + auto it = request.find(key); + return it != request.end() && !it->is_null(); +} + +/// \brief A request field, or the fallback when it is missing or null +template +T value_or(const json& request, const char* key, T fallback) { + return has_value(request, key) ? request.at(key).get() : fallback; +} } ///@brief Normalize messages by merging consecutive user messages (like Ollama does) @@ -554,44 +568,44 @@ void RestHandler::ensure_embed_model_loaded(const std::string& model_tag) { ///@param options the options JSON object ///@param request the request JSON object void RestHandler::configure_chat_engine_parameters(const json& options, const json& request) { - if (request.contains("temperature")) { + if (has_value(request, "temperature")) { float temperature = request["temperature"]; auto_chat_engine->set_temperature(temperature); } - if (request.contains("top_p")) { + if (has_value(request, "top_p")) { float top_p = request["top_p"]; auto_chat_engine->set_topp(top_p); } - if (request.contains("top_k")) { + if (has_value(request, "top_k")) { int top_k = request["top_k"]; auto_chat_engine->set_topk(top_k); } - if (request.contains("min_p")) { + if (has_value(request, "min_p")) { int min_p = request["min_p"]; auto_chat_engine->set_minp(min_p); } - if (request.contains("presence_penalty")) { + if (has_value(request, "presence_penalty")) { float presence_penalty = request["presence_penalty"]; auto_chat_engine->set_presence_penalty(presence_penalty); } - if (request.contains("frequency_penalty")) { + if (has_value(request, "frequency_penalty")) { float frequency_penalty = request["frequency_penalty"]; auto_chat_engine->set_frequency_penalty(frequency_penalty); } - if (request.contains("repetition_penalty")) { + if (has_value(request, "repetition_penalty")) { float repetition_penalty = request["repetition_penalty"]; auto_chat_engine->set_repetition_penalty(repetition_penalty); } - if (request.contains("think")) { + if (has_value(request, "think")) { bool enable_thinking = request["think"]; auto_chat_engine->configure_parameter("enable_think", enable_thinking); } - if (request.contains("reasoning_effort")) { + if (has_value(request, "reasoning_effort")) { std::string reasoning_effort = request["reasoning_effort"]; auto_chat_engine->configure_parameter("reasoning_effort", reasoning_effort); } - if (request.contains("image-max-tokens")) { + if (has_value(request, "image-max-tokens")) { int image_max_tokens = request["image-max-tokens"]; auto_chat_engine->configure_parameter("image_max_tokens", image_max_tokens); } @@ -1158,12 +1172,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, try { // Extract OpenAI-style parameters json current_messages = request["messages"]; - std::string model = request.value("model", current_model_tag); - const std::string request_backend = request.value("backend", std::string()); - bool stream = request.value("stream", false); - int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); - json tools = request.value("tools", json::array()); - json options = request.value("options", json::object()); + std::string model = value_or(request, "model", current_model_tag); + const std::string request_backend = value_or(request, "backend", std::string()); + bool stream = value_or(request, "stream", false); + int length_limit = value_or(request, "max_tokens", value_or(request, "max_completion_tokens", 4096)); + json tools = value_or(request, "tools", json::array()); + json options = value_or(request, "options", json::object()); // tool_choice, parallel_tool_calls and response_format become a decoding constraint chat_constraint_request constraint_request; @@ -1243,9 +1257,9 @@ void RestHandler::handle_openai_chat_completion(const json& request, lm_uniform_input_t uniformed_input; uniformed_input.messages = current_messages; uniformed_input.tools = tools; - const std::optional openai_chat_budget = request.contains("max_tokens") + const std::optional openai_chat_budget = has_value(request, "max_tokens") ? std::optional(request.at("max_tokens").get()) - : request.contains("max_completion_tokens") + : has_value(request, "max_completion_tokens") ? std::optional(request.at("max_completion_tokens").get()) : std::nullopt; uniformed_input.requested_max_new_tokens = diff --git a/src/test/openai_api_test/test_errors.py b/src/test/openai_api_test/test_errors.py index ecedaba45..d46cd6673 100644 --- a/src/test/openai_api_test/test_errors.py +++ b/src/test/openai_api_test/test_errors.py @@ -87,6 +87,15 @@ def uncompilable_schema(api: Api) -> None: expect_400(api, {"response_format": json_schema_format("bad", schema)}, "cannot build the decoding grammar") +@case +def null_options_are_unset(api: Api) -> None: + """Options sent as null, as some clients do for unset ones, are not an error.""" + body = api.body(user("Say hi."), max_tokens=16, tools=WEATHER_TOOLS, tool_choice=None, + parallel_tool_calls=None, response_format=None, reasoning_effort=None, + presence_penalty=None, stream=None, max_completion_tokens=None) + flm_api.check_message_contract(api.ok(body)) + + @case def server_still_serves(api: Api) -> None: """After the rejections a normal request still succeeds.""" From 507208609c57e2ae4806ca7a04bf4f7a1976c9c1 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Mon, 19 Oct 2026 10:38:24 -0700 Subject: [PATCH 18/18] docs(license): add xgrammar, picojson and dlpack notices xgrammar is now statically linked into flm, with picojson and dlpack compiled in through it. Their notices go into THIRD_PARTY_NOTICES.txt, which the installers and the .deb already ship: xgrammar's Apache-2.0 attribution and NOTICE text (Apache-2.0 4(a), 4(d)), picojson's BSD-2 copyright and disclaimer, and dlpack's Apache-2.0 attribution. The README credits xgrammar next to tokenizers-cpp and minja. Co-Authored-By: Claude --- README.md | 1 + THIRD_PARTY_NOTICES.txt | 69 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/README.md b/README.md index 2ddff9494..e91e9bd5c 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ No model rewrites, no tuning — it just works. - Inspired by the widely adopted [llama.cpp](https://github.com/ggml-org/llama.cpp) and [Ollama](https://github.com/ollama/ollama) - Tokenization accelerated with [MLC-ai/tokenizers-cpp](https://github.com/mlc-ai/tokenizers-cpp) - Chat formatting via [Google/minja](https://github.com/google/minja) +- Constrained decoding (tool calls, structured output) powered by [MLC-ai/xgrammar](https://github.com/mlc-ai/xgrammar) - Low-level kernels optimized using the powerful [IRON](https://github.com/amd/iron)+[AIE-MLIR](https://github.com/Xilinx/mlir-aie) --- diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 4d594a501..da01da576 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -36,6 +36,9 @@ Components 21. biovault_bfloat16 22. Unicode data (unicode-ident Rust crate) 23. Microsoft Visual C++ Runtime + 24. XGrammar + 25. PicoJSON + 26. DLPack ============================================================================== 1. FFmpeg @@ -760,6 +763,72 @@ Homepage: https://visualstudio.microsoft.com/license-terms/ Used in: Windows: msvcp140.dll, vcruntime140.dll, vcruntime140_1.dll, redistributed under the Visual Studio redistributable terms. +============================================================================== +24. XGrammar +============================================================================== + +License: Apache License 2.0 +Homepage: https://github.com/mlc-ai/xgrammar +Used in: Statically linked into flm (v0.2.8). + +Copyright (c) 2024 by XGrammar Contributors + +Licensed under the Apache License, Version 2.0. The license text is +reproduced at the end of this file. + +XGrammar's NOTICE file reads: + + XGrammar + + Copyright (c) 2024 by XGrammar Contributors + +============================================================================== +25. PicoJSON +============================================================================== + +License: BSD 2-Clause License +Homepage: https://github.com/kazuho/picojson +Used in: Compiled into flm (via XGrammar). + +Copyright 2009-2010 Cybozu Labs, Inc. +Copyright 2011-2014 Kazuho Oku +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +============================================================================== +26. DLPack +============================================================================== + +License: Apache License 2.0 +Homepage: https://github.com/dmlc/dlpack +Used in: Compiled into flm (via XGrammar, header only). + +Copyright 2017 by Contributors + +Licensed under the Apache License, Version 2.0. The license text is +reproduced at the end of this file. + ============================================================================== Full license texts ==============================================================================