feat(preprocessor): unified chat templates with vendored jinja.cpp and per-model backend selection

.gitmodules

@@ -40,3 +40,7 @@
 [submodule "third_party/flatbuffers"]
 	path = third_party/flatbuffers
 	url = https://github.com/google/flatbuffers
+[submodule "third_party/jinja.cpp"]
+	path = third_party/jinja.cpp
+	url = https://github.com/Aharrypotter/jinja.cpp.git
+	branch = main

CMakeLists.txt

@@ -13,6 +13,7 @@ option(MLLM_ENABLE_TEST "Enable MLLM test" ON)
 option(MLLM_ENABLE_BENCHMARK "Enable MLLM benchmark" ON)
 option(MLLM_ENABLE_EXAMPLE "Enable MLLM example" ON)
 option(MLLM_ENABLE_TOOLS "Enable MLLM tools" ON)
+option(MLLM_ENABLE_JINJA_CHAT_TEMPLATE "Enable the experimental Jinja chat-template renderer" OFF)
 option(MLLM_ENABLE_PY_MLLM "Enable MLLM python binding" OFF)
 option(MLLM_ENABLE_JIT "Enable MLLM jit" OFF)
 option(MLLM_BUILD_ARM_BACKEND "Enable MLLM ARM backend" OFF)
@@ -268,6 +269,26 @@ set(MLLM_JSON_INCLUDE_DIR
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third_party/json>
     $<INSTALL_INTERFACE:include/third_party/>)
 
+if(MLLM_ENABLE_JINJA_CHAT_TEMPLATE)
+  # jinja.cpp is a submodule of the mllm fork, which carries the
+  # Transformers-compatibility commits. Keep it behind the mllm chat-template
+  # wrapper so downstream users do not inherit its public namespace or JSON
+  # bridge. mllm's own nlohmann/json is listed first and therefore wins over the
+  # copy the submodule vendors.
+  if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/jinja.cpp/jinja.hpp)
+    message(
+      FATAL_ERROR
+        "MLLM_ENABLE_JINJA_CHAT_TEMPLATE=ON requires the third_party/jinja.cpp submodule. "
+        "Run: git submodule update --init third_party/jinja.cpp")
+  endif()
+  add_library(mllm_jinja_cpp INTERFACE)
+  target_include_directories(mllm_jinja_cpp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/json)
+  target_include_directories(
+    mllm_jinja_cpp SYSTEM INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/jinja.cpp
+                                    ${CMAKE_CURRENT_SOURCE_DIR}/third_party/jinja.cpp/third_party)
+  target_compile_definitions(mllm_jinja_cpp INTERFACE UJSON_USE_ORDERED_JSON=1 JINJA_PRESERVE_JSON_OBJECT_ORDER=1)
+endif()
+
 add_subdirectory(third_party/fmt)
 add_subdirectory(third_party/xxHash)
 set(FLATBUFFERS_BUILD_TESTS OFF)

examples/minicpm4/main.cpp

@@ -43,7 +43,7 @@ MLLM_MAIN({
 
   {
     auto minicpm4_cfg = mllm::models::minicpm4::MiniCPM4Config(config_path.get());
-    auto minicpm4_tokenizer = mllm::models::minicpm4::MiniCPM4Tokenizer(tokenizer_path.get());
+    auto minicpm4_tokenizer = mllm::models::minicpm4::MiniCPM4Tokenizer(tokenizer_path.get(), minicpm4_cfg);
     auto minicpm4 = mllm::models::minicpm4::MiniCPM4ForCausalLM(minicpm4_cfg);
 
     auto param = mllm::load(model_path.get(), file_version);

examples/minicpm5/README.md

@@ -8,6 +8,18 @@ it does not materialize a persistent 16-head KV cache.
 The runner implements the official no-tool chat-template branch, including optional system text and
 `enable_thinking=true|false`. Tool schemas, tool calls, and multi-turn history are outside this first product surface.
 
+### Chat template backend
+
+`config.json` selects how the runner renders the prompt through
+`chat_template_backend`: `legacy` (default) keeps the byte-stable formatter
+above; `jinja_required` renders the checkpoint's own `chat_template.jinja`,
+discovered next to `tokenizer.json`, with the vendored `jinja.cpp` engine and
+requires a build with `-DMLLM_ENABLE_JINJA_CHAT_TEMPLATE=ON`. A
+`jinja_required` model fails at load time instead of falling back when Jinja
+support or the template is missing. See
+`mllm/preprocessor/chat_template/README.md` for the pipeline and the
+Transformers parity gate.
+
 ## Convert
 
 Keep embeddings and norms in FP32; the supplied configuration packs transformer Linear weights and the independent

examples/minicpm5/main.cpp

@@ -108,7 +108,10 @@ MLLM_MAIN({
 
     auto parameters = mllm::load(model_path.get(), file_version);
     mllm::models::minicpm5::validateModelConfigMatch(config, parameters);
-    auto tokenizer = mllm::models::minicpm5::MiniCPM5Tokenizer(tokenizer_path.get());
+    // The chat-template backend comes from config.json (chat_template_backend);
+    // a Jinja template is discovered next to tokenizer.json.
+    auto tokenizer = mllm::models::minicpm5::MiniCPM5Tokenizer(tokenizer_path.get(), config);
+    fmt::print("chat template backend: {}\n", mllm::preprocessor::chatTemplateBackendName(tokenizer.chatTemplateBackend()));
     auto model = mllm::models::minicpm5::MiniCPM5ForCausalLM(config);
     model.load(parameters);
 

examples/qwen3/main.cpp

@@ -35,7 +35,7 @@ MLLM_MAIN({
 
   {
     auto qwen3_cfg = mllm::models::qwen3::Qwen3Config(config_path.get());
-    auto qwen3_tokenizer = mllm::models::qwen3::Qwen3Tokenizer(tokenizer_path.get());
+    auto qwen3_tokenizer = mllm::models::qwen3::Qwen3Tokenizer(tokenizer_path.get(), qwen3_cfg);
     auto qwen3 = mllm::models::qwen3::Qwen3ForCausalLM(qwen3_cfg);
 
     auto param = mllm::load(model_path.get(), file_version);

examples/qwen3_5/README.md

@@ -71,6 +71,22 @@ registers `Qwen35PortableVideoDecoderSmoke`. This backend-specific smoke checks
 the committed MP4 fixture, exact sampled indices and RGB hash, and Qwen3.5 patch
 geometry. It is separate from the default model correctness tests.
 
+### Chat template backend
+
+`config.json` selects how the runner renders the prompt through
+`chat_template_backend`: `legacy` (default) keeps the byte-stable single-turn
+runner prompt; `jinja_required` renders the checkpoint's own official `chat_template.jinja`,
+discovered next to `tokenizer.json`, with the vendored `jinja.cpp` engine and
+requires a build with `-DMLLM_ENABLE_JINJA_CHAT_TEMPLATE=ON`. A
+`jinja_required` model fails at load time instead of falling back when Jinja
+support or the template is missing. See
+`mllm/preprocessor/chat_template/README.md` for the pipeline and the
+Transformers parity gate.
+Both backends emit one `<|vision_start|><|image_pad|><|vision_end|>` or
+`<|vision_start|><|video_pad|><|vision_end|>` placeholder per media input;
+the runner then expands video placeholders into timestamped frame markers and
+image/video placeholders into per-patch tokens exactly as before.
+
 ## Quantization
 
 The user-facing W4A8 configuration uses dynamic INT8 activations with INT4

examples/qwen3_5/main.cpp

@@ -209,9 +209,10 @@ MLLM_MAIN({
     auto param = mllm::load(model_path.get(), file_version);
     mllm::models::qwen3_5::validateModelConfigMatch(cfg, param);
 
-    auto tokenizer = mllm::models::qwen3_5::Qwen3_5Tokenizer(tokenizer_path.get(), cfg.image_min_pixels, cfg.image_max_pixels,
-                                                             cfg.vision_patch_size, cfg.vision_temporal_patch_size,
-                                                             cfg.vision_spatial_merge_size);
+    // The chat-template backend comes from config.json (chat_template_backend);
+    // a Jinja template is discovered next to tokenizer.json.
+    auto tokenizer = mllm::models::qwen3_5::Qwen3_5Tokenizer(tokenizer_path.get(), cfg);
+    fmt::print("chat template backend: {}\n", mllm::preprocessor::chatTemplateBackendName(tokenizer.chatTemplateBackend()));
     auto model = mllm::models::qwen3_5::Qwen3_5ForCausalLM(cfg);
     fmt::print("{}: {} layers ({} full attention + {} GDN)\n", mllm::models::qwen3_5::modelNameForConfig(cfg),
                cfg.num_hidden_layers, cfg.numFullAttentionLayers(), cfg.numGDNLayers());

examples/qwen3_moe/main.cpp

@@ -39,7 +39,7 @@ MLLM_MAIN({
 
   {
     auto qwen3_moe_cfg = mllm::models::qwen3_moe::Qwen3MoeConfig(config_path.get());
-    auto qwen3_moe_tokenizer = mllm::models::qwen3_moe::Qwen3Tokenizer(tokenizer_path.get());
+    auto qwen3_moe_tokenizer = mllm::models::qwen3_moe::Qwen3Tokenizer(tokenizer_path.get(), qwen3_moe_cfg);
     auto qwen3_moe = mllm::models::qwen3_moe::Qwen3MoeForCausalLM(qwen3_moe_cfg);
 
     auto param = mllm::load(model_path.get(), file_version);

examples/qwen_ascend/main.cpp

@@ -215,7 +215,7 @@ MLLM_MAIN({
       fmt::print("Prompt: {}\n", prompt_text);
       fmt::print("Max new tokens: {}\n", gen_max_new_tokens);
 
-      auto tokenizer = mllm::models::qwen_ascend::QwenAscendTokenizer(tokenizer_path.get());
+      auto tokenizer = mllm::models::qwen_ascend::QwenAscendTokenizer(tokenizer_path.get(), cfg);
       mllm::models::qwen_ascend::QwenAscendMessage msg;
       msg.prompt = prompt_text;
       auto inputs = tokenizer.convertMessage(msg);

examples/qwen_npu/main.cpp

@@ -39,11 +39,12 @@ int main(int argc, char** argv) {
   const std::string npu_model_path = "/data/local/tmp/zhanghao/models/qwen1.5-1.8b-chat-rot-qnn.mllm";
   const std::string cpu_decode_model_path = "/data/local/tmp/zhanghao/models/qwen1.5-1.8b-chat-rot_q4_0.mllm";
 
-  auto qwen_tokenizer = mllm::models::qwen_npu::QwenTokenizer("tokenizer.json", "qwen_merges.txt");
-
   mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV1;
 
   auto cpu_cfg = mllm::models::qwen_npu::QwenNPUConfig(config_path);
+  // The chat-template backend comes from the config (chat_template_backend); a
+  // Jinja template is discovered next to tokenizer.json.
+  auto qwen_tokenizer = mllm::models::qwen_npu::QwenTokenizer("tokenizer.json", "qwen_merges.txt", cpu_cfg);
 
   std::unique_ptr<mllm::nn::StaticCache> shared_kv_cache = nullptr;
   std::unique_ptr<mllm::models::qwen_npu::QwenForCausalLM> npu_model;

mllm/CMakeLists.txt

@@ -30,6 +30,12 @@ add_library(
 
 target_include_directories(MllmRT PUBLIC ${MLLM_INCLUDE_DIR} ${MLLM_JSON_INCLUDE_DIR})
 target_link_libraries(MllmRT PUBLIC fmt::fmt-header-only STDEXEC::stdexec xxHash::xxhash flatbuffers)
+if(MLLM_ENABLE_JINJA_CHAT_TEMPLATE)
+  target_link_libraries(MllmRT PRIVATE mllm_jinja_cpp)
+  target_compile_definitions(MllmRT PRIVATE MLLM_ENABLE_JINJA_CHAT_TEMPLATE=1)
+else()
+  target_compile_definitions(MllmRT PRIVATE MLLM_ENABLE_JINJA_CHAT_TEMPLATE=0)
+endif()
 
 # Compiler stack need mlir
 if(MLLM_ENABLE_MLLM_COMPILE_STACK)

mllm/models/minicpm4/configuration_minicpm4.hpp

@@ -4,6 +4,7 @@
 
 #include "mllm/core/aops/LinearOp.hpp"
 #include "mllm/engine/ConfigFile.hpp"
+#include "mllm/preprocessor/chat_template/ChatTemplate.hpp"
 #include <vector>
 
 namespace mllm::models::minicpm4 {
@@ -58,6 +59,8 @@ struct MiniCPM4Config : protected ConfigFile {
       }
     }
 
+    chat_template_backend = preprocessor::parseChatTemplateBackend(data().value("chat_template_backend", "legacy"));
+
     linear_impl_type = aops::str2LinearImplTypes(data()["linear_impl_type"]);
   }
 
@@ -92,6 +95,7 @@ struct MiniCPM4Config : protected ConfigFile {
   std::vector<float> long_factor;
 
   aops::LinearImplTypes linear_impl_type = aops::LinearImplTypes::kDefault;
+  preprocessor::ChatTemplateBackend chat_template_backend = preprocessor::ChatTemplateBackend::Legacy;
 };
 
 }  // namespace mllm::models::minicpm4

mllm/models/minicpm4/tokenization_minicpm4.hpp

@@ -1,24 +1,30 @@
 // tokenization_minicpm4.hpp
 #pragma once
 
+#include <filesystem>
+#include <utility>
 #include <vector>
 #include <unordered_map>
 #include "mllm/preprocessor/tokenizers/BPEUTF8.hpp"  //BPEUTF8, MiniCPM4 use LlamaTokenizer!
 #include "mllm/models/ARGeneration.hpp"
+#include "mllm/models/minicpm4/configuration_minicpm4.hpp"
+#include "mllm/preprocessor/chat_template/LegacyChatMl.hpp"
 #include "mllm/preprocessor/tokenizers/Unicode.hpp"
 #include "mllm/preprocessor/tokenizers/AutoTokenizer.hpp"
 
 namespace mllm::models::minicpm4 {
 
 struct MiniCPM4Message {
   std::string prompt;
-  static inline std::string message_template = "<|im_start|>user\n{{{prompt}}}<|im_end|>\n<|im_start|>assistant\n";
 };
 
 class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
  public:
-  explicit MiniCPM4Tokenizer(const std::string& file_path) {
+  // The chat-template backend is selected explicitly; see Qwen3Tokenizer.
+  explicit MiniCPM4Tokenizer(const std::string& file_path, preprocessor::ChatPreprocessorConfig chat_template = {})
+      : chat_preprocessor_(std::move(chat_template), preprocessor::renderLegacyChatMlSingleTurn) {
     bpe_.initFromSentencePieceJson(file_path);
+    chat_preprocessor_.setControlTokens(bpe_.controlTokens());
 
     special_tokens_trie_.add("<|im_start|>");
     special_tokens_trie_.add("<|im_end|>");
@@ -109,10 +115,23 @@ class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
     return decode_string(result);
   }
 
+  MiniCPM4Tokenizer(const std::string& file_path, const MiniCPM4Config& config, std::filesystem::path model_directory = {})
+      : MiniCPM4Tokenizer(file_path,
+                          preprocessor::ChatPreprocessorConfig{
+                              .backend = config.chat_template_backend,
+                              .template_options = {.model_directory = model_directory.empty()
+                                                                          ? std::filesystem::path(file_path).parent_path()
+                                                                          : std::move(model_directory)}}) {}
+
+  preprocessor::ChatTemplateBackend chatTemplateBackend() const noexcept { return chat_preprocessor_.backend(); }
+
+  // Renders the single-turn runner prompt through the configured backend.
+  std::string renderChatTemplate(const MiniCPM4Message& message) const {
+    return chat_preprocessor_.render(preprocessor::makeSingleTurnChatTemplateRequest(message.prompt, "", std::nullopt));
+  }
+
   ARGenerationOutputPast convertMessage(const MiniCPM4Message& message) {
-    auto applied_string = MiniCPM4Message::message_template;
-    size_t pos = applied_string.find("{{{prompt}}}");
-    applied_string.replace(pos, 12, message.prompt);
+    auto applied_string = renderChatTemplate(message);
 
     auto tokens = tokenize(applied_string);
 
@@ -130,6 +149,7 @@ class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
   }
 
  private:
+  preprocessor::ChatPreprocessor chat_preprocessor_;
   preprocessor::BPEUTF8 bpe_;
 
   std::string decode_token(const std::string& token) {

mllm/models/minicpm5/chat_template_minicpm5.hpp

@@ -0,0 +1,54 @@
+// Copyright (c) MLLM Team.
+// Licensed under the MIT License.
+#pragma once
+
+#include <string>
+
+#include "mllm/preprocessor/chat_template/ChatTemplate.hpp"
+
+namespace mllm::models::minicpm5 {
+
+// Builds the Transformers-style request for one MiniCPM5 runner turn.
+inline preprocessor::ChatTemplateRequest makeMiniCPM5ChatTemplateRequest(const std::string& prompt, const std::string& system,
+                                                                        bool enable_thinking) {
+  preprocessor::ChatTemplateRequest request;
+  if (!system.empty()) { request.messages.push_back({{"role", "system"}, {"content", system}}); }
+  request.messages.push_back({{"role", "user"}, {"content", prompt}});
+  request.add_generation_prompt = true;
+  request.extra_context["enable_thinking"] = enable_thinking;
+  return request;
+}
+
+// Migration renderer for the pre-Jinja MiniCPM5 runner prompt. It accepts the
+// optional-system plus single-user shape produced by
+// makeMiniCPM5ChatTemplateRequest and keeps those bytes stable; other shapes
+// fail closed instead of approximating the official template.
+inline std::string renderLegacyMiniCPM5ChatTemplate(const preprocessor::ChatTemplateRequest& request) {
+  using preprocessor::ChatTemplateError;
+  if (request.tools.has_value()) {
+    throw ChatTemplateError("legacy MiniCPM5 chat template does not support tools; select the jinja_required backend");
+  }
+  const auto& messages = request.messages;
+  std::size_t index = 0;
+  std::string output = "<s>";
+  if (index < messages.size() && messages[index].value("role", "") == "system") {
+    const auto& content = messages[index].at("content");
+    if (!content.is_string()) { throw ChatTemplateError("legacy MiniCPM5 chat template requires string system content"); }
+    output += "<|im_start|>system\n" + content.get<std::string>() + "<|im_end|>\n";
+    ++index;
+  }
+  if (index + 1 != messages.size() || messages[index].value("role", "") != "user") {
+    throw ChatTemplateError("legacy MiniCPM5 chat template supports an optional system message followed by exactly one "
+                            "user message; select the jinja_required backend for multi-turn requests");
+  }
+  const auto& content = messages[index].at("content");
+  if (!content.is_string()) { throw ChatTemplateError("legacy MiniCPM5 chat template requires string user content"); }
+  output += "<|im_start|>user\n" + content.get<std::string>() + "<|im_end|>\n";
+  if (request.add_generation_prompt) {
+    output += "<|im_start|>assistant\n";
+    output += request.extra_context.value("enable_thinking", false) ? "<think>\n" : "<think>\n\n</think>\n\n";
+  }
+  return output;
+}
+
+}  // namespace mllm::models::minicpm5

mllm/models/minicpm5/configuration_minicpm5.hpp

@@ -12,6 +12,7 @@
 #include "mllm/core/ParameterFile.hpp"
 #include "mllm/core/aops/LinearOp.hpp"
 #include "mllm/engine/ConfigFile.hpp"
+#include "mllm/preprocessor/chat_template/ChatTemplate.hpp"
 
 namespace mllm::models::minicpm5 {
 
@@ -51,6 +52,8 @@ struct MiniCPM5Config : protected ConfigFile {
       }
     }
 
+    chat_template_backend = preprocessor::parseChatTemplateBackend(config.value("chat_template_backend", "legacy"));
+
     if (config.contains("linear_impl_type")) {
       const auto linear_impl_name = config["linear_impl_type"].get<std::string>();
       linear_impl_type = aops::str2LinearImplTypes(linear_impl_name);
@@ -93,6 +96,7 @@ struct MiniCPM5Config : protected ConfigFile {
   std::vector<int64_t> eos_token_ids = {1, 130073};
   int32_t max_cache_length = 2048;
   aops::LinearImplTypes linear_impl_type = aops::LinearImplTypes::kDefault;
+  preprocessor::ChatTemplateBackend chat_template_backend = preprocessor::ChatTemplateBackend::Legacy;
 };
 
 inline auto matchesOfficialMiniCPM5_1BRuntimeContract(const MiniCPM5Config& config) -> bool {