[gemma_cpp] Add support for Qwen3 models (0.6B, 1.7B and 4B).

deepseek/deepseek_tensors.cc

@@ -58,16 +58,9 @@ void TensorInfoRegistry::AddDeepSeekModelTensors(const ModelConfig& config) {
                    });
   };
 
-  if (config.HasMLA()) {
-    // DeepSeek models have an untied output head.
-    Add(no_suffix, {
-                       .base_name = "lm_head",
-                       .source_names = {"lm_head.weight", "lm_head"},
-                       .axes = {0, 1},
-                       .shape = {config.vocab_size, config.model_dim},
-                       .min_size = Type::kBF16,
-                   });
-  }
+  // DeepSeek models have an untied output head.
+  // Note: This is now moved to gemma/tensor_info.cc.
+
   if (config.hc_mult > 1) {
     add_hc_collapse("hc_head", "");
   }

gemma/configs.cc

@@ -716,6 +716,85 @@ static ModelConfig ConfigT5Gemma_S_S() {
   return config;
 }
 
+static ModelConfig ConfigBaseQwen3() {
+  ModelConfig config = ConfigNoSSM();
+  config.vocab_size = 151936;
+  config.max_seq_len = 32768;
+  config.eos_id = 151645;
+  config.secondary_eos_id = 151643;
+  return config;
+}
+
+static LayerConfig LayerConfigQwen3(size_t model_dim, size_t ff_hidden_dim,
+                                    size_t heads, size_t kv_heads,
+                                    size_t qkv_dim) {
+  LayerConfig config;
+  config.model_dim = model_dim;
+  config.ff_hidden_dim = ff_hidden_dim;
+  config.heads = heads;
+  config.kv_heads = kv_heads;
+  config.qkv_dim = qkv_dim;
+  config.optimized_gating = false;
+  config.post_norm = PostNormType::None;
+  config.activation = ActivationType::Silu;
+  config.use_qk_norm = true;
+  return config;
+}
+
+static ModelConfig ConfigQwen3_600M() {
+  ModelConfig config = ConfigBaseQwen3();
+  config.display_name = "Qwen3_0.6B";
+  config.model = Model::QWEN3_600M;
+  config.wrapping = PromptWrapping::GEMMA_IT;
+  config.model_dim = 1024;
+
+  LayerConfig layer_config =
+      LayerConfigQwen3(config.model_dim, 3072, 16, 8, 128);
+  config.num_layers = 28;
+  config.layer_configs = {config.num_layers, layer_config};
+  config.query_scale = QueryScaleType::SqrtKeySize;
+  config.use_global_timescale = true;
+  config.attention_window_sizes =
+      FixedAttentionWindowSizes<28>(config.max_seq_len);
+  return config;
+}
+
+static ModelConfig ConfigQwen3_2B() {
+  ModelConfig config = ConfigBaseQwen3();
+  config.display_name = "Qwen3_1.7B";
+  config.model = Model::QWEN3_2B;
+  config.wrapping = PromptWrapping::GEMMA_IT;
+  config.model_dim = 2048;
+
+  LayerConfig layer_config =
+      LayerConfigQwen3(config.model_dim, 6144, 16, 8, 128);
+  config.num_layers = 28;
+  config.layer_configs = {config.num_layers, layer_config};
+  config.query_scale = QueryScaleType::SqrtKeySize;
+  config.use_global_timescale = true;
+  config.attention_window_sizes =
+      FixedAttentionWindowSizes<28>(config.max_seq_len);
+  return config;
+}
+
+static ModelConfig ConfigQwen3_4B() {
+  ModelConfig config = ConfigBaseQwen3();
+  config.display_name = "Qwen3_4B";
+  config.model = Model::QWEN3_4B;
+  config.wrapping = PromptWrapping::GEMMA_IT;
+  config.model_dim = 2560;
+
+  LayerConfig layer_config =
+      LayerConfigQwen3(config.model_dim, 9728, 32, 8, 128);
+  config.num_layers = 36;
+  config.layer_configs = {config.num_layers, layer_config};
+  config.query_scale = QueryScaleType::SqrtKeySize;
+  config.use_global_timescale = true;
+  config.attention_window_sizes =
+      FixedAttentionWindowSizes<36>(config.max_seq_len);
+  return config;
+}
+
 static ModelConfig ConfigFromModel(Model model) {
   switch (model) {
     case Model::GEMMA2_2B:
@@ -756,6 +835,12 @@ static ModelConfig ConfigFromModel(Model model) {
       return ConfigDeepSeek4_Flash();
     case Model::T5GEMMA_S_S:
       return ConfigT5Gemma_S_S();
+    case Model::QWEN3_600M:
+      return ConfigQwen3_600M();
+    case Model::QWEN3_2B:
+      return ConfigQwen3_2B();
+    case Model::QWEN3_4B:
+      return ConfigQwen3_4B();
     default:
       HWY_ABORT("Model type %d unknown.", static_cast<int>(model));
   }
@@ -803,6 +888,12 @@ const char* ModelPrefix(Model model) {
       return "deepseek4-flash";
     case Model::T5GEMMA_S_S:
       return "t5gemma-s-s";
+    case Model::QWEN3_600M:
+      return "qwen3-0_6b";
+    case Model::QWEN3_2B:
+      return "qwen3-2b";
+    case Model::QWEN3_4B:
+      return "qwen3-4b";
     default:
       HWY_ABORT("Model type %d unknown.", static_cast<int>(model));
   }
@@ -1004,6 +1095,13 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) {
     case 27:
       return (layer_types & kDeduced448) ? Model::PALIGEMMA2_3B_448
                                          : Model::PALIGEMMA2_3B_224;
+    case 28:
+      if (blob_path.path.find("qwen3-2b") != std::string::npos ||
+          blob_path.path.find("qwen3-1_7b") != std::string::npos) {
+        return Model::QWEN3_2B;
+      }
+      return Model::QWEN3_600M;
+
     case 30:
       return Model::GEMMA4_26B_MOE;
 
@@ -1012,6 +1110,8 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) {
                                          : Model::GEMMA3_4B_LM;
     case 35:
       return Model::GEMMA4_2B;
+    case 36:
+      return Model::QWEN3_4B;
     case 42:
       if (layer_types & kDeducedViT) {
         return (layer_types & kDeduced448) ? Model::PALIGEMMA2_10B_448

gemma/configs.h

@@ -274,6 +274,9 @@ enum class Model {
   DEEPSEEK4_FLASH,
   // T5Gemma family - starting with S/S.
   T5GEMMA_S_S,
+  QWEN3_600M,
+  QWEN3_2B,  // 1.7B rounded up for readability.
+  QWEN3_4B,
   kSentinel,
 };
 
@@ -290,6 +293,11 @@ static inline bool IsPaliGemma(Model model) {
   return false;
 }
 
+static inline bool IsQwen3(Model model) {
+  return model == Model::QWEN3_600M || model == Model::QWEN3_2B ||
+         model == Model::QWEN3_4B;
+}
+
 static inline bool IsObsolete(Model model) {
   const size_t i = static_cast<size_t>(model);
   if (i == 5 || i == 6 || i == 8 || i == 9) return true;
@@ -642,6 +650,15 @@ struct ModelConfig : public IFields {
     return false;
   }
 
+  bool IsQwen3() const {
+    return gcpp::IsQwen3(model);
+  }
+
+  bool HasLmHead() const {
+    // QWEN3_4B has tied embeddings and thus no separate LM head.
+    return model == Model::QWEN3_600M || model == Model::QWEN3_2B || HasMLA();
+  }
+
   // Synthesized config for the multi-token-prediction block (DeepSeek V4):
   // a full extra layer (dense MLA + MoE) used for speculative decoding, not
   // part of the main stack. Only valid when `num_mtp_layers > 0`.

gemma/gemma.cc

@@ -223,15 +223,18 @@ static float EmbeddingScaling(size_t model_dim) {
       hwy::ConvertScalarTo<BF16>(sqrtf(static_cast<float>(model_dim))));
 }
 
+static bool HasEmbeddingScaling(const ModelConfig& model_config) {
+  return !(model_config.IsQwen3() || model_config.HasMLA());
+}
+
 static HWY_INLINE void EmbedTokenFromWeights(int token, size_t x_row,
                                              const ModelConfig& model_config,
                                              const MatPtr& embedding,
                                              MatStorageT<float>& x) {
   const size_t model_dim = model_config.model_dim;
-  // DeepSeek does not scale embeddings by sqrt(model_dim).
+  // Qwen3/DeepSeekV4 does not scale embeddings by sqrt(model_dim).
   const float emb_scaling =
-      model_config.HasMLA() ? 1.0f : EmbeddingScaling(model_dim);
-
+      HasEmbeddingScaling(model_config) ? EmbeddingScaling(model_dim) : 1.0f;
   HWY_DASSERT(token >= 0);
   HWY_DASSERT(token < static_cast<int>(model_config.vocab_size));
 

gemma/tensor_info.cc

@@ -59,6 +59,16 @@ void TensorInfoRegistry::AddModelTensors(const ModelConfig& config) {
                      .min_size = Type::kBF16,
                  });
   AddDeepSeekModelTensors(config);
+  if (config.HasLmHead()) {
+    Add(no_suffix,
+        {
+            .base_name = "lm_head",
+            .source_names = {"lm_head/weight", "lm_head.weight", "lm_head"},
+            .axes = {0, 1},
+            .shape = {config.vocab_size, config.model_dim},
+            .min_size = Type::kBF16,
+        });
+  }
   Add(no_suffix, {
                      .base_name = "enc_norm_bias",
                      .source_names = {"img/Transformer/encoder_norm/bias"},

gemma/tokenizer.cc

@@ -22,6 +22,7 @@
 #include <utility>
 #include <vector>
 
+#include "hwy/base.h"
 #include "gemma/configs.h"  // PromptWrapping
 #include "tokenizer/sentencepiece_tokenizer.h"
 
@@ -75,6 +76,11 @@ GemmaChatTemplate::GemmaChatTemplate(const GemmaTokenizer& tokenizer,
     sot_user_ = {105, 2364, 107};
     sot_model_ = {105, 4368, 107};
     eot_ = {106, 107};
+  } else if (IsQwen3(model)) {
+    prepend_bos_ = false;
+    HWY_ASSERT(tokenizer.Encode("<|im_start|>user\n", &sot_user_));
+    HWY_ASSERT(tokenizer.Encode("<|im_start|>assistant\n", &sot_model_));
+    HWY_ASSERT(tokenizer.Encode("<|im_end|>\n", &eot_));
   } else {
     sot_user_.reserve(3);
     if (!tokenizer.Encode("<start_of_turn>user\n", &sot_user_)) return;
@@ -101,7 +107,9 @@ std::vector<int> GemmaChatTemplate::Apply(size_t pos,
 
   // Start with BOS, or prepend end_of_turn if this is a continuation.
   if (pos == 0) {
-    out.push_back(BOS_ID);
+    if (prepend_bos_) {
+      out.push_back(BOS_ID);
+    }
   } else {
     out.insert(out.cend(), eot_.cbegin(), eot_.cend());
   }

gemma/tokenizer.h

@@ -78,6 +78,7 @@ class GemmaChatTemplate {
   std::vector<int> pali_sep_;
   std::vector<int> vlm_soi_;
   std::vector<int> vlm_eoi_;
+  bool prepend_bos_ = true;
 };
 
 std::vector<int> WrapAndTokenize(const GemmaTokenizer& tokenizer,

gemma/weights.h

@@ -16,7 +16,6 @@
 #ifndef THIRD_PARTY_GEMMA_CPP_GEMMA_WEIGHTS_H_
 #define THIRD_PARTY_GEMMA_CPP_GEMMA_WEIGHTS_H_
 
-#include <math.h>  // isnan
 #include <stddef.h>
 #include <stdint.h>
 
@@ -252,7 +251,7 @@ struct LayerWeightsPtrs {
 
   MatPtr key_norm_scale;    // at least BF16.
   MatPtr query_norm_scale;  // at least BF16.
-  
+
   MatPtr router_scale;
   MatPtr p_expert_sc;
   MatPtr post_ffw1_ns;
@@ -788,7 +787,7 @@ struct WeightsPtrs {
 
     func(TENSOR_ARGS(embedder_input_embedding, kMustRead));
     func(TENSOR_ARGS(final_norm_scale, kMustRead));
-    if (config_.HasMLA()) {
+    if (config_.HasLmHead()) {
       func(TENSOR_ARGS(lm_head, kMustRead));
     }
     if (config_.hc_mult > 1) {

python/configs.cc

@@ -108,6 +108,9 @@ PYBIND11_MODULE(configs, py_module) {
       .value("GEMMA4_2B", gcpp::Model::GEMMA4_2B)
       .value("DEEPSEEK4_FLASH", gcpp::Model::DEEPSEEK4_FLASH)
       .value("T5GEMMA_S_S", gcpp::Model::T5GEMMA_S_S)
+      .value("QWEN3_600M", gcpp::Model::QWEN3_600M)
+      .value("QWEN3_2B", gcpp::Model::QWEN3_2B)
+      .value("QWEN3_4B", gcpp::Model::QWEN3_4B)
       // Insert new models above this line.
   .value("PALIGEMMA_448", gcpp::Model::PALIGEMMA_448);