From 0b65f0fb631e35f679f68f0155c09a4fb5e9614a Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 1 Sep 2026 15:53:43 +0800 Subject: [PATCH 1/3] fix(dequant): support resumable safetensors conversion --- gptqmodel/utils/model_dequant.py | 46 ++++++++++++++++++++++---------- scripts/dequantize_model.py | 13 ++++++++- tests/test_model_dequant_fp8.py | 22 +++++++++++++++ 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/gptqmodel/utils/model_dequant.py b/gptqmodel/utils/model_dequant.py index e92c3327a..da3100788 100644 --- a/gptqmodel/utils/model_dequant.py +++ b/gptqmodel/utils/model_dequant.py @@ -91,17 +91,16 @@ def list_safetensor_files(model_path: Path) -> Tuple[list, Optional[dict]]: def finalize_for_save(tensor: torch.Tensor, target_dtype: torch.dtype) -> torch.Tensor: - """Cast to ``target_dtype`` when floating point and move to CPU with optimal layout.""" + """Cast floating tensors, move to CPU, and use safetensors-compatible layout.""" if torch.is_floating_point(tensor): tensor = tensor.to(target_dtype) - tensor_cpu = tensor.to("cpu") - if tensor_cpu.ndim == 4: - tensor_cpu = tensor_cpu.contiguous(memory_format=torch.channels_last) - else: - tensor_cpu = tensor_cpu.contiguous() - return tensor_cpu + # safetensors requires the default contiguous layout. A 4D channels-last + # tensor is contiguous in PyTorch's channels-last sense, but save_file() + # rejects it because Tensor.is_contiguous() is false without a memory-format + # argument. + return tensor.to("cpu").contiguous() def _is_deepseek_v4_routed_expert_weight_key( @@ -1471,13 +1470,19 @@ def convert_compressed_pack_file( return tensors -def copy_aux_files(model_path: Path, output_path: Path, skip: Iterable[str]) -> None: +def copy_aux_files( + model_path: Path, + output_path: Path, + skip: Iterable[str], + *, + dirs_exist_ok: bool = False, +) -> None: for item in model_path.iterdir(): if item.name in skip: continue target = output_path / item.name if item.is_dir(): - shutil.copytree(item, target) + shutil.copytree(item, target, dirs_exist_ok=dirs_exist_ok) else: shutil.copy2(item, target) @@ -1488,14 +1493,14 @@ def dequantize_model( *, target_dtype: torch.dtype = torch.bfloat16, device: Optional[str] = None, + resume: bool = False, ) -> None: model_path = Path(model_path) output_path = Path(output_path) - if output_path.exists(): + if output_path.exists() and not resume: raise FileExistsError(f"Output path {output_path} already exists") - - output_path.mkdir(parents=True) + output_path.mkdir(parents=True, exist_ok=resume) config = load_json(model_path / "config.json") quant_cfg = config.get("quantization_config", {}) or {} @@ -1562,6 +1567,19 @@ def dequantize_model( try: for idx, filename in enumerate(files): path = model_path / filename + output_file = output_path / filename + if resume and output_file.exists(): + # Opening the shard validates its safetensors header before it + # is accepted as complete. Tensor views are memory-mapped, so + # collecting names and sizes does not load the shard into RAM. + with safe_open(output_file, framework="pt", device="cpu") as existing_reader: + for name in existing_reader.keys(): + tensor = existing_reader.get_tensor(name) + weight_map[str(name)] = filename + total_size += tensor.element_size() * tensor.numel() + LOG.debug("Reusing completed output shard '%s'", filename) + pb.subtitle(f"{filename} (existing)").next().draw() + continue LOG.debug("Processing shard '%s' for format %s on device %s", filename, fmt, open_device) if fmt == "fp8": with safe_open(path, framework="pt", device=open_device) as reader: @@ -1620,7 +1638,7 @@ def dequantize_model( raise ValueError(f"Unsupported format {fmt}") if tensors: - save_file(tensors, str(output_path / filename)) + save_file(tensors, str(output_file)) weight_map.update({str(name): filename for name in tensors}) total_size += sum(t.element_size() * t.numel() for t in tensors.values()) else: @@ -1649,4 +1667,4 @@ def dequantize_model( write_json(output_path / "config.json", new_config) skip_files = set(files) | {"config.json", "model.safetensors.index.json"} - copy_aux_files(model_path, output_path, skip_files) + copy_aux_files(model_path, output_path, skip_files, dirs_exist_ok=resume) diff --git a/scripts/dequantize_model.py b/scripts/dequantize_model.py index 377a7fd53..8b7f5b882 100755 --- a/scripts/dequantize_model.py +++ b/scripts/dequantize_model.py @@ -66,6 +66,11 @@ def _parse_args() -> argparse.Namespace: default="cpu", help="Device to stage tensors during dequantization (cpu, cuda, cuda:7, ...)", ) + parser.add_argument( + "--resume", + action="store_true", + help="Reuse valid output shards already present in the output directory", + ) parser.add_argument( "--env", action="append", @@ -106,7 +111,13 @@ def main() -> None: } print(f"[dequantize_model] parsed args: {debug_payload}") - dequantize_model(model_path, output_path, target_dtype=dtype, device=device) + dequantize_model( + model_path, + output_path, + target_dtype=dtype, + device=device, + resume=args.resume, + ) if __name__ == "__main__": diff --git a/tests/test_model_dequant_fp8.py b/tests/test_model_dequant_fp8.py index 5653f9815..32f0aa91a 100644 --- a/tests/test_model_dequant_fp8.py +++ b/tests/test_model_dequant_fp8.py @@ -45,6 +45,17 @@ def test_finalize_for_save_keeps_non_4d_tensors_contiguous(): assert out.is_contiguous() +def test_finalize_for_save_converts_channels_last_to_default_contiguous(): + tensor = torch.randn(2, 3, 4, 5).contiguous(memory_format=torch.channels_last) + assert not tensor.is_contiguous() + + out = finalize_for_save(tensor, torch.bfloat16) + + assert out.dtype is torch.bfloat16 + assert out.device.type == "cpu" + assert out.is_contiguous() + + def test_ignored_layers_are_honored_by_non_fp8_converters(tmp_path): ignored_weight = torch.randn(2, 2, dtype=torch.bfloat16) shard_path = tmp_path / "ignored.safetensors" @@ -173,8 +184,19 @@ def test_dequantize_model_fp8_allows_partial_edge_blocks(tmp_path): str(model_dir / shard_name), ) _write_index(model_dir, shard_name, ["linear.weight", "linear.weight_scale_inv"]) + aux_dir = model_dir / "aux" + aux_dir.mkdir() + (aux_dir / "metadata.json").write_text("{}", encoding="utf-8") dequantize_model(model_dir, output_dir, target_dtype=torch.bfloat16, device="cpu") + dequantize_model( + model_dir, + output_dir, + target_dtype=torch.bfloat16, + device="cpu", + resume=True, + ) + assert (output_dir / "aux" / "metadata.json").read_text(encoding="utf-8") == "{}" with safe_open(output_dir / shard_name, framework="pt", device="cpu") as reader: weight_out = reader.get_tensor("linear.weight") From 75f9a6a43bac6b9f5989dcd96c0fed05a38f81b1 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 1 Sep 2026 15:54:20 +0800 Subject: [PATCH 2/3] feat(models): add GLM-5 Next quantization support --- README.md | 3 +- gptqmodel/models/auto.py | 2 + gptqmodel/models/definitions/__init__.py | 1 + gptqmodel/models/definitions/glm5_next.py | 86 ++++++++ gptqmodel/models/moe_lifecycle.py | 22 +- requirements.txt | 2 +- tests/models/test_glm5_next.py | 29 +++ tests/test_glm5_next_support.py | 249 ++++++++++++++++++++++ 8 files changed, 386 insertions(+), 8 deletions(-) create mode 100644 gptqmodel/models/definitions/glm5_next.py create mode 100644 tests/models/test_glm5_next.py create mode 100644 tests/test_glm5_next_support.py diff --git a/README.md b/README.md index d8f034b62..94397a2e5 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ ## Latest News 🗞️🚀 +* 09/01/2026 7.4.0-dev `main`: ✨ Added `glm5_next` / GLM-5.3-Flash hybrid KDA/DSA MoE quantization support. * 08/31/2026 7.4.0-dev `main`: ✨ Added Qwen3.8-Flash-Next (`qwen4_exp`) quantization. * 08/26/2026 7.4.0-dev `main`: ✨ Added NVIDIA `LocateAnything-3B` quantization support. * 08/25/2026 7.4.0-dev `main`: ✨ Added Tencent `HunyuanOCR` quantization support. @@ -267,7 +268,7 @@ Selected public references where teams or companies explicitly mention GPT-QMode | DeepSeek-V2/V3/V3.2/V4/R1 | ✅ | GPT-OSS | ✅ | LongCat Flash | ✅ | OLMo2/3 / LLaDA2 | ✅ | Yi | ✅ | | DeepSeek-V2 Lite / VL / VL2 / OCR2 | ✅ | Granite / Granite MoE | ✅ | LongLLaMA | ✅ | Ovis 1.6/2/2.5/2.6 MoE/2.6 Next | ✅ | Seed-OSS | ✅ | | Dream | ✅ | GRIN-MoE | ✅ | Instella | ✅ | Phi 1-4 | ✅ | Voxtral | ✅ | -| ERNIE 4.5 / MoE / VL MoE | ✅ | GLM 4/4V/4.5V/4.6V/5/5.1/OCR/ASR | ✅ | GLM4 MoE / Lite / 4.5V MoE | ✅ | MiniCPM 3/O/V/V 4_6 | ✅ | PanGu-α | ✅ | +| ERNIE 4.5 / MoE / VL MoE | ✅ | GLM 4/4V/4.5V/4.6V/5/5.1/5.3/OCR/ASR | ✅ | GLM4 MoE / Lite / 4.5V MoE | ✅ | MiniCPM 3/O/V/V 4_6 | ✅ | PanGu-α | ✅ | | XVERSE | ✅ | Brumby | ✅ | Hymba | ✅ | Mistral | ✅ | Qwen 1/2/3/3.5 | ✅ | | MiniMax M2/M3 | ✅ | AfMoE | ✅ | Bailing-MoE | ✅ | LFM2 / LFM2-VL / LFM2-MoE | ✅ | Marin | ✅ | | InternVL Chat | ✅ | Laguna | ✅ | Mimo / Mimo V2 | ✅ | Zamba / Zamba2 | ✅ | Intern S1 / S2 Preview | ✅ | diff --git a/gptqmodel/models/auto.py b/gptqmodel/models/auto.py index c1ac7de67..1c2eb371a 100644 --- a/gptqmodel/models/auto.py +++ b/gptqmodel/models/auto.py @@ -105,6 +105,7 @@ from .definitions.glm4v import Glm4vGPTQ # noqa: E402 from .definitions.glm4v_moe import Glm4vMoeQModel, Glm4vMoeTextQModel # noqa: E402 from .definitions.glm_moe_dsa import GlmMoeDsaQModel # noqa: E402 +from .definitions.glm5_next import Glm5NextQModel # noqa: E402 from .definitions.glm_ocr import GlmOCRGPTQ # noqa: E402 from .definitions.glmasr import GlmASRGPTQ # noqa: E402 from .definitions.gpt2 import GPT2QModel # noqa: E402 @@ -241,6 +242,7 @@ "glm4_moe": GLM4MoEGPTQ, "glm4_moe_lite": Glm4MoeLiteQModel, "glm_moe_dsa": GlmMoeDsaQModel, + "glm5_next": Glm5NextQModel, "gpt_bigcode": GptBigCodeQModel, "codegen": CodeGenQModel, "cohere": LlamaQModel, # 100% llama clone diff --git a/gptqmodel/models/definitions/__init__.py b/gptqmodel/models/definitions/__init__.py index 1df76a3f8..1b1df4f82 100644 --- a/gptqmodel/models/definitions/__init__.py +++ b/gptqmodel/models/definitions/__init__.py @@ -41,6 +41,7 @@ from .glmasr import GlmASRGPTQ from .glm_ocr import GlmOCRGPTQ from .glm_moe_dsa import GlmMoeDsaQModel +from .glm5_next import Glm5NextQModel from .gpt2 import GPT2QModel from .gpt_bigcode import GptBigCodeQModel from .gpt_neo import GptNeoQModel diff --git a/gptqmodel/models/definitions/glm5_next.py b/gptqmodel/models/definitions/glm5_next.py new file mode 100644 index 000000000..0b5356f67 --- /dev/null +++ b/gptqmodel/models/definitions/glm5_next.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from transformers import AutoModelForImageTextToText + +from ...utils.model import move_to +from ..base import BaseQModel +from ..moe_lifecycle import GateUpDownMoELifecycleHooks + + +class Glm5NextQModel(BaseQModel): + """GLM-5.3-Flash hybrid KDA/DSA multimodal MoE model.""" + + loader = AutoModelForImageTextToText + require_load_processor = True + require_trust_remote_code = False + layer_modules_strict = False + + dynamic_expert_index = "n_routed_experts" + pre_lm_head_norm_module = "model.language_model.norm" + + # The checkpoint carries an auxiliary MTP decoder after the 45 inference + # layers. Transformers intentionally ignores it while loading; preserve it + # verbatim when a quantized checkpoint is saved. + out_of_model_tensors = {"prefixes": ["model.language_model.layers.45"]} + + moe_lifecycle_hooks = GateUpDownMoELifecycleHooks() + + # Quantize Q/K/V/O in KDA, the post-LoRA Q/KV and output projections in DSA, + # and both dense-MLP and routed-expert projections. + # KDA state/gate projections, DSA LoRA/indexer projections, routers, shared + # experts, norms, and hyper-connections remain in the native dtype. + module_tree = [ + "model", + "language_model", + "layers", + "#", + { + "input_layernorm": ("input_layernorm:!",), + "self_attn": ( + # KDA (linear-attention) layers. + "q_proj:0:q", + "k_proj:0:k", + "v_proj:0:v", + # DSA layers. The low-rank input and sparse indexer stay dense. + "q_a_proj:!", + "kv_a_proj_with_mqa:!", + "indexer.wq_b:!", + "indexer.wk:!", + "indexer.weights_proj:!", + "q_b_proj:0:q", + "kv_b_proj:0:k:v", + "o_proj:1", + ), + "post_attention_layernorm": ("post_attention_layernorm:!",), + "mlp:moe": { + # Dense fallback used by the first three decoder layers. + "": ("gate_proj:0:gate", "up_proj:0:up", "down_proj:1:down"), + "gate": ("gate:!", "e_score_correction_bias:!"), + "experts:routed:expert_activation=experts._apply_gate": { + "#": ("gate_proj:0:gate", "up_proj:0:up", "down_proj:1:down"), + }, + "shared_experts:shared": ( + "gate_proj:!", + "up_proj:!", + "down_proj:!", + ), + }, + }, + ] + + def update_layer_replay_kwargs_from_output(self, layer, layer_output, layer_input_kwargs, target_device): + """Pass full DSA selections to subsequent shared-indexer DSA layers.""" + + if not isinstance(layer_output, tuple) or len(layer_output) < 2: + return layer_input_kwargs + + topk_indices = layer_output[1] + if topk_indices is not None: + layer_input_kwargs["prev_topk_indices"] = move_to(topk_indices, device=target_device) + return layer_input_kwargs + + +__all__ = ["Glm5NextQModel"] diff --git a/gptqmodel/models/moe_lifecycle.py b/gptqmodel/models/moe_lifecycle.py index 2666281b4..e59f3899a 100644 --- a/gptqmodel/models/moe_lifecycle.py +++ b/gptqmodel/models/moe_lifecycle.py @@ -306,6 +306,16 @@ def __init__(self, gate_proj_name: str = None, up_proj_name: str = None, down_pr f"Got: gate={self.gate_proj_name}, up={self.up_proj_name}, down={self.down_proj_name}" ) + def apply_expert_activation(self, experts_module, expert, gate_out, up_out): + """Apply the model's fused expert gate when it exposes one.""" + + fused_gate = getattr(experts_module, "_apply_gate", None) + if callable(fused_gate): + return fused_gate(torch.cat([gate_out, up_out], dim=-1)) + if hasattr(expert, "act_fn"): + return expert.act_fn(gate_out) * up_out + return torch.nn.functional.silu(gate_out) * up_out + def _extract_moe_block_prefix(self, subset: Dict[str, Any], moe_block: nn.Module) -> Optional[str]: """ Extract moe_block_prefix from subset keys. @@ -365,8 +375,6 @@ def forward_to_all_experts( order they appear in the subset/module tree, then calls the original routed forward for the final output. """ - import torch.nn.functional as F - if not processor or not original_forward: error_msg = "Missing processor or original_forward" log.error(error_msg) @@ -482,10 +490,12 @@ def run_routed_experts(): gate_out = gate_module(expert_input) up_out = up_module(expert_input) - if hasattr(expert, 'act_fn'): - intermediate = expert.act_fn(gate_out) * up_out - else: - intermediate = F.silu(gate_out) * up_out + intermediate = self.apply_expert_activation( + experts_module=experts_module, + expert=expert, + gate_out=gate_out, + up_out=up_out, + ) del gate_out, up_out get_callable_module(down_key)(intermediate) diff --git a/requirements.txt b/requirements.txt index b3ab2dd98..d31b90015 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ datasets>=3.6.0 pyarrow>=21.0 dill>=0.3.8 torchao>=0.16.0 -defuser>=0.0.26 +defuser>=0.0.27 diff --git a/tests/models/test_glm5_next.py b/tests/models/test_glm5_next.py new file mode 100644 index 000000000..f1dff57d4 --- /dev/null +++ b/tests/models/test_glm5_next.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from model_test import ModelTest + + +class TestGlm5Next(ModelTest): + NATIVE_MODEL_ID = "/monster/data/model/GLM-5.3-Flash-REAP50-BF16" + TRUST_REMOTE_CODE = False + USE_FLASH_ATTN = False + EVAL_TASKS_SLOW = { + "arc_challenge": { + "chat_template": False, + "acc": {"value": 0.49146757679180886, "floor_pct": 0.04}, + "acc_norm": {"value": 0.5273037542662116, "floor_pct": 0.04}, + }, + } + EVAL_TASKS_FAST = ModelTest.derive_fast_eval_tasks(EVAL_TASKS_SLOW) + + MODEL_COMPAT_FAST_LAYER_POSITION = "first" + + # The REAP50 BF16 checkpoint is 331 GB. Even after 4-bit quantization, the + # model plus KDA workspaces should retain multi-GPU loading headroom. + EVAL_SINGLE_GPU = False + + def test_glm5_next(self): + self.quantize_and_evaluate() diff --git a/tests/test_glm5_next_support.py b/tests/test_glm5_next_support.py new file mode 100644 index 000000000..e5992034c --- /dev/null +++ b/tests/test_glm5_next_support.py @@ -0,0 +1,249 @@ +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +from gptqmodel.models import auto +from gptqmodel.models.definitions.glm5_next import Glm5NextQModel +from gptqmodel.models.loader import _convert_model_with_defuser +from gptqmodel.models.moe_lifecycle import GateUpDownMoELifecycleHooks +from gptqmodel.utils.model import find_modules +from gptqmodel.utils.structure import LazyTurtle +from defuser.model_registry import MODEL_CONFIG + + +def _tiny_text_config(): + glm5_next = pytest.importorskip("transformers.models.glm5_next") + return glm5_next.Glm5NextTextConfig( + vocab_size=64, + hidden_size=32, + intermediate_size=64, + moe_intermediate_size=16, + num_hidden_layers=5, + num_attention_heads=4, + num_key_value_heads=4, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + q_lora_rank=16, + kv_lora_rank=8, + qk_nope_head_dim=8, + qk_rope_head_dim=0, + v_head_dim=8, + index_topk=4, + index_kpool=2, + index_head_dim=8, + index_n_heads=2, + linear_head_dim=8, + linear_num_heads=4, + hc_mult=2, + max_position_embeddings=32, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + + +def test_glm5_next_model_type_selects_definition(monkeypatch): + fake_config = SimpleNamespace(model_type="glm5_next") + monkeypatch.setattr(auto, "resolve_trust_remote_code", lambda path, trust_remote_code=False: trust_remote_code) + monkeypatch.setattr(auto.AutoConfig, "from_pretrained", lambda *args, **kwargs: fake_config) + + assert auto.check_and_get_model_definition("glm-5.3-flash-fixture") is Glm5NextQModel + + +def test_glm5_next_module_tree_matches_reference_quantization_boundary(): + config = SimpleNamespace(text_config=SimpleNamespace(n_routed_experts=2)) + layer_modules = Glm5NextQModel.simple_layer_modules( + model_config=config, + quantize_config=SimpleNamespace(dynamic=None), + ) + flat = {name for block in layer_modules for name in block} + + assert { + "self_attn.q_proj", + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.q_b_proj", + "self_attn.kv_b_proj", + "self_attn.o_proj", + "mlp.experts.0.gate_proj", + "mlp.experts.1.up_proj", + "mlp.experts.1.down_proj", + } <= flat + assert { + "self_attn.q_a_proj", + "self_attn.kv_a_proj_with_mqa", + "self_attn.indexer.wq_b", + "self_attn.indexer.wk", + "self_attn.indexer.weights_proj", + "mlp.shared_experts.gate_proj", + }.isdisjoint(flat) + + full = { + name + for block in Glm5NextQModel.full_layer_modules(model_config=config) + for name in block + } + assert "self_attn.q_a_proj:!" in full + assert "mlp.shared_experts.gate_proj:!" in full + assert Glm5NextQModel.out_of_model_tensors == { + "prefixes": ["model.language_model.layers.45"] + } + + +def test_glm5_next_quantizes_every_loaded_decoder_layer(): + layer = SimpleNamespace(config=SimpleNamespace(num_hidden_layers=45)) + assert "should_quantize_layer" not in Glm5NextQModel.__dict__ + assert all( + Glm5NextQModel.should_quantize_layer( + layer=layer, + layer_name=f"model.language_model.layers.{index}", + layer_index=index, + quantize_config=SimpleNamespace(), + ) + for index in range(45) + ) + + +def test_glm5_next_defuser_registry_preserves_expert_forward(): + glm5_next = pytest.importorskip("transformers.models.glm5_next") + assert "glm5_next" in MODEL_CONFIG + text_model = glm5_next.Glm5NextTextModel(_tiny_text_config()).eval() + + class TinyOuter(nn.Module): + def __init__(self, language_model): + super().__init__() + self.config = SimpleNamespace(model_type="glm5_next") + self.model = nn.Module() + self.model.language_model = language_model + + model = TinyOuter(text_model) + experts = model.model.language_model.layers[3].mlp.experts + hidden_states = torch.randn(4, text_model.config.hidden_size) + topk_indices = torch.tensor([[0], [1], [0], [1]]) + topk_weights = torch.ones(4, 1) + + with torch.no_grad(): + expected = experts(hidden_states, topk_indices, topk_weights) + + assert _convert_model_with_defuser(Glm5NextQModel, model, cleanup_original=False) is True + + with torch.no_grad(): + actual = experts(hidden_states, topk_indices, topk_weights) + torch.testing.assert_close(actual, expected) + + modules = find_modules(model) + assert "model.language_model.layers.0.self_attn.q_proj" in modules + assert "model.language_model.layers.3.self_attn.q_b_proj" in modules + assert "model.language_model.layers.3.self_attn.kv_b_proj" in modules + assert "model.language_model.layers.3.mlp.experts.0.gate_proj" in modules + assert "model.language_model.layers.3.mlp.experts.0.up_proj" in modules + assert "model.language_model.layers.3.mlp.experts.0.down_proj" in modules + + layer_modules = Glm5NextQModel.simple_layer_modules( + model_config=SimpleNamespace(text_config=text_model.config), + quantize_config=SimpleNamespace(dynamic=None), + ) + suffixes = {name for block in layer_modules for name in block} + matched = {name for name in modules if any(name.endswith(suffix) for suffix in suffixes)} + assert "model.language_model.layers.0.self_attn.q_proj" in matched + assert "model.language_model.layers.3.self_attn.q_b_proj" in matched + assert "model.language_model.layers.3.mlp.experts.0.gate_proj" in matched + assert not any(name.endswith("self_attn.indexer.weights_proj") for name in matched) + + +def test_glm5_next_lazy_turtle_materializes_defused_experts(tmp_path): + glm5_next = pytest.importorskip("transformers.models.glm5_next") + config = glm5_next.Glm5NextConfig( + text_config=_tiny_text_config().to_dict(), + vision_config={ + "depth": 1, + "hidden_size": 16, + "intermediate_size": 32, + "projection_intermediate_size": 32, + "out_hidden_size": 32, + "num_heads": 4, + "in_channels": 3, + "image_size": 4, + "patch_size": 2, + "spatial_merge_size": 1, + "temporal_patch_size": 1, + }, + image_token_id=60, + video_token_id=61, + image_start_token_id=62, + image_end_token_id=63, + video_start_token_id=58, + video_end_token_id=59, + ) + source = glm5_next.Glm5NextForConditionalGeneration(config).eval() + packed_experts = source.model.language_model.layers[3].mlp.experts + expected_gate, expected_up = packed_experts.gate_up_proj[0].detach().chunk(2, dim=0) + expected_down = packed_experts.down_proj[0].detach().clone() + source.save_pretrained(tmp_path) + + with torch.device("meta"): + shell = glm5_next.Glm5NextForConditionalGeneration(config).eval() + assert _convert_model_with_defuser(Glm5NextQModel, shell, cleanup_original=False) is True + + turtle = LazyTurtle.maybe_create( + model_local_path=str(tmp_path), + config=shell.config, + model_init_kwargs={"device_map": {"": "cpu"}}, + module_tree=Glm5NextQModel.module_tree, + hf_conversion_map_reversed=Glm5NextQModel.resolve_hf_conversion_map_reversed( + target_model=shell + ), + target_model=shell, + ) + assert turtle is not None + + layer = shell.model.language_model.layers[3] + turtle.materialize_submodule( + target_model=shell, + target_submodule=layer, + device=torch.device("cpu"), + module_path="model.language_model.layers.3", + show_progress=False, + ) + + expert0 = layer.mlp.experts[0] + torch.testing.assert_close(expert0.gate_proj.weight, expected_gate) + torch.testing.assert_close(expert0.up_proj.weight, expected_up) + torch.testing.assert_close(expert0.down_proj.weight, expected_down) + + +def test_glm5_next_replay_propagates_dsa_topk_indices(): + model_def = Glm5NextQModel.__new__(Glm5NextQModel) + topk_indices = torch.tensor([[[1, 2], [0, 3]]]) + kwargs = {} + + returned = model_def.update_layer_replay_kwargs_from_output( + layer=SimpleNamespace(), + layer_output=(torch.zeros(1, 2, 4), topk_indices), + layer_input_kwargs=kwargs, + target_device=torch.device("cpu"), + ) + assert returned is kwargs + torch.testing.assert_close(kwargs["prev_topk_indices"], topk_indices) + + +def test_glm5_next_moe_calibration_uses_clamped_fused_gate(): + class Experts: + @staticmethod + def _apply_gate(gate_up): + gate, up = gate_up.chunk(2, dim=-1) + return torch.nn.functional.silu(gate.clamp(max=1.0)) * up.clamp(-1.0, 1.0) + + gate = torch.tensor([[3.0, -2.0]]) + up = torch.tensor([[4.0, -4.0]]) + actual = GateUpDownMoELifecycleHooks().apply_expert_activation( + experts_module=Experts(), + expert=SimpleNamespace(), + gate_out=gate, + up_out=up, + ) + expected = Experts._apply_gate(torch.cat([gate, up], dim=-1)) + torch.testing.assert_close(actual, expected) From 5ac982a2dc4fcf2153cd3dd112920374b1f9a2af Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Tue, 1 Sep 2026 15:59:23 +0800 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94397a2e5..713565556 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ ## Latest News 🗞️🚀 -* 09/01/2026 7.4.0-dev `main`: ✨ Added `glm5_next` / GLM-5.3-Flash hybrid KDA/DSA MoE quantization support. +* 09/01/2026 7.4.0-dev `main`: ✨ Added `glm5_next` / GLM-5.3-Flash quantization support. * 08/31/2026 7.4.0-dev `main`: ✨ Added Qwen3.8-Flash-Next (`qwen4_exp`) quantization. * 08/26/2026 7.4.0-dev `main`: ✨ Added NVIDIA `LocateAnything-3B` quantization support. * 08/25/2026 7.4.0-dev `main`: ✨ Added Tencent `HunyuanOCR` quantization support.