From 0a84f30effcee7b78b4b66c2718eac4b3db7f1ec Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Wed, 2 Sep 2026 20:28:40 +0800 Subject: [PATCH 1/2] fix: normalize multimodal calibration conversations --- .../models/definitions/base_qwen2_5_omni.py | 4 +- gptqmodel/models/definitions/base_qwen2_vl.py | 4 +- gptqmodel/models/definitions/base_qwen3_vl.py | 4 +- .../models/definitions/cohere_compass.py | 4 +- gptqmodel/models/definitions/deepseek_vl.py | 4 +- .../models/definitions/deepseek_vl_v2.py | 4 +- .../models/definitions/ernie4_5_vl_moe.py | 4 +- gptqmodel/models/definitions/hunyuan_vl.py | 4 +- gptqmodel/models/definitions/inkling.py | 4 +- .../models/definitions/intern_s2_preview.py | 4 +- gptqmodel/models/definitions/interns1.py | 4 +- gptqmodel/models/definitions/internvl_chat.py | 4 +- gptqmodel/models/definitions/lfm2_vl.py | 4 +- .../models/definitions/locateanything.py | 4 +- gptqmodel/models/definitions/minicpm_o.py | 4 +- gptqmodel/models/definitions/minicpmv.py | 4 +- gptqmodel/models/definitions/minicpmv_4_6.py | 4 +- gptqmodel/models/definitions/muse_glimmer.py | 4 +- gptqmodel/models/definitions/ovis2.py | 4 +- gptqmodel/models/definitions/ovis2_5.py | 4 +- gptqmodel/utils/calibration.py | 35 ++++++++++ tests/models/test_intern_s2_preview.py | 64 ++++++++++++++++++- tests/test_prepare_dataset.py | 47 ++++++++++++++ 23 files changed, 185 insertions(+), 41 deletions(-) diff --git a/gptqmodel/models/definitions/base_qwen2_5_omni.py b/gptqmodel/models/definitions/base_qwen2_5_omni.py index 639b81490..d097520ba 100644 --- a/gptqmodel/models/definitions/base_qwen2_5_omni.py +++ b/gptqmodel/models/definitions/base_qwen2_5_omni.py @@ -11,7 +11,7 @@ from transformers import AutoModelForTextToWaveform, AutoProcessor, ProcessorMixin from ...utils.audio import process_audio_info -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import extract_vision_info, fetch_image from ...utils.model import MODALITY from ...utils.offload import offload_to_disk @@ -225,7 +225,7 @@ def load_processor(self) -> ProcessorMixin: def prepare_dataset(self, calibration_dataset, calibration_dataset_concat_size=None, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): text = processor.apply_chat_template( batch, tokenize=False, add_generation_prompt=True ) diff --git a/gptqmodel/models/definitions/base_qwen2_vl.py b/gptqmodel/models/definitions/base_qwen2_vl.py index c2e557068..25c3f2bfd 100644 --- a/gptqmodel/models/definitions/base_qwen2_vl.py +++ b/gptqmodel/models/definitions/base_qwen2_vl.py @@ -9,7 +9,7 @@ from PIL import Image from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import extract_vision_info, fetch_image from ...utils.model import MODALITY, get_module, move_to from ...utils.offload import offload_to_disk @@ -295,7 +295,7 @@ def load_processor(self) -> ProcessorMixin: def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): text = processor.apply_chat_template( batch, tokenize=False, add_generation_prompt=True ) diff --git a/gptqmodel/models/definitions/base_qwen3_vl.py b/gptqmodel/models/definitions/base_qwen3_vl.py index 8525328d6..e41e176f1 100644 --- a/gptqmodel/models/definitions/base_qwen3_vl.py +++ b/gptqmodel/models/definitions/base_qwen3_vl.py @@ -9,7 +9,7 @@ from PIL import Image from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import extract_vision_info, fetch_image from ...utils.model import MODALITY, get_module, move_to from ...utils.offload import offload_to_disk @@ -182,7 +182,7 @@ def load_processor(self) -> ProcessorMixin: def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): text = processor.apply_chat_template( batch, tokenize=False, add_generation_prompt=True ) diff --git a/gptqmodel/models/definitions/cohere_compass.py b/gptqmodel/models/definitions/cohere_compass.py index db5afdce0..36cd65328 100644 --- a/gptqmodel/models/definitions/cohere_compass.py +++ b/gptqmodel/models/definitions/cohere_compass.py @@ -9,7 +9,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.device import get_device from ...utils.model import MODALITY, get_module_by_name_prefix, move_to, nested_move_to from ...utils.offload import offload_to_disk @@ -372,7 +372,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del kwargs processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append(self.prepare_inputs_for_conversations(processor, batch)) del processor return calib_data diff --git a/gptqmodel/models/definitions/deepseek_vl.py b/gptqmodel/models/definitions/deepseek_vl.py index 5b3357325..74859a662 100644 --- a/gptqmodel/models/definitions/deepseek_vl.py +++ b/gptqmodel/models/definitions/deepseek_vl.py @@ -8,7 +8,7 @@ import torch from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -108,7 +108,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del kwargs processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append(self.prepare_inputs_for_conversations(processor, batch)) del processor return calib_data diff --git a/gptqmodel/models/definitions/deepseek_vl_v2.py b/gptqmodel/models/definitions/deepseek_vl_v2.py index c5b650e1f..d5a9a9598 100644 --- a/gptqmodel/models/definitions/deepseek_vl_v2.py +++ b/gptqmodel/models/definitions/deepseek_vl_v2.py @@ -10,7 +10,7 @@ from torch import nn from transformers import AutoModelForCausalLM, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import fetch_image from ...utils.model import MODALITY, get_module, move_to from ...utils.offload import offload_to_disk @@ -164,7 +164,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del batch_size, kwargs processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, 1, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, 1, process_func=self.preprocess_dataset): conversation = batch[0] inputs = processor( conversations=conversation, diff --git a/gptqmodel/models/definitions/ernie4_5_vl_moe.py b/gptqmodel/models/definitions/ernie4_5_vl_moe.py index c9226fa29..4a9dd8a4c 100644 --- a/gptqmodel/models/definitions/ernie4_5_vl_moe.py +++ b/gptqmodel/models/definitions/ernie4_5_vl_moe.py @@ -12,7 +12,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin from transformers.masking_utils import create_causal_mask -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import extract_vision_info, fetch_image from ...utils.model import MODALITY, get_module, get_module_by_name_prefix, move_to from ...utils.offload import offload_to_disk @@ -370,7 +370,7 @@ def load_processor(self) -> ProcessorMixin: def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): text = processor.apply_chat_template( batch, tokenize=False, add_generation_prompt=True ) diff --git a/gptqmodel/models/definitions/hunyuan_vl.py b/gptqmodel/models/definitions/hunyuan_vl.py index 2c0f95cb6..bc02204f0 100644 --- a/gptqmodel/models/definitions/hunyuan_vl.py +++ b/gptqmodel/models/definitions/hunyuan_vl.py @@ -7,7 +7,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.looper_helpers import normalize_device_like from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk @@ -113,7 +113,7 @@ def prepare_dataset( del kwargs processor = self.load_processor() calibration_data = [] - for batch in batched( + for batch in batched_conversations( calibration_dataset, batch_size, process_func=self.preprocess_dataset, diff --git a/gptqmodel/models/definitions/inkling.py b/gptqmodel/models/definitions/inkling.py index ac71c3455..924456b0e 100644 --- a/gptqmodel/models/definitions/inkling.py +++ b/gptqmodel/models/definitions/inkling.py @@ -9,7 +9,7 @@ from transformers import AutoModelForMultimodalLM, AutoProcessor, ProcessorMixin from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -128,7 +128,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del kwargs processor = self.load_processor() calibration_data = [] - for batch in batched(calibration_dataset, batch_size): + for batch in batched_conversations(calibration_dataset, batch_size): calibration_data.append(self.prepare_inputs_for_conversations(processor, batch)) del processor return calibration_data diff --git a/gptqmodel/models/definitions/intern_s2_preview.py b/gptqmodel/models/definitions/intern_s2_preview.py index 500941cfa..57ac93490 100644 --- a/gptqmodel/models/definitions/intern_s2_preview.py +++ b/gptqmodel/models/definitions/intern_s2_preview.py @@ -10,7 +10,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -180,7 +180,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del kwargs processor = self.load_processor() calibration_data = [] - for batch in batched( + for batch in batched_conversations( calibration_dataset, batch_size, process_func=self.preprocess_dataset, diff --git a/gptqmodel/models/definitions/interns1.py b/gptqmodel/models/definitions/interns1.py index e15d4cc71..1921f13dc 100644 --- a/gptqmodel/models/definitions/interns1.py +++ b/gptqmodel/models/definitions/interns1.py @@ -9,7 +9,7 @@ from PIL import Image from transformers import AutoModelForCausalLM, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -130,7 +130,7 @@ def replace_image_with_pil(sample): def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): batched_samples = [] for sample in batch: batched_samples.append(self.replace_image_with_pil(sample)) diff --git a/gptqmodel/models/definitions/internvl_chat.py b/gptqmodel/models/definitions/internvl_chat.py index 5b2e69fbf..41bc1aa8c 100644 --- a/gptqmodel/models/definitions/internvl_chat.py +++ b/gptqmodel/models/definitions/internvl_chat.py @@ -14,7 +14,7 @@ from torchvision.transforms.functional import InterpolationMode from transformers import AutoModel, GenerationConfig -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import fetch_image from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk @@ -384,7 +384,7 @@ def pre_quantize_generate_hook_end(self): def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del batch_size, kwargs calib_data = [] - for batch in batched(calibration_dataset, 1): + for batch in batched_conversations(calibration_dataset, 1): calib_data.append(self.prepare_inputs_for_conversation(batch[0])) return calib_data diff --git a/gptqmodel/models/definitions/lfm2_vl.py b/gptqmodel/models/definitions/lfm2_vl.py index bc0b42330..659b4eebc 100644 --- a/gptqmodel/models/definitions/lfm2_vl.py +++ b/gptqmodel/models/definitions/lfm2_vl.py @@ -10,7 +10,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin from transformers.masking_utils import create_causal_mask -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to, nested_move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -127,7 +127,7 @@ def prepare_inputs_for_conversations( def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append(self.prepare_inputs_for_conversations(processor, batch)) del processor return calib_data diff --git a/gptqmodel/models/definitions/locateanything.py b/gptqmodel/models/definitions/locateanything.py index d8b49aa32..18a8be18e 100644 --- a/gptqmodel/models/definitions/locateanything.py +++ b/gptqmodel/models/definitions/locateanything.py @@ -10,7 +10,7 @@ import torch from transformers import AutoModel, AutoProcessor -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -104,7 +104,7 @@ def prepare_dataset( del batch_size, kwargs processor = self.load_processor() calibration_data = [] - for batch in batched(calibration_dataset, 1): + for batch in batched_conversations(calibration_dataset, 1): messages = batch[0] text = processor.py_apply_chat_template( messages, diff --git a/gptqmodel/models/definitions/minicpm_o.py b/gptqmodel/models/definitions/minicpm_o.py index 7fb2c6246..b5b48baae 100644 --- a/gptqmodel/models/definitions/minicpm_o.py +++ b/gptqmodel/models/definitions/minicpm_o.py @@ -12,7 +12,7 @@ from transformers.generation.utils import GenerationMixin from ...utils.audio import process_audio_info -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import fetch_image from ...utils.model import MODALITY, move_to, nested_move_to from ...utils.offload import offload_to_disk @@ -391,7 +391,7 @@ def prepare_inputs_for_conversations( def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.processor or self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append( self.prepare_inputs_for_conversations( processor, diff --git a/gptqmodel/models/definitions/minicpmv.py b/gptqmodel/models/definitions/minicpmv.py index 1ec451d4f..52dc70b67 100644 --- a/gptqmodel/models/definitions/minicpmv.py +++ b/gptqmodel/models/definitions/minicpmv.py @@ -9,7 +9,7 @@ from PIL import Image from transformers import AutoModel, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import fetch_image from ...utils.model import MODALITY, move_to, nested_move_to from ...utils.offload import offload_to_disk @@ -160,7 +160,7 @@ def prepare_inputs_for_conversations( def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append( self.prepare_inputs_for_conversations( processor, diff --git a/gptqmodel/models/definitions/minicpmv_4_6.py b/gptqmodel/models/definitions/minicpmv_4_6.py index 3b0228d46..794b272ad 100644 --- a/gptqmodel/models/definitions/minicpmv_4_6.py +++ b/gptqmodel/models/definitions/minicpmv_4_6.py @@ -7,7 +7,7 @@ from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to, nested_move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -112,7 +112,7 @@ def prepare_inputs_for_conversations( def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calib_data.append( self.prepare_inputs_for_conversations( processor, diff --git a/gptqmodel/models/definitions/muse_glimmer.py b/gptqmodel/models/definitions/muse_glimmer.py index 7a67a11a5..cb89dc677 100644 --- a/gptqmodel/models/definitions/muse_glimmer.py +++ b/gptqmodel/models/definitions/muse_glimmer.py @@ -10,7 +10,7 @@ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...utils.attn_mask import normalize_seq_mask -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY from ..base import BaseQModel @@ -79,7 +79,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): del kwargs processor = self.load_processor() calibration_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): calibration_data.append(self.prepare_inputs_for_conversations(processor, batch)) del processor return calibration_data diff --git a/gptqmodel/models/definitions/ovis2.py b/gptqmodel/models/definitions/ovis2.py index b95b98bb6..8932f3854 100644 --- a/gptqmodel/models/definitions/ovis2.py +++ b/gptqmodel/models/definitions/ovis2.py @@ -8,7 +8,7 @@ from PIL import Image from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.image import extract_vision_info, fetch_image from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk @@ -94,7 +94,7 @@ def process_vision_info( def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): processor = self.load_processor() calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): text = processor.apply_chat_template( batch, tokenize=False, add_generation_prompt=True ) diff --git a/gptqmodel/models/definitions/ovis2_5.py b/gptqmodel/models/definitions/ovis2_5.py index 46820eb8b..5b8906ee9 100644 --- a/gptqmodel/models/definitions/ovis2_5.py +++ b/gptqmodel/models/definitions/ovis2_5.py @@ -9,7 +9,7 @@ from PIL import Image from transformers import AutoModelForCausalLM, AutoProcessor, ProcessorMixin -from ...utils.calibration import batched +from ...utils.calibration import batched_conversations from ...utils.model import MODALITY, move_to from ...utils.offload import offload_to_disk from .._const import CPU @@ -103,7 +103,7 @@ def replace_image_with_pil(sample): def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs): calib_data = [] - for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset): + for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset): for sample in batch: sample = self.replace_image_with_pil(sample) input_ids, pixel_values, grid_thws = self.model.preprocess_inputs( diff --git a/gptqmodel/utils/calibration.py b/gptqmodel/utils/calibration.py index 3b3e28c60..766f83a53 100644 --- a/gptqmodel/utils/calibration.py +++ b/gptqmodel/utils/calibration.py @@ -52,6 +52,41 @@ def batched(iterable, batch_size: int, process_func=None): yield batch +def normalize_chat_calibration_sample(sample: Any) -> Any: + """Normalize text-like calibration rows to processor chat conversations. + + Multimodal processors expect one conversation per calibration sample, while + the public quantization API also accepts raw text and common dataset rows. + Already structured conversations are returned unchanged. + """ + + if isinstance(sample, dict): + if "messages" in sample: + return sample["messages"] + if isinstance(sample.get("text"), str): + sample = sample["text"] + + if isinstance(sample, str): + return [ + { + "role": "user", + "content": [{"type": "text", "text": sample}], + } + ] + + return sample + + +def batched_conversations(iterable, batch_size: int, process_func=None): + """Yield chat-normalized calibration samples in fixed-size batches.""" + + def normalize_and_process(sample): + sample = normalize_chat_calibration_sample(sample) + return process_func(sample) if process_func is not None else sample + + yield from batched(iterable, batch_size, process_func=normalize_and_process) + + def prepare_calibration_dataset( qmodel, calibration_dataset: CalibrationInputType, diff --git a/tests/models/test_intern_s2_preview.py b/tests/models/test_intern_s2_preview.py index ec57489fa..11cb775ea 100644 --- a/tests/models/test_intern_s2_preview.py +++ b/tests/models/test_intern_s2_preview.py @@ -12,6 +12,68 @@ from gptqmodel.models.definitions import intern_s2_preview +def test_prepare_dataset_wraps_raw_text_as_chat_messages(monkeypatch): + calls = [] + + class FakeProcessor: + def apply_chat_template(self, conversations, **kwargs): + calls.append((conversations, kwargs)) + return { + "input_ids": torch.tensor([[1, 2, 3]]), + "attention_mask": torch.tensor([[1, 1, 1]]), + } + + instance = object.__new__(intern_s2_preview.InternS2PreviewQModel) + monkeypatch.setattr(instance, "load_processor", lambda: FakeProcessor()) + + prepared = instance.prepare_dataset(["first", "second"], batch_size=2) + + assert len(prepared) == 1 + assert calls == [ + ( + [ + [{"role": "user", "content": [{"type": "text", "text": "first"}]}], + [{"role": "user", "content": [{"type": "text", "text": "second"}]}], + ], + { + "tokenize": True, + "add_generation_prompt": True, + "processor_kwargs": {"padding": True}, + "return_dict": True, + "return_tensors": "pt", + }, + ) + ] + + +def test_prepare_dataset_preserves_multimodal_conversations(monkeypatch): + calls = [] + + class FakeProcessor: + def apply_chat_template(self, conversations, **kwargs): + calls.append(conversations) + return { + "input_ids": torch.tensor([[1, 2, 3]]), + "attention_mask": torch.tensor([[1, 1, 1]]), + } + + conversation = [ + { + "role": "user", + "content": [ + {"type": "image", "image": "image.jpg"}, + {"type": "text", "text": "describe it"}, + ], + } + ] + instance = object.__new__(intern_s2_preview.InternS2PreviewQModel) + monkeypatch.setattr(instance, "load_processor", lambda: FakeProcessor()) + + instance.prepare_dataset([conversation], batch_size=1) + + assert calls == [[conversation]] + + def test_causal_mask_compat_drops_removed_cache_position(monkeypatch): module_name = "tests.fake_modeling_intern_s2_preview" modeling_module = types.ModuleType(module_name) @@ -112,4 +174,4 @@ class TestInternS2Preview(ModelTest): EVAL_SINGLE_GPU = False def test_intern_s2_preview(self): - self.quantize_and_evaluate() \ No newline at end of file + self.quantize_and_evaluate() diff --git a/tests/test_prepare_dataset.py b/tests/test_prepare_dataset.py index f1fff2845..a35082a8e 100644 --- a/tests/test_prepare_dataset.py +++ b/tests/test_prepare_dataset.py @@ -9,6 +9,7 @@ import torch from gptqmodel.models.base import BaseQModel +from gptqmodel.utils.calibration import batched_conversations, normalize_chat_calibration_sample from gptqmodel.utils.data import collate_data @@ -91,6 +92,52 @@ def _sample_dataset(): ] +def test_normalize_chat_calibration_sample_supports_public_text_inputs(): + expected = [ + { + "role": "user", + "content": [{"type": "text", "text": "hello"}], + } + ] + + assert normalize_chat_calibration_sample("hello") == expected + assert normalize_chat_calibration_sample({"text": "hello", "source": "dataset"}) == expected + + +def test_normalize_chat_calibration_sample_unwraps_messages_and_preserves_conversations(): + conversation = [ + { + "role": "user", + "content": [ + {"type": "image", "image": "image.jpg"}, + {"type": "text", "text": "describe it"}, + ], + } + ] + + assert normalize_chat_calibration_sample({"messages": conversation}) is conversation + assert normalize_chat_calibration_sample(conversation) is conversation + + +def test_batched_conversations_normalizes_before_model_preprocessing(): + processed = [] + + def preprocess(sample): + processed.append(sample) + return sample + + batches = list( + batched_conversations( + ["first", {"text": "second"}], + batch_size=2, + process_func=preprocess, + ) + ) + + assert batches == [processed] + assert [sample[0]["content"][0]["text"] for sample in processed] == ["first", "second"] + + def test_prepare_dataset_concat_without_separator(): qmodel = _make_qmodel() dataset = copy.deepcopy(_sample_dataset()) From eda9924fb17a1c4b2a08fdfff1af36930cfce437 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Wed, 2 Sep 2026 20:53:41 +0800 Subject: [PATCH 2/2] fix: align multimodal capture inputs with embeddings --- gptqmodel/looper/stage_inputs_capture.py | 30 ++++++++++++++++++++---- tests/test_stage_inputs_capture.py | 25 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/gptqmodel/looper/stage_inputs_capture.py b/gptqmodel/looper/stage_inputs_capture.py index d3567dfa5..096d9b9b9 100644 --- a/gptqmodel/looper/stage_inputs_capture.py +++ b/gptqmodel/looper/stage_inputs_capture.py @@ -52,6 +52,27 @@ def _materialize_modules_with_direct_meta_tensors(self, device: torch.device) -> device=device, ) + def _resolve_forward_device( + self, + example: Dict[str, Any], + fallback: torch.device, + ) -> torch.device: + """Resolve where model inputs must live for the pre-layer forward.""" + + if not torch.is_tensor(example.get("input_ids")): + return fallback + + try: + embedding = self.gptq_model.get_input_embeddings() + except Exception: + return fallback + + if not isinstance(embedding, torch.nn.Module): + return fallback + + embedding_device = get_device(embedding) + return fallback if embedding_device == META else embedding_device + def cache_inputs( self, layers: Sequence[torch.nn.Module], @@ -248,14 +269,15 @@ def _resolve_module_name(mod: torch.nn.Module) -> Optional[str]: try: for batch_index, example in enumerate(calibration_data, start=1): if self.gptq_model.ATTENTION_MASKS_REQUIRED_FOR_INPUT: - data_device = self.gptq_model.quantize_config.device + forward_device = self.gptq_model.quantize_config.device else: - data_device = ( + forward_device = ( self.gptq_model.quantize_config.device if _has_vision_inputs(example) else cur_layer_device ) - example = self.gptq_model.move_input_capture_example(example, data_device) + forward_device = self._resolve_forward_device(example, forward_device) + example = self.gptq_model.move_input_capture_example(example, forward_device) try: with ctx( DEVICE_THREAD_POOL.read_lock(self.gptq_model.quantize_config.device), @@ -264,7 +286,7 @@ def _resolve_module_name(mod: torch.nn.Module) -> Optional[str]: self.gptq_model.run_input_capture( example, use_cache=use_cache, - data_device=data_device, + data_device=forward_device, ) except StopForward: pass diff --git a/tests/test_stage_inputs_capture.py b/tests/test_stage_inputs_capture.py index 67f2b07f3..aab0f9c5e 100644 --- a/tests/test_stage_inputs_capture.py +++ b/tests/test_stage_inputs_capture.py @@ -153,6 +153,31 @@ def test_cache_inputs_warns_when_caller_name_differs_from_full_name(self): self.assertIn("model.layers.42", message) self.assertIn("legacy.path.layer_0", message) + def test_forward_device_prefers_materialized_embedding_device(self): + layer = FakeLayer() + capture, _, _, gptq_model = self._make_capture(layer) + embedding = nn.Embedding(8, 4) + gptq_model.get_input_embeddings.return_value = embedding + + resolved = capture._resolve_forward_device( + {"input_ids": torch.tensor([[1, 2, 3]])}, + fallback=torch.device("meta"), + ) + + self.assertEqual(resolved, embedding.weight.device) + + def test_forward_device_uses_fallback_for_unmaterialized_embedding(self): + layer = FakeLayer() + capture, _, _, gptq_model = self._make_capture(layer) + gptq_model.get_input_embeddings.return_value = nn.Embedding(8, 4, device="meta") + + resolved = capture._resolve_forward_device( + {"input_ids": torch.tensor([[1, 2, 3]])}, + fallback=torch.device("cpu"), + ) + + self.assertEqual(resolved, torch.device("cpu")) + if __name__ == "__main__": unittest.main()