fix: normalize multimodal calibration inputs

gptqmodel/looper/stage_inputs_capture.py

@@ -52,6 +52,27 @@ def _materialize_modules_with_direct_meta_tensors(self, device: torch.device) ->
                     device=device,
                 )
 
+    def _resolve_forward_device(
+        self,
+        example: Dict[str, Any],
+        fallback: torch.device,
+    ) -> torch.device:
+        """Resolve where model inputs must live for the pre-layer forward."""
+
+        if not torch.is_tensor(example.get("input_ids")):
+            return fallback
+
+        try:
+            embedding = self.gptq_model.get_input_embeddings()
+        except Exception:
+            return fallback
+
+        if not isinstance(embedding, torch.nn.Module):
+            return fallback
+
+        embedding_device = get_device(embedding)
+        return fallback if embedding_device == META else embedding_device
+
     def cache_inputs(
         self,
         layers: Sequence[torch.nn.Module],
@@ -248,14 +269,15 @@ def _resolve_module_name(mod: torch.nn.Module) -> Optional[str]:
         try:
             for batch_index, example in enumerate(calibration_data, start=1):
                 if self.gptq_model.ATTENTION_MASKS_REQUIRED_FOR_INPUT:
-                    data_device = self.gptq_model.quantize_config.device
+                    forward_device = self.gptq_model.quantize_config.device
                 else:
-                    data_device = (
+                    forward_device = (
                         self.gptq_model.quantize_config.device
                         if _has_vision_inputs(example)
                         else cur_layer_device
                     )
-                example = self.gptq_model.move_input_capture_example(example, data_device)
+                forward_device = self._resolve_forward_device(example, forward_device)
+                example = self.gptq_model.move_input_capture_example(example, forward_device)
                 try:
                     with ctx(
                         DEVICE_THREAD_POOL.read_lock(self.gptq_model.quantize_config.device),
@@ -264,7 +286,7 @@ def _resolve_module_name(mod: torch.nn.Module) -> Optional[str]:
                         self.gptq_model.run_input_capture(
                             example,
                             use_cache=use_cache,
-                            data_device=data_device,
+                            data_device=forward_device,
                         )
                 except StopForward:
                     pass

gptqmodel/models/definitions/base_qwen2_5_omni.py

@@ -11,7 +11,7 @@
 from transformers import AutoModelForTextToWaveform, AutoProcessor, ProcessorMixin
 
 from ...utils.audio import process_audio_info
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import extract_vision_info, fetch_image
 from ...utils.model import MODALITY
 from ...utils.offload import offload_to_disk
@@ -225,7 +225,7 @@ def load_processor(self) -> ProcessorMixin:
     def prepare_dataset(self, calibration_dataset, calibration_dataset_concat_size=None, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             text = processor.apply_chat_template(
                 batch, tokenize=False, add_generation_prompt=True
             )

gptqmodel/models/definitions/base_qwen2_vl.py

@@ -9,7 +9,7 @@
 from PIL import Image
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import extract_vision_info, fetch_image
 from ...utils.model import MODALITY, get_module, move_to
 from ...utils.offload import offload_to_disk
@@ -295,7 +295,7 @@ def load_processor(self) -> ProcessorMixin:
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             text = processor.apply_chat_template(
                 batch, tokenize=False, add_generation_prompt=True
             )

gptqmodel/models/definitions/base_qwen3_vl.py

@@ -9,7 +9,7 @@
 from PIL import Image
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import extract_vision_info, fetch_image
 from ...utils.model import MODALITY, get_module, move_to
 from ...utils.offload import offload_to_disk
@@ -182,7 +182,7 @@ def load_processor(self) -> ProcessorMixin:
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             text = processor.apply_chat_template(
                 batch, tokenize=False, add_generation_prompt=True
             )

gptqmodel/models/definitions/cohere_compass.py

@@ -9,7 +9,7 @@
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.device import get_device
 from ...utils.model import MODALITY, get_module_by_name_prefix, move_to, nested_move_to
 from ...utils.offload import offload_to_disk
@@ -372,7 +372,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del kwargs
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(self.prepare_inputs_for_conversations(processor, batch))
         del processor
         return calib_data

gptqmodel/models/definitions/deepseek_vl.py

@@ -8,7 +8,7 @@
 import torch
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -108,7 +108,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del kwargs
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(self.prepare_inputs_for_conversations(processor, batch))
         del processor
         return calib_data

gptqmodel/models/definitions/deepseek_vl_v2.py

@@ -10,7 +10,7 @@
 from torch import nn
 from transformers import AutoModelForCausalLM, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import fetch_image
 from ...utils.model import MODALITY, get_module, move_to
 from ...utils.offload import offload_to_disk
@@ -164,7 +164,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del batch_size, kwargs
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, 1, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, 1, process_func=self.preprocess_dataset):
             conversation = batch[0]
             inputs = processor(
                 conversations=conversation,

gptqmodel/models/definitions/ernie4_5_vl_moe.py

@@ -12,7 +12,7 @@
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 from transformers.masking_utils import create_causal_mask
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import extract_vision_info, fetch_image
 from ...utils.model import MODALITY, get_module, get_module_by_name_prefix, move_to
 from ...utils.offload import offload_to_disk
@@ -370,7 +370,7 @@ def load_processor(self) -> ProcessorMixin:
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             text = processor.apply_chat_template(
                 batch, tokenize=False, add_generation_prompt=True
             )

gptqmodel/models/definitions/hunyuan_vl.py

@@ -7,7 +7,7 @@
 
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.looper_helpers import normalize_device_like
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
@@ -113,7 +113,7 @@ def prepare_dataset(
         del kwargs
         processor = self.load_processor()
         calibration_data = []
-        for batch in batched(
+        for batch in batched_conversations(
             calibration_dataset,
             batch_size,
             process_func=self.preprocess_dataset,

gptqmodel/models/definitions/inkling.py

@@ -9,7 +9,7 @@
 from transformers import AutoModelForMultimodalLM, AutoProcessor, ProcessorMixin
 from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -128,7 +128,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del kwargs
         processor = self.load_processor()
         calibration_data = []
-        for batch in batched(calibration_dataset, batch_size):
+        for batch in batched_conversations(calibration_dataset, batch_size):
             calibration_data.append(self.prepare_inputs_for_conversations(processor, batch))
         del processor
         return calibration_data

gptqmodel/models/definitions/intern_s2_preview.py

@@ -10,7 +10,7 @@
 
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -180,7 +180,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del kwargs
         processor = self.load_processor()
         calibration_data = []
-        for batch in batched(
+        for batch in batched_conversations(
             calibration_dataset,
             batch_size,
             process_func=self.preprocess_dataset,

gptqmodel/models/definitions/interns1.py

@@ -9,7 +9,7 @@
 from PIL import Image
 from transformers import AutoModelForCausalLM, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -130,7 +130,7 @@ def replace_image_with_pil(sample):
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             batched_samples = []
             for sample in batch:
                 batched_samples.append(self.replace_image_with_pil(sample))

gptqmodel/models/definitions/internvl_chat.py

@@ -14,7 +14,7 @@
 from torchvision.transforms.functional import InterpolationMode
 from transformers import AutoModel, GenerationConfig
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import fetch_image
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
@@ -384,7 +384,7 @@ def pre_quantize_generate_hook_end(self):
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del batch_size, kwargs
         calib_data = []
-        for batch in batched(calibration_dataset, 1):
+        for batch in batched_conversations(calibration_dataset, 1):
             calib_data.append(self.prepare_inputs_for_conversation(batch[0]))
         return calib_data
 

gptqmodel/models/definitions/lfm2_vl.py

@@ -10,7 +10,7 @@
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 from transformers.masking_utils import create_causal_mask
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to, nested_move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -127,7 +127,7 @@ def prepare_inputs_for_conversations(
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(self.prepare_inputs_for_conversations(processor, batch))
         del processor
         return calib_data

gptqmodel/models/definitions/locateanything.py

@@ -10,7 +10,7 @@
 import torch
 from transformers import AutoModel, AutoProcessor
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -104,7 +104,7 @@ def prepare_dataset(
         del batch_size, kwargs
         processor = self.load_processor()
         calibration_data = []
-        for batch in batched(calibration_dataset, 1):
+        for batch in batched_conversations(calibration_dataset, 1):
             messages = batch[0]
             text = processor.py_apply_chat_template(
                 messages,

gptqmodel/models/definitions/minicpm_o.py

@@ -12,7 +12,7 @@
 from transformers.generation.utils import GenerationMixin
 
 from ...utils.audio import process_audio_info
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import fetch_image
 from ...utils.model import MODALITY, move_to, nested_move_to
 from ...utils.offload import offload_to_disk
@@ -391,7 +391,7 @@ def prepare_inputs_for_conversations(
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.processor or self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(
                 self.prepare_inputs_for_conversations(
                     processor,

gptqmodel/models/definitions/minicpmv.py

@@ -9,7 +9,7 @@
 from PIL import Image
 from transformers import AutoModel, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import fetch_image
 from ...utils.model import MODALITY, move_to, nested_move_to
 from ...utils.offload import offload_to_disk
@@ -160,7 +160,7 @@ def prepare_inputs_for_conversations(
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(
                 self.prepare_inputs_for_conversations(
                     processor,

gptqmodel/models/definitions/minicpmv_4_6.py

@@ -7,7 +7,7 @@
 
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY, move_to, nested_move_to
 from ...utils.offload import offload_to_disk
 from .._const import CPU
@@ -112,7 +112,7 @@ def prepare_inputs_for_conversations(
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calib_data.append(
                 self.prepare_inputs_for_conversations(
                     processor,

gptqmodel/models/definitions/muse_glimmer.py

@@ -10,7 +10,7 @@
 from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
 
 from ...utils.attn_mask import normalize_seq_mask
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.model import MODALITY
 from ..base import BaseQModel
 
@@ -79,7 +79,7 @@ def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         del kwargs
         processor = self.load_processor()
         calibration_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             calibration_data.append(self.prepare_inputs_for_conversations(processor, batch))
         del processor
         return calibration_data

gptqmodel/models/definitions/ovis2.py

@@ -8,7 +8,7 @@
 from PIL import Image
 from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin
 
-from ...utils.calibration import batched
+from ...utils.calibration import batched_conversations
 from ...utils.image import extract_vision_info, fetch_image
 from ...utils.model import MODALITY, move_to
 from ...utils.offload import offload_to_disk
@@ -94,7 +94,7 @@ def process_vision_info(
     def prepare_dataset(self, calibration_dataset, batch_size: int = 1, **kwargs):
         processor = self.load_processor()
         calib_data = []
-        for batch in batched(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
+        for batch in batched_conversations(calibration_dataset, batch_size, process_func=self.preprocess_dataset):
             text = processor.apply_chat_template(
                 batch, tokenize=False, add_generation_prompt=True
             )