Add support of video MiniCPM-V_4_5 (#2044)

* add video support

* Finally fixed accuracy in WWB

* some optimisations

* apply review comments

* skip minicpm_v4_5 for genai at the moment

* style fix

* fix CI

* Fix video temporal ids

* Add batched temporal resampling

Author: pavel-esir

optimum/intel/openvino/modeling_visual_language.py

@@ -1296,6 +1296,7 @@ def forward(
             audio_embed_sizes=audio_embed_sizes,
             audio_attention_mask=audio_attention_mask,
             input_mode=input_mode,
+            temporal_ids=temporal_ids,
             **kwargs,
         )
 
@@ -1648,6 +1649,7 @@ def preprocess_inputs(
         tokenizer: Optional[PreTrainedTokenizer] = None,
         config: Optional[PretrainedConfig] = None,
         video: Optional["VideoInput"] = None,
+        video_temporal_ids: Optional[List[List[int]]] = None,
         audio: Optional[np.ndarray] = None,
     ):
         if processor is None:
@@ -2572,6 +2574,7 @@ def get_vision_embeddings(self, pixel_values, input_ids=None, **kwargs):
         if input_ids is not None and input_ids.shape[1] == 1:
             return None
         tgt_sizes = kwargs["tgt_sizes"]
+        temporal_ids = kwargs.get("temporal_ids")
         pixel_values_list = pixel_values
         vision_hidden_states = []
         all_pixel_values = []
@@ -2607,14 +2610,17 @@ def get_vision_embeddings(self, pixel_values, input_ids=None, **kwargs):
                     pixel_values=all_pixel_values, patch_attention_mask=patch_attn_mask, position_ids=position_ids
                 )[0]
             )
-            vision_embedding = self.resampling(vision_embedding, tgt_sizes)
+            temporal_groups = None
+            if temporal_ids is not None:
+                temporal_groups = [group for sample_groups in temporal_ids for group in sample_groups]
+            vision_embedding = self.resampling(vision_embedding, tgt_sizes, temporal_groups)
 
             start = 0
-            for pixel_value in pixel_values_list:
-                img_cnt = len(pixel_value)
-                if img_cnt > 0:
-                    vision_hidden_states.append(vision_embedding[start : start + img_cnt])
-                    start += img_cnt
+            for sample_idx, pixel_value in enumerate(pixel_values_list):
+                output_count = len(temporal_ids[sample_idx]) if temporal_ids is not None else len(pixel_value)
+                if output_count > 0:
+                    vision_hidden_states.append(vision_embedding[start : start + output_count])
+                    start += output_count
                 else:
                     vision_hidden_states.append([])
         else:  # no image
@@ -2623,7 +2629,7 @@ def get_vision_embeddings(self, pixel_values, input_ids=None, **kwargs):
                 vision_hidden_states.append(dummy_feature)
         return vision_hidden_states
 
-    def resampling(self, x, tgt_sizes):
+    def resampling(self, x, tgt_sizes, temporal_groups=None):
         bs = x.shape[0]
 
         patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]
@@ -2642,6 +2648,50 @@ def resampling(self, x, tgt_sizes):
         pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed, batch_first=True, padding_value=0.0).permute(
             1, 0, 2
         )  # BLD => L * B * D
+        # A temporal group is represented by one fixed set of resampler query tokens, so concatenate
+        # its frame features and positions before inference instead of producing tokens for every frame.
+        if temporal_groups is not None:
+            group_features = []
+            group_positions = []
+            group_masks = []
+            frame_start = 0
+            for group in temporal_groups:
+                group_ids = [int(temporal_id) for temporal_id in group]
+                if not group_ids:
+                    raise ValueError("Temporal groups must not be empty")
+                frame_end = frame_start + len(group_ids)
+                if frame_end > bs:
+                    raise ValueError("Temporal groups contain more frames than were encoded")
+
+                group_pos_embed = pos_embed[:, frame_start:frame_end].permute(1, 0, 2)
+                temporal_positions = np.asarray([max(temporal_id, 0) for temporal_id in group_ids], dtype=np.float32)
+                temporal_embed = self._get_1d_sincos_pos_embed_from_grid_new(
+                    self.embed_dim, temporal_positions[:, None]
+                ).squeeze(1)
+                temporal_embed[np.asarray(group_ids) < 0] = 0
+                group_pos_embed += torch.from_numpy(temporal_embed).to(group_pos_embed.dtype).unsqueeze(1)
+
+                group_features.append(x[frame_start:frame_end].reshape(-1, x.shape[-1]))
+                group_positions.append(group_pos_embed.reshape(-1, self.embed_dim))
+                group_masks.append(key_padding_mask[frame_start:frame_end].reshape(-1))
+                frame_start = frame_end
+
+            if frame_start != bs:
+                raise ValueError(f"Temporal groups cover {frame_start} frames, but {bs} were encoded")
+
+            image_feature = torch.nn.utils.rnn.pad_sequence(group_features, batch_first=True, padding_value=0.0)
+            pos_embed = torch.nn.utils.rnn.pad_sequence(group_positions, batch_first=True, padding_value=0.0).permute(
+                1, 0, 2
+            )
+            key_padding_mask = torch.nn.utils.rnn.pad_sequence(group_masks, batch_first=True, padding_value=True)
+            return torch.from_numpy(
+                self.resampler(
+                    image_feature=image_feature,
+                    pos_embed=pos_embed,
+                    key_padding_mask=key_padding_mask,
+                )
+            )
+
         res = torch.from_numpy(self.resampler(image_feature=x, pos_embed=pos_embed, key_padding_mask=key_padding_mask))
         return res
 
@@ -2761,14 +2811,57 @@ def preprocess_inputs(
         tokenizer: Optional[PreTrainedTokenizer] = None,
         config: Optional[PretrainedConfig] = None,
         video: Optional["VideoInput"] = None,
+        video_temporal_ids: Optional[List[List[int]]] = None,
         audio: Optional[np.ndarray] = None,
     ):
         if processor is None:
             raise ValueError("Processor is required.")
-        if video is not None:
-            raise ValueError("Video input is not supported")
         if audio is not None:
             raise ValueError("Audio input is not supported")
+        if video is not None and (getattr(config, "version", 0) or 0) >= 4.5:
+            from PIL import Image as PILImage
+
+            video_frames = [
+                frame if isinstance(frame, PILImage.Image) else PILImage.fromarray(np.asarray(frame))
+                for frame in video
+            ]
+            if len(video_frames) == 0:
+                raise ValueError("Video input must contain at least one frame")
+
+            image_inputs = [] if image is None else image if isinstance(image, list) else [image]
+            visual_inputs = image_inputs + video_frames
+            temporal_ids = [[-1] for _ in image_inputs]
+            if video_temporal_ids is None:
+                temporal_ids.extend(
+                    list(range(group_start, min(group_start + 6, len(video_frames))))
+                    for group_start in range(0, len(video_frames), 6)
+                )
+            else:
+                if sum(len(group) for group in video_temporal_ids) != len(video_frames):
+                    raise ValueError("video_temporal_ids must contain one ID for every video frame")
+                temporal_ids.extend(video_temporal_ids)
+            messages = [
+                {
+                    "role": "user",
+                    "content": "(<image>./</image>)" * len(visual_inputs) + "\n" + text,
+                }
+            ]
+            chat_template_processor = (
+                processor if getattr(processor, "chat_template", None) is not None else processor.tokenizer
+            )
+            prompt = chat_template_processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            inputs = processor(
+                prompt,
+                visual_inputs,
+                temporal_ids=temporal_ids,
+                max_slice_nums=1,
+                use_image_id=False,
+                return_tensors="pt",
+            )
+            inputs.pop("image_sizes", None)
+            return inputs
+        if video is not None:
+            raise ValueError("Video input is not supported")
         if getattr(processor, "chat_template", None) is not None:
             messages = [{"role": "user", "content": text if image is None else "(<image>./</image>)\n" + text}]
             prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)

tests/openvino/test_genai.py

@@ -353,6 +353,7 @@ def test_compare_outputs(self, model_arch):
     "llama4",
     "maira2",
     "minicpmv",  # transformers output is empty with tiny model on transformers 4.57
+    "minicpm_v4_5",  # tokenizer argument unsupported by this generic test
     "smolvlm",
     "videochat_flash_qwen",  # GenAI requires video input; image-only not supported
     "qwen3_omni_moe",

tests/openvino/test_seq2seq.py

@@ -589,6 +589,7 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin):
         "videochat_flash_qwen",
         "internvl_chat",
         "minicpmv",
+        "minicpm_v4_5",
         "minicpmo",
         "llava-qwen2",
         "phi3_v",
@@ -620,6 +621,7 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin):
         "qwen3_vl",
         "videochat_flash_qwen",
         "muse_glimmer",
+        "minicpm_v4_5",
         "gemma4",
         "gemma4_moe",
         "gemma4_unified",
@@ -756,7 +758,7 @@ def compare_outputs(inputs, ov_model, transformers_model, generation_config):
                 transformers_inputs["inputs"] = transformers_inputs.pop("input_ids")
             ov_outputs = ov_model.generate(**inputs, generation_config=generation_config)
             # original minicpmv, internvl always skip input tokens in generation results, while transformers based approach provide them
-            if model_arch in ["minicpmv", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
+            if model_arch in ["minicpmv", "minicpm_v4_5", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
                 ov_outputs = ov_outputs[:, inputs["input_ids"].shape[1] :]
             with torch.no_grad():
                 transformers_outputs = transformers_model.generate(
@@ -846,7 +848,7 @@ def compare_outputs(inputs, ov_model, transformers_model, generation_config):
         self._check_device_and_request(ov_model, test_device, False)
 
         # pytorch minicpmv and internvl_chat are not designed to be used via forward
-        if model_arch not in ["minicpmv", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
+        if model_arch not in ["minicpmv", "minicpm_v4_5", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
             set_seed(SEED)
             ov_outputs = ov_model(**inputs)
             set_seed(SEED)
@@ -883,8 +885,8 @@ def compare_outputs(inputs, ov_model, transformers_model, generation_config):
             transformers_inputs["past_key_values"] = DynamicCache()
 
         with torch.no_grad():
-            if model_arch in ["minicpmo"]:
-                # `generate` method for minicpmo requires tokenizer
+            if model_arch in ["minicpmo", "minicpm_v4_5"]:
+                # MiniCPM's custom generate method requires the tokenizer to resolve terminator IDs.
                 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=model_arch in REMOTE_CODE_MODELS)
                 additional_inputs["tokenizer"] = tokenizer
             transformers_outputs = transformers_model.generate(
@@ -895,7 +897,7 @@ def compare_outputs(inputs, ov_model, transformers_model, generation_config):
                 transformers_outputs = transformers_outputs[1].sequences
 
         # original minicpmv, internvl always skip input tokens in generation results, while transformers based approach provide them
-        if model_arch in ["minicpmv", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
+        if model_arch in ["minicpmv", "minicpm_v4_5", "minicpmo", "internvl_chat", "videochat_flash_qwen"]:
             ov_outputs = ov_outputs[:, inputs["input_ids"].shape[1] :]
         self.assertTrue(
             torch.equal(ov_outputs, transformers_outputs),
@@ -915,11 +917,14 @@ def compare_outputs(inputs, ov_model, transformers_model, generation_config):
                 num_frames = 32
             input_video, _ = load_video(video_path, num_frames=num_frames, backend="opencv")
             question = "Why is this video funny?"
-            inputs = ov_model.preprocess_inputs(**preprocessors, text=question, video=input_video)
+            video_kwargs = {"video_temporal_ids": [[0, 100]]} if model_arch == "minicpm_v4_5" else {}
+            inputs = ov_model.preprocess_inputs(**preprocessors, text=question, video=input_video, **video_kwargs)
             compare_outputs(inputs, ov_model, transformers_model, gen_config)
 
             # check video+image scenario
-            inputs = ov_model.preprocess_inputs(**preprocessors, text=question, video=input_video, image=image)
+            inputs = ov_model.preprocess_inputs(
+                **preprocessors, text=question, video=input_video, image=image, **video_kwargs
+            )
             compare_outputs(inputs, ov_model, transformers_model, gen_config)
 
         if model_arch in self.SUPPORT_AUDIO:
@@ -983,9 +988,13 @@ def test_minicpmv_resampler_padding_mask(self, model_arch):
         tgt_sizes = torch.tensor([[8, 8], [6, 5], [4, 4]], dtype=torch.int32)
         set_seed(SEED)
         image_features = torch.randn(3, 64, transformers_model.config.vision_config.hidden_size)
+        temporal_groups = [[0, 1], [100]] if model_arch == "minicpm_v4_5" else None
         with torch.no_grad():
-            transformers_outputs = transformers_model.resampler(image_features, tgt_sizes)
-        ov_outputs = ov_model.resampling(image_features, tgt_sizes)
+            if temporal_groups is None:
+                transformers_outputs = transformers_model.resampler(image_features, tgt_sizes)
+            else:
+                transformers_outputs = transformers_model.resampler(image_features, tgt_sizes, temporal_groups)
+        ov_outputs = ov_model.resampling(image_features, tgt_sizes, temporal_groups)
         max_diff = torch.abs(ov_outputs - transformers_outputs).max()
         self.assertLess(max_diff, 1e-4, f"Max abs diff {max_diff}")
 

tests/openvino/utils_tests.py

@@ -1140,6 +1140,7 @@ def get_num_sdpa(model):
     "ltx2": "ltx2-video-transformer",
     "ltx2.3": "ltx2-video-transformer",
     "mistral-nemo": "mistral",
+    "minicpm_v4_5": "minicpmv",
     "mixtral_awq": "mixtral",
     "nanollava_vision_tower": "siglip",
     "opt125m": "opt",