feat(cpu): add Qwen3.5 4B multimodal support

Author: Aharrypotter

examples/qwen3_5/README.md

@@ -2,13 +2,14 @@
 
 This example supports the text towers of
 [`Qwen/Qwen3.5-0.8B`](https://huggingface.co/Qwen/Qwen3.5-0.8B) and
-[`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B), plus single-image,
-ordered multi-image, and bounded short-video inference for Qwen3.5-0.8B.
+[`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B), including
+single-image, ordered multi-image, and bounded short-video inference for both
+model sizes.
 
 | Model | Text | Images | Decoded RGB-frame API | Local H.264 MP4 | Hidden size | GDN / full-attention layers |
 | --- | --- | --- | --- | --- | ---: | ---: |
 | 0.8B | Yes | Yes | Yes | Yes, portable backend only | 1024 | 18 / 6 |
-| 4B | Yes | No | No | No | 2560 | 24 / 8 |
+| 4B | Yes | Yes | Yes | Yes, portable backend only | 2560 | 24 / 8 |
 
 All configurations support batch size 1 and a maximum cache length of 2048
 tokens. `Qwen3_5ForCausalLM::resetState()` clears the full-attention KV cache
@@ -112,7 +113,7 @@ python examples/qwen3_5/validate_converted_model.py \
   --model-name Qwen3.5-0.8B
 ```
 
-### Qwen3.5-0.8B single-image
+### Qwen3.5-0.8B multimodal
 
 ```bash
 python examples/qwen3_5/validate_checkpoint.py \
@@ -163,6 +164,33 @@ python examples/qwen3_5/validate_converted_model.py \
   --model-name Qwen3.5-4B
 ```
 
+### Qwen3.5-4B multimodal
+
+```bash
+python examples/qwen3_5/validate_checkpoint.py \
+  /path/to/Qwen3.5-4B \
+  --quant-config examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \
+  --runtime-config examples/qwen3_5/config_4B_multimodal_w4a32_kai.json
+
+python -m pymllm.mobile.utils.mllm_convertor \
+  --input_path /path/to/Qwen3.5-4B \
+  --output_path /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \
+  --model_name Qwen3.5-4B-Multimodal \
+  --cfg_path examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \
+  --pipeline w4a32_kai_pipeline \
+  --include_prefix model.language_model. \
+  --include_prefix model.visual. \
+  --format v2 \
+  --verbose
+
+python examples/qwen3_5/validate_converted_model.py \
+  /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \
+  /path/to/Qwen3.5-4B \
+  --quant-config examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \
+  --runtime-config examples/qwen3_5/config_4B_multimodal_w4a32_kai.json \
+  --model-name Qwen3.5-4B-Multimodal
+```
+
 The 4B converted tensor data exceeds 4 GiB. Use model-file V2 so descriptor
 sizes and offsets remain 64-bit, and provide at least 32 GiB of available host
 memory plus working disk space for conversion.
@@ -209,12 +237,23 @@ mllm-qwen3-5-runner \
   --config_path examples/qwen3_5/config_4B_w4a32_kai.json \
   --prompt "Give a one-sentence introduction." \
   --max_new_tokens 32
+
+# Qwen3.5-4B multimodal; repeat --image_path for ordered multi-image input,
+# or replace it with --video_path in a decoder-enabled build.
+mllm-qwen3-5-runner \
+  --model_path /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \
+  --model_version v2 \
+  --tokenizer_path /path/to/Qwen3.5-4B/tokenizer.json \
+  --config_path examples/qwen3_5/config_4B_multimodal_w4a32_kai.json \
+  --image_path /path/to/image.jpg \
+  --prompt "Describe the image." \
+  --max_new_tokens 32
 ```
 
 Omit `--prompt` for the interactive loop. Repeat `--image_path` to attach
 multiple still images in order; the same ordered image list is used for each
 independent prompt in that process. Omit `--image_path` to run text-only
-inference with either the text-only or multimodal 0.8B model. `--video_path`
+inference with either the text-only or multimodal model of the selected size. `--video_path`
 requires a decoder-enabled build and cannot be combined with `--image_path`.
 Use `--video_fps`, `--video_max_frames`, `--video_max_bytes`,
 `--video_max_tokens`, `--video_max_decoded_pixels`, and

examples/qwen3_5/config_4B_multimodal_w4a32_kai.json

@@ -0,0 +1,97 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "model_type": "qwen3_5",
+  "image_token_id": 248056,
+  "video_token_id": 248057,
+  "vision_start_token_id": 248053,
+  "vision_end_token_id": 248054,
+  "image_min_pixels": 65536,
+  "image_max_pixels": 262144,
+  "text_config": {
+    "attention_bias": false,
+    "attn_output_gate": true,
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "intermediate_size": 9216,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 4,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "vocab_size": 248320
+  },
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 24,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1024,
+    "in_channels": 3,
+    "intermediate_size": 4096,
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 2560,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "tie_word_embeddings": true,
+  "max_cache_length": 2048,
+  "linear_impl_type": "KaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_qai8dxp1x8_qsi4c32p8x8_1x8x32"
+}

examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json

@@ -0,0 +1,21 @@
+{
+  "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.q_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [8192, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.k_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.v_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.o_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_qkv\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [8192, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_z\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_a\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [32, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_b\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [32, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.out_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.gate_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [9216, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.up_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [9216, 2560], "replace": true}},
+  "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.down_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 9216], "replace": true}},
+  "^model\\.language_model\\.embed_tokens\\.weight$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [248320, 2560], "replace": false, "rename": "lm_head_out.weight"}},
+  "^model\\.visual\\.blocks\\.\\d+\\.attn\\.qkv\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [3072, 1024], "replace": true}},
+  "^model\\.visual\\.blocks\\.\\d+\\.attn\\.proj\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 1024], "replace": true}},
+  "^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc1\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 1024], "replace": true}},
+  "^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 4096], "replace": true}},
+  "^model\\.visual\\.merger\\.linear_fc1\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 4096], "replace": true}},
+  "^model\\.visual\\.merger\\.linear_fc2\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}}
+}

examples/qwen3_5/test_validators.py

@@ -161,6 +161,35 @@ def test_official_08b_multimodal_recipe_and_descriptors(self) -> None:
             103,
         )
 
+    def test_official_4b_multimodal_recipe_and_descriptors(self) -> None:
+        config = _load_json("config_4B_multimodal_w4a32_kai.json")
+        quant_config = _load_json("quant_cfg_4B_multimodal_w4a32_kai.json")
+
+        checkpoint_config = copy.deepcopy(config)
+        checkpoint_config["vision_config"].update(
+            {"initializer_range": 0.02, "model_type": "qwen3_5"}
+        )
+        validate_multimodal_config_contract(checkpoint_config, config)
+        quantized_names, _ = validate_kai_recipe_contract(
+            checkpoint_config["text_config"],
+            quant_config,
+            config,
+            checkpoint_config["vision_config"],
+        )
+        self.assertEqual(len(quantized_names), 445)
+        self.assertEqual(len(expected_vision_shapes(config["vision_config"])), 297)
+
+        expected = _expected_descriptors(config, quant_config)
+        dtype_counts = Counter(dtype for dtype, _, _ in expected.values())
+        self.assertEqual(len(expected), 626)
+        self.assertEqual(dtype_counts[BYTE], 347)
+        self.assertEqual(dtype_counts[FLOAT32], 279)
+        self.assertEqual(
+            sum(name.startswith("model.visual.") for name in expected),
+            199,
+        )
+        self.assertEqual(sum(size for _, _, size in expected.values()), 5_126_041_600)
+
     def test_multimodal_recipe_rejects_text_runtime_and_contract_drift(self) -> None:
         config = _load_json("config_0.8B_multimodal_w4a32_kai.json")
         text_runtime = _load_json("config_0.8B_w4a32_kai.json")
@@ -179,6 +208,17 @@ def test_multimodal_recipe_rejects_text_runtime_and_contract_drift(self) -> None
         with self.assertRaisesRegex(AssertionError, "image_max_pixels"):
             validate_multimodal_config_contract(config, mutated)
 
+        config_4b = _load_json("config_4B_multimodal_w4a32_kai.json")
+        wrong_vision = copy.deepcopy(config_4b)
+        wrong_vision["vision_config"]["depth"] = 12
+        with self.assertRaisesRegex(
+            AssertionError, "official Qwen3.5-4B"
+        ):
+            validate_multimodal_config_contract(config_4b, wrong_vision)
+
+        with self.assertRaisesRegex(AssertionError, "runtime is Qwen3.5-0.8B"):
+            validate_multimodal_config_contract(config_4b, config)
+
     def test_variant_resolution_rejects_runtime_semantic_mismatches(self) -> None:
         text_config = _load_json("config_4B_w4a32_kai.json")["text_config"]
         mutations = (