Skip to content

Commit 3517891

Browse files
openvino-agentosabnisCopilotrkazantsclaude
authored
[OpenVINO] Support FLUX.2 Models (#1809)
Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com> Co-authored-by: Sabnis, Omkar <omkar.sabnis@intel.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Kazantsev, Roman <roman.kazantsev@intel.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 671dfbf commit 3517891

13 files changed

Lines changed: 195 additions & 16 deletions

File tree

‎docs/source/openvino/models.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ Here is the list of the supported architectures :
181181
- Latent Consistency
182182
- Stable Diffusion 3
183183
- Flux
184+
- Flux.2
184185
- Sana
185186
- SanaSprint
186187
- LTX

‎optimum/exporters/openvino/convert.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1339,12 +1339,17 @@ def get_flux_models_for_export(pipeline, exporter, int_dtype, float_dtype):
13391339
# Text encoder
13401340
text_encoder = getattr(pipeline, "text_encoder", None)
13411341
if text_encoder is not None:
1342+
text_encoder.config.model_type
1343+
model_type = "clip-text"
1344+
if text_encoder.config.model_type in ["qwen3"]:
1345+
model_type = "qwen3-text-encoder"
1346+
13421347
text_encoder_config_constructor = TasksManager.get_exporter_config_constructor(
13431348
model=text_encoder,
13441349
exporter=exporter,
13451350
library_name="diffusers",
13461351
task="feature-extraction",
1347-
model_type="clip-text",
1352+
model_type=model_type,
13481353
)
13491354
text_encoder_export_config = text_encoder_config_constructor(
13501355
pipeline.text_encoder.config, int_dtype=int_dtype, float_dtype=float_dtype
@@ -1355,6 +1360,10 @@ def get_flux_models_for_export(pipeline, exporter, int_dtype, float_dtype):
13551360
transformer.config.text_encoder_projection_dim = transformer.config.joint_attention_dim
13561361
transformer.config.requires_aesthetics_score = getattr(pipeline.config, "requires_aesthetics_score", False)
13571362
transformer.config.time_cond_proj_dim = None
1363+
1364+
if not hasattr(transformer.config, "pooled_projection_dim") and hasattr(transformer.config, "joint_attention_dim"):
1365+
transformer.config.pooled_projection_dim = transformer.config.joint_attention_dim
1366+
13581367
export_config_constructor = TasksManager.get_exporter_config_constructor(
13591368
model=transformer,
13601369
exporter=exporter,
@@ -1370,6 +1379,12 @@ def get_flux_models_for_export(pipeline, exporter, int_dtype, float_dtype):
13701379

13711380
# VAE Encoder https://github.com/huggingface/diffusers/blob/v0.11.1/src/diffusers/models/vae.py#L565
13721381
vae_encoder = copy.deepcopy(pipeline.vae)
1382+
# vae_scaling_factor is used at inference to scale latents
1383+
vae_scaling_factor = None
1384+
if hasattr(vae_encoder, "config") and getattr(vae_encoder.config, "scaling_factor", None) is not None:
1385+
vae_scaling_factor = float(vae_encoder.config.scaling_factor)
1386+
vae_encoder.register_to_config(scaling_factor=vae_scaling_factor)
1387+
13731388
vae_encoder.forward = lambda sample: {"latent_parameters": vae_encoder.encode(x=sample)["latent_dist"].parameters}
13741389
vae_config_constructor = TasksManager.get_exporter_config_constructor(
13751390
model=vae_encoder,
@@ -1386,6 +1401,22 @@ def get_flux_models_for_export(pipeline, exporter, int_dtype, float_dtype):
13861401

13871402
# VAE Decoder https://github.com/huggingface/diffusers/blob/v0.11.1/src/diffusers/models/vae.py#L600
13881403
vae_decoder = copy.deepcopy(pipeline.vae)
1404+
if vae_scaling_factor is not None:
1405+
vae_decoder.register_to_config(scaling_factor=float(vae_scaling_factor))
1406+
# The transformer operates on normalized latents
1407+
# Before the VAE decoder can reconstruct pixels, the latents must be denormalized back
1408+
if (
1409+
hasattr(vae_decoder, "bn")
1410+
and hasattr(vae_decoder.bn, "running_mean")
1411+
and hasattr(vae_decoder.bn, "running_var")
1412+
):
1413+
vae_decoder.register_to_config(
1414+
**{
1415+
"bn_running_mean_data": vae_decoder.bn.running_mean.detach().cpu().tolist(),
1416+
"bn_running_var_data": vae_decoder.bn.running_var.detach().cpu().tolist(),
1417+
"bn_eps": float(getattr(vae_decoder.bn, "eps", 1e-5)),
1418+
}
1419+
)
13891420
vae_decoder.forward = lambda latent_sample: vae_decoder.decode(z=latent_sample)
13901421
vae_config_constructor = TasksManager.get_exporter_config_constructor(
13911422
model=vae_decoder,

‎optimum/exporters/openvino/input_generators.py‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,13 @@ def __init__(
592592
):
593593
self.task = task
594594
self.batch_size = batch_size
595-
self.pooled_projection_dim = normalized_config.config.pooled_projection_dim
595+
config = normalized_config.config
596+
pooled_projection_dim = getattr(config, "pooled_projection_dim", None)
597+
# FrozenDict / dict requires get() method for attribute access
598+
# getattr() will just return None
599+
if pooled_projection_dim is None and hasattr(config, "get"):
600+
pooled_projection_dim = config.get("pooled_projection_dim", None)
601+
self.pooled_projection_dim = pooled_projection_dim
596602

597603
def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
598604
shape = [self.batch_size, self.pooled_projection_dim]
@@ -728,6 +734,8 @@ def __init__(
728734
super().__init__(task, normalized_config, batch_size, num_channels, width, height, **kwargs)
729735
if getattr(normalized_config, "in_channels", None):
730736
self.num_channels = normalized_config.in_channels // 4
737+
self.config = normalized_config.config
738+
self.is_flux2 = self.config.get("_class_name", "") == "Flux2Transformer2DModel"
731739

732740
def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
733741
if input_name in ["hidden_states", "sample"]:
@@ -736,12 +744,16 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
736744
if input_name == "img_ids":
737745
img_ids_height = self.height // 2
738746
img_ids_width = self.width // 2
747+
img_ids_shape = [self.batch_size, img_ids_height * img_ids_width, 3]
748+
img_ids_shape = (
749+
[self.batch_size, img_ids_height * img_ids_width, 3]
750+
if is_diffusers_version("<", "0.31.0")
751+
else [img_ids_height * img_ids_width, 3]
752+
)
753+
if self.is_flux2:
754+
img_ids_shape = [self.batch_size, img_ids_height * img_ids_width, 4]
739755
return self.random_int_tensor(
740-
(
741-
[self.batch_size, img_ids_height * img_ids_width, 3]
742-
if is_diffusers_version("<", "0.31.0")
743-
else [img_ids_height * img_ids_width, 3]
744-
),
756+
img_ids_shape,
745757
min_value=0,
746758
max_value=min(img_ids_height, img_ids_width),
747759
framework=framework,
@@ -764,11 +776,14 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
764776
if input_name == "txt_ids":
765777
import torch
766778

779+
is_flux2 = self.normalized_config.config.get("_class_name", "") == "Flux2Transformer2DModel"
767780
shape = (
768781
[self.batch_size, self.sequence_length, 3]
769782
if is_diffusers_version("<", "0.31.0")
770783
else [self.sequence_length, 3]
771784
)
785+
if is_flux2:
786+
shape = [self.batch_size, self.sequence_length, 4]
772787
dtype = DTYPE_MAPPER.pt(float_dtype)
773788
return torch.full(shape, 0, dtype=dtype)
774789
return super().generate(input_name, framework, int_dtype, float_dtype)

‎optimum/exporters/openvino/model_configs.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2314,6 +2314,35 @@ def inputs(self) -> Dict[str, Dict[int, str]]:
23142314
}
23152315

23162316

2317+
@register_in_tasks_manager("qwen3-text-encoder", *["feature-extraction"], library_name="diffusers")
2318+
class Qwen3TextEncoderOpenVINOConfig(CLIPTextOpenVINOConfig):
2319+
@property
2320+
def inputs(self) -> Dict[str, Dict[int, str]]:
2321+
return {
2322+
"input_ids": {0: "batch_size", 1: "sequence_length"},
2323+
"attention_mask": {0: "batch_size", 1: "sequence_length"},
2324+
}
2325+
2326+
@property
2327+
def outputs(self) -> Dict[str, Dict[int, str]]:
2328+
common_outputs = {"last_hidden_state": {0: "batch_size", 1: "sequence_length"}}
2329+
2330+
num_layers = getattr(self._normalized_config, "num_layers", None)
2331+
if num_layers is None:
2332+
num_layers = getattr(self._normalized_config, "num_hidden_layers", 0)
2333+
2334+
for i in range(int(num_layers) + 1):
2335+
common_outputs[f"hidden_states.{i}"] = {0: "batch_size", 1: "sequence_length"}
2336+
2337+
return common_outputs
2338+
2339+
@property
2340+
def values_override(self) -> Optional[Dict[str, Any]]:
2341+
values = super().values_override or {}
2342+
values.update({"output_hidden_states": True, "return_dict": True, "use_cache": False})
2343+
return values
2344+
2345+
23172346
@register_in_tasks_manager("sana-transformer", *["semantic-segmentation"], library_name="diffusers")
23182347
class SanaTransformerOpenVINOConfig(UNetOpenVINOConfig):
23192348
NORMALIZED_CONFIG_CLASS = NormalizedConfig.with_args(
@@ -2446,8 +2475,19 @@ class FluxTransformerOpenVINOConfig(SD3TransformerOpenVINOConfig):
24462475

24472476
@property
24482477
def inputs(self):
2478+
self.is_flux2 = self._config.get("_class_name", "") == "Flux2Transformer2DModel"
2479+
24492480
common_inputs = super().inputs
24502481
common_inputs.pop("sample", None)
2482+
2483+
config = self._normalized_config.config
2484+
pooled_projection_dim = config.get("pooled_projection_dim", None)
2485+
2486+
# FLUX.2 (klein): replaced CLIP+T5 with a single Qwen3 encoder
2487+
# Qwen3 produces sequence embeddings but no pooled embedding
2488+
if pooled_projection_dim is None:
2489+
common_inputs.pop("pooled_projections", None)
2490+
24512491
common_inputs["hidden_states"] = {0: "batch_size", 1: "packed_height_width"}
24522492
common_inputs["txt_ids"] = (
24532493
{0: "batch_size", 1: "sequence_length"} if is_diffusers_version("<", "0.31.0") else {0: "sequence_length"}

‎optimum/intel/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
"OVLatentConsistencyModelImg2ImgPipeline",
106106
"OVLTXPipeline",
107107
"OVFluxPipeline",
108+
"OVFlux2KleinPipeline",
108109
"OVFluxImg2ImgPipeline",
109110
"OVFluxInpaintPipeline",
110111
"OVFluxFillPipeline",
@@ -132,6 +133,7 @@
132133
"OVLatentConsistencyModelImg2ImgPipeline",
133134
"OVLTXPipeline",
134135
"OVFluxPipeline",
136+
"OVFlux2KleinPipeline",
135137
"OVFluxImg2ImgPipeline",
136138
"OVFluxInpaintPipeline",
137139
"OVFluxFillPipeline",

‎optimum/intel/openvino/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
if is_diffusers_available():
9696
from .modeling_diffusion import (
9797
OVDiffusionPipeline,
98+
OVFlux2KleinPipeline,
9899
OVFluxFillPipeline,
99100
OVFluxImg2ImgPipeline,
100101
OVFluxInpaintPipeline,

‎optimum/intel/openvino/modeling_diffusion.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from collections import OrderedDict
2222
from pathlib import Path
2323
from tempfile import gettempdir
24+
from types import SimpleNamespace
2425
from typing import Any, Dict, List, Optional, Tuple, Union
2526

2627
import numpy as np
@@ -126,6 +127,11 @@
126127
else:
127128
CacheMixin = object
128129

130+
if is_diffusers_version(">=", "0.37.0"):
131+
from diffusers import Flux2KleinPipeline
132+
else:
133+
Flux2KleinPipeline = object
134+
129135
DIFFUSION_MODEL_TRANSFORMER_SUBFOLDER = "transformer"
130136
DIFFUSION_MODEL_TEXT_ENCODER_3_SUBFOLDER = "text_encoder_3"
131137

@@ -764,6 +770,7 @@ def _reshape_transformer(
764770
batch_size *= 2
765771

766772
is_ltx = self.__class__.__name__.startswith("OVLTX")
773+
is_flux2 = self.__class__.__name__.startswith("OVFlux2")
767774
if is_ltx:
768775
height = height // self.vae_spatial_compression_ratio if height > 0 else -1
769776
width = width // self.vae_spatial_compression_ratio if width > 0 else -1
@@ -804,8 +811,12 @@ def _reshape_transformer(
804811
if is_diffusers_version("<", "0.31.0")
805812
else [packed_height_width, 3]
806813
)
814+
if is_flux2:
815+
shapes[inputs] = [batch_size, packed_height_width, 4]
807816
elif inputs.get_any_name() == "txt_ids":
808817
shapes[inputs] = [batch_size, -1, 3] if is_diffusers_version("<", "0.31.0") else [-1, 3]
818+
if is_flux2:
819+
shapes[inputs] = [batch_size, -1, 4]
809820
elif inputs.get_any_name() in ["height", "width", "num_frames", "rope_interpolation_scale"]:
810821
shapes[inputs] = inputs.get_partial_shape()
811822
else:
@@ -1163,6 +1174,7 @@ def forward(
11631174
attention_mask: Optional[Union[np.ndarray, torch.Tensor]] = None,
11641175
output_hidden_states: Optional[bool] = None,
11651176
return_dict: bool = False,
1177+
**kwargs,
11661178
):
11671179
self.compile()
11681180
model_inputs = {"input_ids": input_ids}
@@ -1278,6 +1290,7 @@ def forward(
12781290
model_inputs["pooled_projections"] = pooled_projections
12791291
if img_ids is not None:
12801292
model_inputs["img_ids"] = img_ids
1293+
12811294
if txt_ids is not None:
12821295
model_inputs["txt_ids"] = txt_ids
12831296
if guidance is not None:
@@ -1403,6 +1416,18 @@ def __init__(self, decoder: OVModelVaeDecoder, encoder: OVModelVaeEncoder):
14031416
self.latents_mean = torch.tensor(self.decoder.config.latents_mean_data)
14041417
if hasattr(self.decoder.config, "latents_std_data"):
14051418
self.latents_std = torch.tensor(self.decoder.config.latents_std_data)
1419+
# Flux2Klein compatibility: pipeline expects self.vae.bn.running_mean/running_var/eps
1420+
self.bn = None
1421+
bn_mean = getattr(self.decoder.config, "bn_running_mean_data", None)
1422+
bn_var = getattr(self.decoder.config, "bn_running_var_data", None)
1423+
bn_eps = float(getattr(self.decoder.config, "bn_eps", 1e-5))
1424+
1425+
if bn_mean is not None and bn_var is not None:
1426+
self.bn = SimpleNamespace(
1427+
running_mean=torch.tensor(bn_mean, dtype=torch.float32),
1428+
running_var=torch.tensor(bn_var, dtype=torch.float32),
1429+
eps=bn_eps,
1430+
)
14061431

14071432
@property
14081433
def _component_names(self) -> List[str]:
@@ -1627,6 +1652,25 @@ class OVFluxPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, FluxPip
16271652
auto_model_class = FluxPipeline
16281653

16291654

1655+
class OVFlux2KleinPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, Flux2KleinPipeline):
1656+
main_input_name = "prompt"
1657+
export_feature = "text-to-image"
1658+
auto_model_class = Flux2KleinPipeline
1659+
1660+
def __call__(self, *args, **kwargs):
1661+
if args and isinstance(args[0], str) and "prompt" not in kwargs:
1662+
user_prompt = args[0]
1663+
raise ValueError(
1664+
f"`prompt` must be passed as a keyword argument for Flux2Klein pipelines because the first positional "
1665+
f"argument is `image`. Use `pipeline(prompt={user_prompt!r})`."
1666+
)
1667+
1668+
if "guidance_scale" in kwargs and kwargs["guidance_scale"] is not None and kwargs["guidance_scale"] != 1.0:
1669+
kwargs["guidance_scale"] = 1.0
1670+
1671+
return super().__call__(*args, **kwargs)
1672+
1673+
16301674
class OVFluxImg2ImgPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, FluxImg2ImgPipeline):
16311675
main_input_name = "image"
16321676
export_feature = "image-to-image"
@@ -1748,6 +1792,11 @@ def _get_ov_class(pipeline_class_name: str, throw_error_if_not_exist: bool = Tru
17481792
SUPPORTED_OV_PIPELINES.append(OVSanaSprintPipeline)
17491793
OV_TEXT2IMAGE_PIPELINES_MAPPING["sana-sprint"] = OVSanaSprintPipeline
17501794

1795+
if is_diffusers_version(">=", "0.37.0"):
1796+
SUPPORTED_OV_PIPELINES.append(OVFlux2KleinPipeline)
1797+
OV_TEXT2IMAGE_PIPELINES_MAPPING["flux2-klein"] = OVFlux2KleinPipeline
1798+
OV_IMAGE2IMAGE_PIPELINES_MAPPING["flux2-klein"] = OVFlux2KleinPipeline
1799+
17511800
SUPPORTED_OV_PIPELINES_MAPPINGS = [
17521801
OV_TEXT2IMAGE_PIPELINES_MAPPING,
17531802
OV_IMAGE2IMAGE_PIPELINES_MAPPING,

‎optimum/intel/openvino/utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
"sam": "OVSamModel",
139139
"sana": "OVSanaPipeline",
140140
"flux": "OVFluxPipeline",
141+
"flux.2-klein": "OVFlux2KleinPipeline",
141142
"flux-fill": "OVFluxFillPipeline",
142143
"pix2struct": "OVModelForPix2Struct",
143144
"latent-consistency": "OVLatentConsistencyModelPipeline",

‎optimum/intel/utils/dummy_openvino_and_diffusers_objects.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,17 @@ def from_pretrained(cls, *args, **kwargs):
213213
requires_backends(cls, ["openvino", "diffusers"])
214214

215215

216+
class OVFlux2KleinPipeline(metaclass=DummyObject):
217+
_backends = ["openvino", "diffusers"]
218+
219+
def __init__(self, *args, **kwargs):
220+
requires_backends(self, ["openvino", "diffusers"])
221+
222+
@classmethod
223+
def from_pretrained(cls, *args, **kwargs):
224+
requires_backends(cls, ["openvino", "diffusers"])
225+
226+
216227
class OVFluxImg2ImgPipeline(metaclass=DummyObject):
217228
_backends = ["openvino", "diffusers"]
218229

0 commit comments

Comments
 (0)