|
21 | 21 | from collections import OrderedDict |
22 | 22 | from pathlib import Path |
23 | 23 | from tempfile import gettempdir |
| 24 | +from types import SimpleNamespace |
24 | 25 | from typing import Any, Dict, List, Optional, Tuple, Union |
25 | 26 |
|
26 | 27 | import numpy as np |
|
126 | 127 | else: |
127 | 128 | CacheMixin = object |
128 | 129 |
|
| 130 | +if is_diffusers_version(">=", "0.37.0"): |
| 131 | + from diffusers import Flux2KleinPipeline |
| 132 | +else: |
| 133 | + Flux2KleinPipeline = object |
| 134 | + |
129 | 135 | DIFFUSION_MODEL_TRANSFORMER_SUBFOLDER = "transformer" |
130 | 136 | DIFFUSION_MODEL_TEXT_ENCODER_3_SUBFOLDER = "text_encoder_3" |
131 | 137 |
|
@@ -764,6 +770,7 @@ def _reshape_transformer( |
764 | 770 | batch_size *= 2 |
765 | 771 |
|
766 | 772 | is_ltx = self.__class__.__name__.startswith("OVLTX") |
| 773 | + is_flux2 = self.__class__.__name__.startswith("OVFlux2") |
767 | 774 | if is_ltx: |
768 | 775 | height = height // self.vae_spatial_compression_ratio if height > 0 else -1 |
769 | 776 | width = width // self.vae_spatial_compression_ratio if width > 0 else -1 |
@@ -804,8 +811,12 @@ def _reshape_transformer( |
804 | 811 | if is_diffusers_version("<", "0.31.0") |
805 | 812 | else [packed_height_width, 3] |
806 | 813 | ) |
| 814 | + if is_flux2: |
| 815 | + shapes[inputs] = [batch_size, packed_height_width, 4] |
807 | 816 | elif inputs.get_any_name() == "txt_ids": |
808 | 817 | shapes[inputs] = [batch_size, -1, 3] if is_diffusers_version("<", "0.31.0") else [-1, 3] |
| 818 | + if is_flux2: |
| 819 | + shapes[inputs] = [batch_size, -1, 4] |
809 | 820 | elif inputs.get_any_name() in ["height", "width", "num_frames", "rope_interpolation_scale"]: |
810 | 821 | shapes[inputs] = inputs.get_partial_shape() |
811 | 822 | else: |
@@ -1163,6 +1174,7 @@ def forward( |
1163 | 1174 | attention_mask: Optional[Union[np.ndarray, torch.Tensor]] = None, |
1164 | 1175 | output_hidden_states: Optional[bool] = None, |
1165 | 1176 | return_dict: bool = False, |
| 1177 | + **kwargs, |
1166 | 1178 | ): |
1167 | 1179 | self.compile() |
1168 | 1180 | model_inputs = {"input_ids": input_ids} |
@@ -1278,6 +1290,7 @@ def forward( |
1278 | 1290 | model_inputs["pooled_projections"] = pooled_projections |
1279 | 1291 | if img_ids is not None: |
1280 | 1292 | model_inputs["img_ids"] = img_ids |
| 1293 | + |
1281 | 1294 | if txt_ids is not None: |
1282 | 1295 | model_inputs["txt_ids"] = txt_ids |
1283 | 1296 | if guidance is not None: |
@@ -1403,6 +1416,18 @@ def __init__(self, decoder: OVModelVaeDecoder, encoder: OVModelVaeEncoder): |
1403 | 1416 | self.latents_mean = torch.tensor(self.decoder.config.latents_mean_data) |
1404 | 1417 | if hasattr(self.decoder.config, "latents_std_data"): |
1405 | 1418 | self.latents_std = torch.tensor(self.decoder.config.latents_std_data) |
| 1419 | + # Flux2Klein compatibility: pipeline expects self.vae.bn.running_mean/running_var/eps |
| 1420 | + self.bn = None |
| 1421 | + bn_mean = getattr(self.decoder.config, "bn_running_mean_data", None) |
| 1422 | + bn_var = getattr(self.decoder.config, "bn_running_var_data", None) |
| 1423 | + bn_eps = float(getattr(self.decoder.config, "bn_eps", 1e-5)) |
| 1424 | + |
| 1425 | + if bn_mean is not None and bn_var is not None: |
| 1426 | + self.bn = SimpleNamespace( |
| 1427 | + running_mean=torch.tensor(bn_mean, dtype=torch.float32), |
| 1428 | + running_var=torch.tensor(bn_var, dtype=torch.float32), |
| 1429 | + eps=bn_eps, |
| 1430 | + ) |
1406 | 1431 |
|
1407 | 1432 | @property |
1408 | 1433 | def _component_names(self) -> List[str]: |
@@ -1627,6 +1652,25 @@ class OVFluxPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, FluxPip |
1627 | 1652 | auto_model_class = FluxPipeline |
1628 | 1653 |
|
1629 | 1654 |
|
| 1655 | +class OVFlux2KleinPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, Flux2KleinPipeline): |
| 1656 | + main_input_name = "prompt" |
| 1657 | + export_feature = "text-to-image" |
| 1658 | + auto_model_class = Flux2KleinPipeline |
| 1659 | + |
| 1660 | + def __call__(self, *args, **kwargs): |
| 1661 | + if args and isinstance(args[0], str) and "prompt" not in kwargs: |
| 1662 | + user_prompt = args[0] |
| 1663 | + raise ValueError( |
| 1664 | + f"`prompt` must be passed as a keyword argument for Flux2Klein pipelines because the first positional " |
| 1665 | + f"argument is `image`. Use `pipeline(prompt={user_prompt!r})`." |
| 1666 | + ) |
| 1667 | + |
| 1668 | + if "guidance_scale" in kwargs and kwargs["guidance_scale"] is not None and kwargs["guidance_scale"] != 1.0: |
| 1669 | + kwargs["guidance_scale"] = 1.0 |
| 1670 | + |
| 1671 | + return super().__call__(*args, **kwargs) |
| 1672 | + |
| 1673 | + |
1630 | 1674 | class OVFluxImg2ImgPipeline(OVDiffusionPipeline, OVTextualInversionLoaderMixin, FluxImg2ImgPipeline): |
1631 | 1675 | main_input_name = "image" |
1632 | 1676 | export_feature = "image-to-image" |
@@ -1748,6 +1792,11 @@ def _get_ov_class(pipeline_class_name: str, throw_error_if_not_exist: bool = Tru |
1748 | 1792 | SUPPORTED_OV_PIPELINES.append(OVSanaSprintPipeline) |
1749 | 1793 | OV_TEXT2IMAGE_PIPELINES_MAPPING["sana-sprint"] = OVSanaSprintPipeline |
1750 | 1794 |
|
| 1795 | +if is_diffusers_version(">=", "0.37.0"): |
| 1796 | + SUPPORTED_OV_PIPELINES.append(OVFlux2KleinPipeline) |
| 1797 | + OV_TEXT2IMAGE_PIPELINES_MAPPING["flux2-klein"] = OVFlux2KleinPipeline |
| 1798 | + OV_IMAGE2IMAGE_PIPELINES_MAPPING["flux2-klein"] = OVFlux2KleinPipeline |
| 1799 | + |
1751 | 1800 | SUPPORTED_OV_PIPELINES_MAPPINGS = [ |
1752 | 1801 | OV_TEXT2IMAGE_PIPELINES_MAPPING, |
1753 | 1802 | OV_IMAGE2IMAGE_PIPELINES_MAPPING, |
|
0 commit comments