From 9317fb3a00ecd27707071e3648790e01f9ed2c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 14:03:01 +0100 Subject: [PATCH 1/9] fix documentation --- _doc/status/patches_diff.rst | 6 ++--- onnx_diagnostic/torch_models/validate.py | 28 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/_doc/status/patches_diff.rst b/_doc/status/patches_diff.rst index 75576ed4..50d5b743 100644 --- a/_doc/status/patches_diff.rst +++ b/_doc/status/patches_diff.rst @@ -33,8 +33,8 @@ helps fixing some issues for many models. Class :class:`PatchDetails ` gives an example on how to retrieve the list of involded patches for a specific model. -Those patches belongs to the following list which depends on transformers and -pytorch versions. +Those patches belong to the following list which depends on :epkg:`transformers`` and +:epkg:`pytorch` versions. .. runpython:: :showcode: @@ -62,7 +62,7 @@ Those two versions leads to the following list of patches. ): pass for patch in details.patched: - print(f"* {patch.family} - {patch.function_to_patch}") + print(f"* {patch.family} - {getattr(patch.function_to_patch, '__name__', patch.function_to_patch)}") print() print() for patch in details.patched: diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index 6b2f48bd..39dfc68b 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2449,6 +2449,34 @@ def call_torch_export_custom( ) ), ) + if "optimization" in opt_stats: + import pandas + + pattern_stats = [] + for k, v in opt_stats.items(): + if "time" in k: + pattern_stats.append(dict(level="main", pattern=k, time_in=v)) + pattern_stats.extend( + [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]] + ) + stat_filename = os.path.join(dump_folder, "optimization_stats.xlsx") + df = pandas.DataFrame(pattern_stats) + df.to_excel(stat_filename, index=False) + cols = [ + "level", + "pattern", + "time_in", + "iteration", + "inlined", + "removed", + "added", + "instances", + ] + agg = {k: "sum" for k in cols if k not in ("level", "pattern")} + agg.update(dict(iteration="max", instances="mean")) + stat_filename = os.path.join(dump_folder, "optimization_stats.agg.xlsx") + df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename) + if "ERR_export_onnx_c" in summary: return summary, data From d2801c20267b3ee0c5e114542644bdbb3e99eac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 14:04:02 +0100 Subject: [PATCH 2/9] dump --- onnx_diagnostic/torch_models/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index 39dfc68b..fb711ab9 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2449,7 +2449,7 @@ def call_torch_export_custom( ) ), ) - if "optimization" in opt_stats: + if "optimization" in opt_stats and dump_folder: import pandas pattern_stats = [] From a9c69cf15c0a5dd0a7b270fef62866b728edaab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 14:20:32 +0100 Subject: [PATCH 3/9] fix --- codecov.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 5df0f7d2..7c548ce7 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,3 @@ ignore: - - onnx_diagnostic/big_models/* - - onnx_diagnostic/ci_models/export* + - "onnx_diagnostic/big_models/*" + - "onnx_diagnostic/ci_models/export*" From 39120114f58b3d443a2fd138e7dab50b7fedbea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 14:47:34 +0100 Subject: [PATCH 4/9] fix validate --- onnx_diagnostic/torch_models/validate.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index fb711ab9..e89ade04 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2463,14 +2463,20 @@ def call_torch_export_custom( df = pandas.DataFrame(pattern_stats) df.to_excel(stat_filename, index=False) cols = [ - "level", - "pattern", - "time_in", - "iteration", - "inlined", - "removed", - "added", - "instances", + c + for c in [ + "level", + "pattern", + "time_in", + "iteration", + "inlined", + "removed", + "added", + "instances", + "changed", + "scale", + ] + if c in df.columns ] agg = {k: "sum" for k in cols if k not in ("level", "pattern")} agg.update(dict(iteration="max", instances="mean")) From fe4c356e71fd9ec96f6028cd75b225719d0d36e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 15:04:00 +0100 Subject: [PATCH 5/9] fix aggregation --- onnx_diagnostic/torch_models/validate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index e89ade04..748e312c 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2480,6 +2480,7 @@ def call_torch_export_custom( ] agg = {k: "sum" for k in cols if k not in ("level", "pattern")} agg.update(dict(iteration="max", instances="mean")) + agg = {k: v for k, v in agg.items() if k in df.columns} stat_filename = os.path.join(dump_folder, "optimization_stats.agg.xlsx") df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename) From d92e136b6eddcb3e7cbcb77f50e0fab782f0ad09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 15:36:47 +0100 Subject: [PATCH 6/9] documentation --- _doc/status/patches_diff.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_doc/status/patches_diff.rst b/_doc/status/patches_diff.rst index 50d5b743..c2b84241 100644 --- a/_doc/status/patches_diff.rst +++ b/_doc/status/patches_diff.rst @@ -33,7 +33,7 @@ helps fixing some issues for many models. Class :class:`PatchDetails ` gives an example on how to retrieve the list of involded patches for a specific model. -Those patches belong to the following list which depends on :epkg:`transformers`` and +Those patches belong to the following list which depends on :epkg:`transformers` and :epkg:`pytorch` versions. .. runpython:: From 21f2bc5e13fbfeaf8cae0c0d5f866d6b25af93e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 15:21:20 +0000 Subject: [PATCH 7/9] output statistics --- onnx_diagnostic/export/api.py | 42 ++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/onnx_diagnostic/export/api.py b/onnx_diagnostic/export/api.py index d57ce3bf..5bf1e0f8 100644 --- a/onnx_diagnostic/export/api.py +++ b/onnx_diagnostic/export/api.py @@ -1,3 +1,4 @@ +import os from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import torch from .onnx_plug import EagerDirectReplacementWithOnnx @@ -138,7 +139,7 @@ def to_onnx( else None ) - return _to_onnx( + proto, opt_stats = _to_onnx( mod, args=args, kwargs=kwargs, @@ -155,11 +156,47 @@ def to_onnx( inline=inline, dispatcher=main_dispatcher, optimize=optimize, + return_optimize_report=True, **(exporter_kwargs or {}), ) + if opt_stats and os.path.exists(filename): + import pandas + + stat_filename = f"{os.path.splitext(filename)[0]}.opt.xlsx" + pattern_stats = [] + for k, v in opt_stats.items(): + if "time" in k: + pattern_stats.append(dict(level="main", pattern=k, time_in=v)) + pattern_stats.extend( + [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]] + ) + df = pandas.DataFrame(pattern_stats) + df.to_excel(stat_filename, index=False) + cols = [ + c + for c in [ + "level", + "pattern", + "time_in", + "iteration", + "inlined", + "removed", + "added", + "instances", + "changed", + "scale", + ] + if c in df.columns + ] + agg = {k: "sum" for k in cols if k not in ("level", "pattern")} + agg.update(dict(iteration="max", instances="mean")) + agg = {k: v for k, v in agg.items() if k in df.columns} + stat_filename = f"{os.path.splitext(filename)[0]}.opt.agg.xlsx" + df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename) + + return proto if exporter in ("dynamo", "onnx-dynamo"): - import os from ..helpers import flatten_object import onnxscript.rewriter.ort_fusions as ort_fusions @@ -226,7 +263,6 @@ def to_onnx( return epo if exporter == "modelbuilder": - import os from ..helpers import flatten_object, string_type from ..helpers.model_builder_helper import create_model_builder, save_model_builder From aee39f230540dec0611a1d4aaa87aa1c40043e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 16:29:37 +0100 Subject: [PATCH 8/9] fix mypy --- onnx_diagnostic/export/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnx_diagnostic/export/api.py b/onnx_diagnostic/export/api.py index 5bf1e0f8..57215dcc 100644 --- a/onnx_diagnostic/export/api.py +++ b/onnx_diagnostic/export/api.py @@ -159,7 +159,7 @@ def to_onnx( return_optimize_report=True, **(exporter_kwargs or {}), ) - if opt_stats and os.path.exists(filename): + if opt_stats and filename and os.path.exists(filename): import pandas stat_filename = f"{os.path.splitext(filename)[0]}.opt.xlsx" From c4846e0656739ead0916025b58345e9bb5653dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 19 Dec 2025 17:23:09 +0100 Subject: [PATCH 9/9] add timeout --- .github/workflows/models448.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/models448.yml b/.github/workflows/models448.yml index 889bea1f..486c5e32 100644 --- a/.github/workflows/models448.yml +++ b/.github/workflows/models448.yml @@ -12,6 +12,8 @@ jobs: run: name: to-${{ matrix.torch }}-tr-${{ matrix.transformers }}-ci ${{ matrix.os }}-${{ matrix.python }} runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: fail-fast: false matrix: @@ -64,5 +66,6 @@ jobs: run: python -m pip freeze - name: Phi-4-multimodal-instruct - vision + continue-on-error: true run: | PYTHONPATH=. python -m onnx_diagnostic.ci_models.export_phi4_mm -m microsoft/Phi-4-multimodal-instruct --device cpu --dtype float16 --exporter custom --no-pretrained --no-second-input --atol 100000164640 --mismatch01 1 --part vision