From 0d147124b5c5ed52af6f3bab8d4d3e9aa7852d2c Mon Sep 17 00:00:00 2001 From: Cortexelus Date: Fri, 31 Jul 2026 04:41:55 +0000 Subject: [PATCH] tensorRT: fix verify_fp8_rope's CHECK 3, which failed on the engine it validates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHECK 3 reported False for the shipped fp8 engine on BOTH architectures, so the documented identity check for that tier has never actually passed. Two separate causes, both about what this metadata can tell you. Its condition was `len(rope_fp32) > 0 and prec_any["FP32"] > 100`. The ">100 FP32 layers" clause describes the fp32-TRUNK recipe. The shipped engine BAKES RoPE's cos/sin as fp32 constants, so there is no fp32 trunk — only the two runtime Fourier chains stay fp32, which is 6 FP32 layers on sm_120 and 16 on sm_90. Neither will ever exceed 100. The other clause tried to prove an internal fp32 island from per-layer dtypes, but those are the dtypes at the FUSED BLOB BOUNDARY and Myelin fuses boundaries differently per arch. Same ONNX, same recipe, same measured fidelity: sm_120 __myl_MaxMinDivReshMulSinCastCosCastConc I/O ['Float','BFloat16'] sm_90 __myl_CastMaxMinDivReshMulCosSinCastCastConc I/O ['BFloat16','BFloat16'] sm_90 absorbed the cast into the blob, so its boundary reads BF16. Opposite verdicts from a check that cannot see inside either blob. "Is the island fp32 internally" is not decidable from this metadata, so the check no longer pretends it is — the boundary dtypes are still printed, labelled informational. CHECK 3 now asserts what IS decidable and does matter: RoPE must be BAKED, i.e. no runtime cos/sin beyond the two Fourier chains (timestep_features, seconds_total), plus some fp32 surviving. If the bake silently failed, RoPE would be computed at runtime in reduced precision and drift at long sequence — the exact bug this recipe exists to prevent. Fidelity belongs to a numerical test, and the check now says so: sweep velocity-cos vs the fp32 engine across t, because calibration and RoPE damage both show up near t=1.0 and are invisible mid-schedule. Also added the diagnostic whose absence cost me the most time: an engine built without ProfilingVerbosity.DETAILED serializes layer names only, the inspector returns strings, and all three checks report False — indistinguishable from "fp8 did not fire". The tool now detects that and says to rebuild with DETAILED, noting that verbosity is metadata-only (measured: identical velocity-cos, speed within 0.7%) so the twin is a valid stand-in for the shipped engine. Verified: True/True/True on DETAILED builds of both arches, with identical counts (176/181 fp8 GEMMs, 96 BF16 fused MHA, 2 cos/sin layers). Still False on a names-only engine, which is correct — unverifiable is not the same as verified. Co-Authored-By: Claude Opus 5 (1M context) --- optimized/tensorRT/scripts/verify_fp8_rope.py | 70 ++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/optimized/tensorRT/scripts/verify_fp8_rope.py b/optimized/tensorRT/scripts/verify_fp8_rope.py index 6986386..ad5f9c2 100644 --- a/optimized/tensorRT/scripts/verify_fp8_rope.py +++ b/optimized/tensorRT/scripts/verify_fp8_rope.py @@ -6,8 +6,11 @@ CHECK 1 fp8 GEMMs fire on the Linears — count gemm-type layers running FP8 E4M3 (~222). CHECK 2 native fused MHA fires — count _gemm_mha_v2 / fmha nodes (expect 96, bf16 or fp16 — NOT 0/unfused, NOT fp8). - CHECK 3 fp32 RoPE-angle island present — Cos/Sin/qk_norm angle-chain layers carry FP32 - AND fp32 survives broadly in the trunk. + CHECK 3 RoPE is BAKED, not computed at runtime — at most 2 cos/sin layers survive + (timestep_features + seconds_total), and some + fp32 remains. Boundary dtypes are reported but + NOT asserted: they differ per arch by Myelin + fusion (see the comment on check3). usage: python verify_fp8_rope.py [label] """ @@ -62,10 +65,31 @@ def main(): print(f"\n{'='*80}\nENGINE: {label}\n path: {path}\n" f" size: {Path(path).stat().st_size/1e9:.2f} GB\n total layers: {len(layers)}") + # An engine built without ProfilingVerbosity.DETAILED serializes layer NAMES only: + # the inspector hands back a list of strings, every per-layer lookup finds nothing, + # and all three checks report False — which reads exactly like "fp8 did not fire" + # when the truth is "nothing was recorded to look at". Say so instead of guessing. + n_dict = sum(1 for ly in layers if isinstance(ly, dict)) + if layers and n_dict == 0: + print(f"\n {'!'*76}") + print(" CANNOT VERIFY: this engine was built without ProfilingVerbosity.DETAILED,") + print(" so it carries layer names only — no per-layer Format/Datatype to inspect.") + print(" The checks below will all report False regardless of what the engine does.") + print(" Rebuild for inspection with:") + print(" cfg = builder.create_builder_config()") + print(" cfg.profiling_verbosity = trt.ProfilingVerbosity.DETAILED") + print(" Verbosity is metadata-only — it changes neither kernel selection nor") + print(" numerics (measured: identical velocity-cos, speed within 0.7%), so the") + print(" DETAILED twin is a valid stand-in for the engine you ship.") + print(f" {'!'*76}") + type_hist = Counter(); gemm_prec = Counter(); prec_any = Counter() gemm_examples = []; mha_nodes = []; mha_examples = []; fp8_layers = 0 rope_fp32 = []; rope_seen = [] MHA_RE = re.compile(r"mha|fmha", re.I) + # timestep_features and seconds_total are the only Fourier chains evaluated at + # runtime; RoPE's own cos/sin are baked constants and must not appear as layers. + MAX_RUNTIME_COSSIN = 2 ROPE_RE = re.compile(r"(cos|sin|q_norm|k_norm|rotary|inv_freq|angle)", re.I) for ly in layers: @@ -125,12 +149,39 @@ def main(): print(f" ({lt}) {nm[:74]}"); [print(f" {r[:86]}") for r in raw] check2 = len(mha_nodes) >= 90 and mha_prec.get("FP8", 0) == 0 - print(f"\n{'#'*80}\nCHECK 3 — FP32 RoPE / qk_norm ISLAND PRESENT?") - print(f" angle/qk_norm-named layers seen: {len(rope_seen)} of which FP32: {len(rope_fp32)}") - for nm in rope_fp32[:6]: - print(f" FP32: {nm}") + # CHECK 3 rewritten 2026-07-31. The old condition was + # len(rope_fp32) > 0 and prec_any["FP32"] > 100 + # and it reported False on BOTH architectures for the engine it exists to validate. + # Two reasons, both about what this metadata can actually tell you: + # + # 1. ">100 FP32 layers" describes the fp32-TRUNK recipe. The shipped engine BAKES + # RoPE's cos/sin as fp32 constants, so there is no fp32 trunk — only the two + # runtime Fourier chains (timestep_features, seconds_total) stay fp32, which is + # 6 FP32 layers on sm_120 and 16 on sm_90. Neither will ever exceed 100. + # + # 2. The per-layer "FP32" here is the dtype at the FUSED BLOB BOUNDARY, not the + # precision of the arithmetic inside it, and Myelin fuses those boundaries + # differently per arch. Measured on the same ONNX and recipe: + # sm_120 __myl_MaxMinDivReshMulSinCastCosCastConc I/O ['Float','BFloat16'] + # sm_90 __myl_CastMaxMinDivReshMulCosSinCastCastConc I/O ['BFloat16','BFloat16'] + # Same engine identity, same measured fidelity, opposite verdicts from a + # name-and-boundary check. So "is the island fp32 inside" is NOT decidable here. + # + # What IS decidable and does matter: RoPE must be BAKED, i.e. no runtime cos/sin + # beyond those two Fourier chains. If the bake silently failed, RoPE would be + # computed at runtime in reduced precision and drift at long sequence — the bug this + # recipe exists to avoid. Fidelity itself belongs to a numerical test, not an + # inspector: sweep velocity-cos vs the fp32 engine across t (calibration and RoPE + # damage both show up at t~1.0, and are invisible mid-schedule). + print(f"\n{'#'*80}\nCHECK 3 — RoPE BAKED (no runtime cos/sin beyond the 2 Fourier chains)?") + print(f" cos/sin-named layers: {len(rope_seen)} (expect <= {MAX_RUNTIME_COSSIN}: " + f"timestep_features + seconds_total)") + for nm in rope_seen[:6]: + print(f" {'FP32' if nm in rope_fp32 else 'reduced-precision boundary'}: {nm}") print(f" precision histogram across ALL layers: {dict(prec_any)}") - check3 = len(rope_fp32) > 0 and prec_any.get("FP32", 0) > 100 + print(f" note: boundary dtypes above are informational — they differ by arch " + f"(Myelin fusion), so they do not decide the island. Use a velocity-cos sweep.") + check3 = len(rope_seen) <= MAX_RUNTIME_COSSIN and prec_any.get("FP32", 0) > 0 out = dict(engine=path, label=label, total_layers=len(layers), gemm_by_precision=dict(gemm_prec), n_gemm=n_gemm, n_gemm_fp8=n_gemm_fp8, @@ -139,10 +190,11 @@ def main(): n_rope_fp32=len(rope_fp32), check1_fp8_gemms=bool(check1), check2_native_mha_fused=bool(check2), - check3_fp32_rope_island=bool(check3)) + n_runtime_cossin=len(rope_seen), + check3_rope_baked=bool(check3)) Path(path).with_suffix(".verify.json").write_text(json.dumps(out, indent=2)) print(f"\n SUMMARY: CHECK1(fp8 gemms>=170)={check1} " - f"CHECK2(mha fused, no fp8)={check2} CHECK3(fp32 island)={check3}") + f"CHECK2(mha fused, no fp8)={check2} CHECK3(rope baked)={check3}") print(f" wrote {Path(path).with_suffix('.verify.json')}")