diff --git a/.claude/skills/add-recipe/SKILL.md b/.claude/skills/add-recipe/SKILL.md index ad8174c9..b2cd2a21 100644 --- a/.claude/skills/add-recipe/SKILL.md +++ b/.claude/skills/add-recipe/SKILL.md @@ -147,6 +147,7 @@ variants: precision: fp8 vram_minimum_gb: description: "..." + supported_hardware: [mi355x] # optional exact hardware-profile allowlist extra_args: [] extra_env: {} diff --git a/models/MiniMaxAI/MiniMax-M3.yaml b/models/MiniMaxAI/MiniMax-M3.yaml index 5acc99a6..939da7be 100644 --- a/models/MiniMaxAI/MiniMax-M3.yaml +++ b/models/MiniMaxAI/MiniMax-M3.yaml @@ -2,8 +2,8 @@ meta: title: "MiniMax-M3" slug: "minimax-m3" provider: "MiniMax" - description: "MiniMax M3 vision-language MoE (427B total / 26B active) for frontier coding, agent toolchains, and 1M-token reasoning via MSA sparse attention — native multimodal (image + video + computer use); BF16 checkpoint with an MXFP8 variant from NVIDIA. Runs on NVIDIA (Hopper/Blackwell) and on AMD CDNA4 (MI350X/MI355X) and CDNA3 (MI300X/MI325X)." - date_updated: 2026-06-12 + description: "MiniMax M3 vision-language MoE (427B total / 26B active) for frontier coding, agent toolchains, and 1M-token reasoning via MSA sparse attention — native multimodal (image + video + computer use); BF16 plus MXFP8 and AMD MI355X MXFP4 variants. Runs on NVIDIA (Hopper/Blackwell) and AMD CDNA4/CDNA3." + date_updated: 2026-06-25 difficulty: advanced tasks: - text @@ -109,6 +109,27 @@ variants: # comfortably, ~4 GPUs for weights alone on Blackwell (B200/B300) or AMD MI350X/MI355X (gfx950)). vram_minimum_gb: 513 description: "NVIDIA-quantized MXFP8 weights — Blackwell (B200/B300) for native MX tensor cores, and AMD CDNA4 (MI350X/MI355X, gfx950) for native MXFP8 Matrix Cores." + mxfp4: + model_id: "amd/MiniMax-M3-MXFP4" + precision: mxfp4 + # 427B × 0.5 bytes (MXFP4) × 1.2 headroom = 256.2 GB. + vram_minimum_gb: 257 + description: "AMD Quark OCP MXFP4 weights and dynamic MXFP4 activations for MI355X (gfx950)." + supported_hardware: + - mi355x + # Match the validated InferenceX single-node baseline. TP8 also leaves + # practical headroom for KV cache and the multimodal encoder. + tp: 8 + hardware_overrides: + mi355x: + docker_image: "vllm/vllm-openai-rocm:nightly" + extra_args: + - "--trust-remote-code" + # Keep the multimodal encoder loaded and run it data-parallel on MI355X. + - "--mm-encoder-tp-mode" + - "data" + - "--mm-encoder-attn-backend" + - "ROCM_AITER_FA" compatible_strategies: - single_node_tp @@ -424,6 +445,29 @@ guide: | For best MXFP8 throughput, prefer Blackwell (B200/B300) for native MX tensor cores, or AMD CDNA4 (MI350X/MI355X, gfx950) for native MXFP8 matrix cores. + ## Quantized Variant (MXFP4 on MI355X) + + [`amd/MiniMax-M3-MXFP4`](https://huggingface.co/amd/MiniMax-M3-MXFP4) + uses static OCP MXFP4 weights and dynamic OCP MXFP4 activations produced with + AMD Quark. It targets AMD Instinct MI355X (gfx950) and requires a ROCm + nightly containing the Quark MXFP4 fix from vLLM PR #45794. + + Select the **mxfp4** variant and **MI355X** hardware in the command builder, + or launch the tested TP8 baseline directly: + + ```bash + vllm serve amd/MiniMax-M3-MXFP4 \ + --tensor-parallel-size 8 \ + --trust-remote-code \ + --block-size 128 \ + --attention-backend TRITON_ATTN \ + --mm-encoder-tp-mode data \ + --mm-encoder-attn-backend ROCM_AITER_FA \ + --tool-call-parser minimax_m3 \ + --enable-auto-tool-choice \ + --reasoning-parser minimax_m3 + ``` + ## Troubleshooting - **`--block-size` mismatch.** MSA's sparse block size is 128; the vLLM KV @@ -447,6 +491,7 @@ guide: | - [Model card](https://huggingface.co/MiniMaxAI/MiniMax-M3) - [MXFP8 variant](https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8) + - [AMD MXFP4 variant](https://huggingface.co/amd/MiniMax-M3-MXFP4) - [MiniMax](https://www.minimax.io/) - [MiniMax Agent](https://agent.minimax.io/) - [MiniMax Platform](https://platform.minimax.io/) diff --git a/scripts/build-recipes-api.mjs b/scripts/build-recipes-api.mjs index 973d2550..17738ed9 100644 --- a/scripts/build-recipes-api.mjs +++ b/scripts/build-recipes-api.mjs @@ -181,7 +181,7 @@ function renderCommand(recipe, variantKey, strategy, hwId, nodeCount, features, const variant = recipe.variants?.[variantKey] || recipe.variants?.default || {}; const hwProfile = taxonomy.hardware_profiles?.[hwId] || {}; - const dockerMeta = computeDockerMeta(recipe, variant, hwProfile); + const dockerMeta = computeDockerMeta(recipe, variant, hwProfile, hwId); const env = result.env || {}; const base = { @@ -253,7 +253,7 @@ function buildVariantRendering(recipe, variantKey, hwId, strategies, taxonomy) { // shard an oversized variant — substitute the largest variant that fits, and // skip multi-node strategies. Mirrors the command builder's UI behavior. if (!scalable && !fitsSingleNode(hwProfile, variant)) { - const fitting = pickFittingVariant(recipe, hwProfile); + const fitting = pickFittingVariant(recipe, hwProfile, hwId); if (!fitting) return null; variantKey = fitting; variant = recipe.variants[fitting]; @@ -386,6 +386,11 @@ function renderAndWriteVariant(recipe, variantKey, altBaseHfId, strategies, taxo if (!variant) return null; if ((recipe.meta?.tasks || []).includes("omni")) return null; + // Hardware/strategy compatibility can shrink between recipe revisions. + // Clear this variant's generated subtree so removed targets do not survive + // as stale, directly-addressable JSON files from an earlier build. + fs.rmSync(path.join(PUBLIC, altBaseHfId), { recursive: true, force: true }); + const hwProfiles = taxonomy.hardware_profiles || {}; const defaultHw = pickDefaultHardware(hwProfiles, variant, recipe); // Mirror the UI rule: `restricted` profiles (TPU, etc.) only surface for diff --git a/src/components/recipes/CommandBuilder.jsx b/src/components/recipes/CommandBuilder.jsx index 2e299a51..a0768c3d 100644 --- a/src/components/recipes/CommandBuilder.jsx +++ b/src/components/recipes/CommandBuilder.jsx @@ -4,7 +4,7 @@ import { useState, useMemo, useCallback, useEffect, useRef } from "react"; import { createPortal } from "react-dom"; import { useSearchParams, useRouter, usePathname } from "next/navigation"; import { Copy, Check, Terminal, Gauge, Sparkles, ChevronDown, Package, Info, Zap, Globe, Wrench, Brain } from "lucide-react"; -import { resolveCommand, recommendStrategy, isPrecisionCompatible, isHardwareSupported, fitsSingleNode, isHardwareScalable, variantRunsOnHardware, pickFittingVariant, pickDefaultHardware, resolveSingleNodeTp, computeDockerMeta, buildDockerRun, resolveOmniCommand, pdPoolModes } from "@/lib/command-synthesis"; +import { resolveCommand, recommendStrategy, isPrecisionCompatible, isHardwareSupported, isVariantHardwareSupported, fitsSingleNode, isHardwareScalable, variantRunsOnHardware, pickFittingVariant, pickDefaultHardware, resolveSingleNodeTp, computeDockerMeta, buildDockerRun, resolveOmniCommand, pdPoolModes } from "@/lib/command-synthesis"; import { resolveOmniTasks } from "@/lib/omni-tasks"; import { TooltipProvider, InfoTip } from "@/components/ui/tooltip"; import { detectPlaceholdersAll, substitute, substituteEnv, loadEndpoints, saveEndpoint, clearEndpoints } from "@/lib/cluster-endpoints"; @@ -347,7 +347,14 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { // eslint-disable-next-line react-hooks/exhaustive-deps }, [recipe, taxonomy]); - const [hwId, setHwId] = useState(searchParams.get("hardware") || defaultHw); + const requestedHwId = searchParams.get("hardware"); + const requestedVariant = recipe.variants?.[searchParams.get("variant") || "default"] || recipe.variants?.default || {}; + const requestedHwProfile = taxonomy.hardware_profiles?.[requestedHwId] || {}; + const requestedHwAllowed = requestedHwId + && isPrecisionCompatible(requestedHwProfile, requestedVariant) + && isHardwareSupported(recipe, requestedHwId) + && isVariantHardwareSupported(requestedVariant, requestedHwId); + const [hwId, setHwId] = useState(requestedHwAllowed ? requestedHwId : defaultHw); // After mount: restore preferences from localStorage in two scopes. // URL params always win (explicit > stored). @@ -361,6 +368,13 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { useEffect(() => { const prefs = loadPreferences(); let restoredFitsHw = null; + // Variant restrictions beat an incompatible hardware query parameter. + // Normalize the URL immediately so copied links match the rendered state. + if (requestedHwId && requestedHwId !== hwId) { + const sp = new URLSearchParams(searchParams.toString()); + sp.set("hardware", hwId); + router.replace(`?${sp.toString()}`, { scroll: false }); + } if (!searchParams.get("hardware") && prefs.hardware) { const v = recipe.variants?.[variant] || recipe.variants?.default || {}; const prefProfile = taxonomy.hardware_profiles?.[prefs.hardware]; @@ -370,7 +384,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { // preference and leave every other recipe with no rendered pill selected. const declaredHere = prefs.hardware in (recipe.meta?.hardware || {}); const restrictedElsewhere = prefProfile?.restricted && !declaredHere; - if (prefProfile?.brand === "NVIDIA" && !restrictedElsewhere && isPrecisionCompatible(prefProfile, v) && isHardwareSupported(recipe, prefs.hardware)) { + if (prefProfile?.brand === "NVIDIA" && !restrictedElsewhere && isPrecisionCompatible(prefProfile, v) && isHardwareSupported(recipe, prefs.hardware) && isVariantHardwareSupported(v, prefs.hardware)) { setHwId(prefs.hardware); restoredFitsHw = prefProfile; } @@ -387,7 +401,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { } // Resolve the hardware this mount actually settles on (URL > restored pref // > default) so the non-scalable fixups below see the right profile. - const resolvedHwId = restoredFitsHw ? prefs.hardware : (searchParams.get("hardware") || defaultHw); + const resolvedHwId = restoredFitsHw ? prefs.hardware : hwId; const resolvedHw = taxonomy.hardware_profiles?.[resolvedHwId]; const resolvedScalable = isHardwareScalable(resolvedHw); // Non-scalable hardware can't shard an oversized variant. If we land on one @@ -397,7 +411,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { if (!searchParams.get("variant") && resolvedHw && !resolvedScalable) { const v = recipe.variants?.[variant] || recipe.variants?.default || {}; if (!fitsSingleNode(resolvedHw, v)) { - const fitting = pickFittingVariant(recipe, resolvedHw); + const fitting = pickFittingVariant(recipe, resolvedHw, resolvedHwId); if (fitting && fitting !== variant) setVariant(fitting); } } @@ -757,12 +771,12 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { const selectVariant = (key) => { setVariant(key); - // Only swap hardware when precision demands it (e.g. NVFP4 needs Blackwell). - // VRAM is not a blocker because multi-node TP/DP can always supply more. + // Swap hardware when precision or an explicit variant allowlist requires + // it. VRAM alone is not a blocker because scalable profiles can add nodes. const v = recipe.variants?.[key] || {}; const currentProfile = taxonomy.hardware_profiles?.[hwId] || {}; const updates = { variant: key }; - if (!isPrecisionCompatible(currentProfile, v)) { + if (!isPrecisionCompatible(currentProfile, v) || !isVariantHardwareSupported(v, hwId)) { const next = pickDefaultHardware(taxonomy.hardware_profiles, v, recipe); setHwId(next); updates.hardware = next; @@ -789,17 +803,17 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { setStrategyOverride(""); const newProfile = taxonomy.hardware_profiles?.[id] || {}; const newScalable = isHardwareScalable(newProfile); - // Non-scalable hardware (single-GPU workstation) can't shard an oversized - // variant. If the active variant doesn't fit the new box, fall to the - // largest variant that does (e.g. BF16 → FP8 on DGX Station). + // If the active variant cannot run on the new hardware (explicit allowlist, + // precision, or a non-scalable VRAM shortfall), fall to the largest variant + // that can run there. let activeVariant = currentVariant; // Folded into the single syncUrl below rather than synced here — a separate // syncUrl call would read stale searchParams and get clobbered (same footgun // as selectVariant). Left undefined when the variant is unchanged so the // existing variant= param is preserved, not deleted. let variantUpdate; - if (!newScalable && !fitsSingleNode(newProfile, currentVariant)) { - const fitting = pickFittingVariant(recipe, newProfile); + if (!variantRunsOnHardware(newProfile, currentVariant, id)) { + const fitting = pickFittingVariant(recipe, newProfile, id); if (fitting && fitting !== variant) { setVariant(fitting); variantUpdate = fitting; @@ -1071,7 +1085,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { const altCudaSuffix = "cu129"; const dockerMeta = useMemo(() => { - const meta = computeDockerMeta(recipe, currentVariant, hwProfile); + const meta = computeDockerMeta(recipe, currentVariant, hwProfile, hwId); if (meta.brandKey !== "nvidia") return meta; // Explicit CUDA map (e.g. `{cu129: ..., cu130: ...}`) — pick the matching @@ -1098,7 +1112,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { return { ...meta, image: next }; } return meta; - }, [recipe, currentVariant, hwProfile, dockerCudaVariant, altCudaSuffix]); + }, [recipe, currentVariant, hwProfile, hwId, dockerCudaVariant, altCudaSuffix]); // `installMode` carries the user's tab choice; `effectiveInstallMode` folds // in constraints that would hide a tab entirely (pip: recipe opt-out or TPU @@ -1224,13 +1238,16 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { {profiles.map(([id, p]) => { const precisionOk = isPrecisionCompatible(p, currentVariant); + const variantHardwareOk = isVariantHardwareSupported(currentVariant, id); const status = recipe.meta?.hardware?.[id]; const isUnsupported = status === "unsupported"; - const disabled = !precisionOk || isUnsupported; + const disabled = !precisionOk || !variantHardwareOk || isUnsupported; const verifiedNote = status === "verified" ? "\n\nVerified — author has tested this hardware end-to-end" : ""; - const reason = !precisionOk + const reason = !variantHardwareOk + ? `${currentVariant.precision?.toUpperCase()} is only supported on ${(currentVariant.supported_hardware || []).map((hw) => taxonomy.hardware_profiles?.[hw]?.display_name || hw).join(", ")}` + : !precisionOk ? `${currentVariant.precision?.toUpperCase()} requires NVIDIA Blackwell` : isUnsupported ? `Not yet supported on ${p.display_name} — this model doesn't run here today, may be enabled in a future release` @@ -1425,6 +1442,7 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { {profiles.map(([id, p]) => { const precisionOk = isPrecisionCompatible(p, currentVariant); + const variantHardwareOk = isVariantHardwareSupported(currentVariant, id); // Only `verified` carries a label; everything else = silent default. // `unsupported` = author opt-out for this model; disables the pill. const status = recipe.meta?.hardware?.[id]; @@ -1433,11 +1451,13 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { // only needs to fit 1× model per node (standard precision // check is enough). The old co-located single-node check // (2× model on one node) is no longer the default UX. - const disabled = !precisionOk || isUnsupported; + const disabled = !precisionOk || !variantHardwareOk || isUnsupported; const verifiedNote = status === "verified" ? "\n\nVerified — author has tested this hardware end-to-end" : ""; - const reason = !precisionOk + const reason = !variantHardwareOk + ? `${currentVariant.precision?.toUpperCase()} is only supported on ${(currentVariant.supported_hardware || []).map((hw) => taxonomy.hardware_profiles?.[hw]?.display_name || hw).join(", ")}` + : !precisionOk ? `${currentVariant.precision?.toUpperCase()} requires NVIDIA Blackwell` : isUnsupported ? `Not yet supported on ${p.display_name} — this model doesn't run here today, may be enabled in a future release` @@ -1480,10 +1500,10 @@ export function CommandBuilder({ recipe, strategies, taxonomy }) { > {Object.entries(recipe.variants || {}).map(([key, v]) => { - // On non-scalable hardware (single-GPU workstation) a variant - // that doesn't fit has nowhere to shard — disable it instead of - // rendering a command that can't run. - const disabled = !hwScalable && !variantRunsOnHardware(hwProfile, v); + // Disable variants excluded by an exact hardware allowlist, + // incompatible precision, or a non-scalable VRAM shortfall. + const disabled = !variantRunsOnHardware(hwProfile, v, hwId); + const hardwareRestricted = !isVariantHardwareSupported(v, hwId); return ( !disabled && selectVariant(key)} title={ disabled - ? `${(v.label || v.precision)?.toUpperCase()} needs ${v.vram_minimum_gb} GB but ${hwProfile.display_name || "this hardware"} has ${hwProfile.vram_gb} GB and can't scale out — pick a smaller-footprint variant` + ? hardwareRestricted + ? `${(v.label || v.precision)?.toUpperCase()} is only supported on ${(v.supported_hardware || []).map((hw) => taxonomy.hardware_profiles?.[hw]?.display_name || hw).join(", ")}` + : `${(v.label || v.precision)?.toUpperCase()} needs ${v.vram_minimum_gb} GB but ${hwProfile.display_name || "this hardware"} has ${hwProfile.vram_gb} GB and can't scale out — pick a smaller-footprint variant` : [ v.description, `Min ${v.vram_minimum_gb} GB to load — add KV cache for serving. Scale out via multi-node if needed.`, diff --git a/src/lib/command-synthesis.js b/src/lib/command-synthesis.js index 340e4834..8ed1bf59 100644 --- a/src/lib/command-synthesis.js +++ b/src/lib/command-synthesis.js @@ -162,6 +162,16 @@ export function isHardwareSupported(recipe, hwId) { return recipe?.meta?.hardware?.[hwId] !== "unsupported"; } +/** + * Variant-level hardware allowlist. Missing/empty means the variant inherits + * the recipe's normal hardware compatibility; otherwise only listed profile + * ids may render or be selected. + */ +export function isVariantHardwareSupported(variant, hwId) { + const supported = variant?.supported_hardware; + return !Array.isArray(supported) || supported.length === 0 || supported.includes(hwId); +} + /** * List hardware profiles compatible with a variant by precision constraint * only. VRAM is NOT a blocking constraint — users can scale out via multi-node @@ -169,7 +179,11 @@ export function isHardwareSupported(recipe, hwId) { */ export function listCompatibleHardware(hwProfiles, variant, recipe) { return Object.entries(hwProfiles) - .filter(([id, p]) => isPrecisionCompatible(p, variant) && isHardwareSupported(recipe, id)) + .filter(([id, p]) => + isPrecisionCompatible(p, variant) + && isHardwareSupported(recipe, id) + && isVariantHardwareSupported(variant, id) + ) .map(([id]) => id); } @@ -206,8 +220,9 @@ export function isHardwareScalable(hwProfile) { * weights already fit single-node. Used to disable variant pills on * non-scalable hardware where the variant has nowhere to shard. */ -export function variantRunsOnHardware(hwProfile, variant) { +export function variantRunsOnHardware(hwProfile, variant, hwId = null) { if (!isPrecisionCompatible(hwProfile, variant)) return false; + if (hwId && !isVariantHardwareSupported(variant, hwId)) return false; if (isHardwareScalable(hwProfile)) return true; return fitsSingleNode(hwProfile, variant); } @@ -219,9 +234,9 @@ export function variantRunsOnHardware(hwProfile, variant) { * key, or null when nothing fits. Used to auto-fall off an oversized variant * (e.g. BF16 → FP8) when the user selects a single-GPU workstation. */ -export function pickFittingVariant(recipe, hwProfile) { +export function pickFittingVariant(recipe, hwProfile, hwId = null) { const fitting = Object.entries(recipe.variants || {}).filter( - ([, v]) => variantRunsOnHardware(hwProfile, v) + ([, v]) => variantRunsOnHardware(hwProfile, v, hwId) ); if (!fitting.length) return null; fitting.sort((a, b) => (b[1].vram_minimum_gb || 0) - (a[1].vram_minimum_gb || 0)); @@ -251,7 +266,10 @@ export function pdFitsSingleNode(hwProfile, variant) { export function pickDefaultHardware(hwProfiles, variant, recipe) { const constraint = PRECISION_HARDWARE_CONSTRAINTS[variant?.precision]; const compatible = Object.entries(hwProfiles).filter( - ([id, p]) => matchesConstraint(p, constraint) && isHardwareSupported(recipe, id) + ([id, p]) => + matchesConstraint(p, constraint) + && isHardwareSupported(recipe, id) + && isVariantHardwareSupported(variant, id) ); if (constraint?.generation === "blackwell") { @@ -282,10 +300,12 @@ export function pickDefaultHardware(hwProfiles, variant, recipe) { // { cu129, cu130 } (NVIDIA CUDA-keyed — explicit paired tags, // auto-suffix is skipped in favor of these) // { nvidia: { cu129, cu130 }, amd, tpu } (mixed: NVIDIA value may be a CUDA map) +// Exact variant+hardware overrides may also set +// variants..hardware_overrides..docker_image // // When a CUDA map is in play, `cudaMap` is returned so the caller can pick by // the user's `dockerCudaVariant` toggle instead of appending `-cu129`/`-cu130`. -export function computeDockerMeta(recipe, variant, hwProfile) { +export function computeDockerMeta(recipe, variant, hwProfile, hwId = null) { // When `model.nightly_required: true` and no explicit `docker_image` pin, // swap the brand defaults to nightly tags so the Install block matches the // nightly pip wheel that's also being rendered. vLLM publishes `:nightly` @@ -308,26 +328,41 @@ export function computeDockerMeta(recipe, variant, hwProfile) { const isTpu = hwProfile?.generation === "tpu"; const isIntel = hwProfile?.generation === "cpu" ||hwProfile?.brand === "Intel"; const brandKey = isTpu ? "tpu" : isAmd ? "amd" : isIntel ? "intel" : "nvidia"; - const override = variant?.docker_image || recipe.model?.docker_image; + // Exact variant+hardware image overrides win over variant-wide and + // model-wide images (for example, an MI355X-only ROCm nightly). + const exactHardwareOverride = hwId + ? variant?.hardware_overrides?.[hwId]?.docker_image + : null; const isCudaMap = (v) => v && typeof v === "object" && ("cu129" in v || "cu130" in v); - let pinned = null; + let pinned = typeof exactHardwareOverride === "string" ? exactHardwareOverride : null; let cudaMap = null; - if (typeof override === "string") { - if (brandKey === "nvidia") pinned = override; - } else if (override && typeof override === "object") { - const isBrandKeyed = "nvidia" in override || "amd" in override || "tpu" in override || "intel" in override; - if (isBrandKeyed) { - const brandValue = override[brandKey]; - if (typeof brandValue === "string") pinned = brandValue; - else if (brandKey === "nvidia" && isCudaMap(brandValue)) cudaMap = brandValue; - } else if (brandKey === "nvidia" && isCudaMap(override)) { - cudaMap = override; + + function applyOverride(override) { + if (!override || pinned || cudaMap) return; + if (typeof override === "string") { + if (brandKey === "nvidia") pinned = override; + return; + } + if (typeof override === "object") { + const isBrandKeyed = "nvidia" in override || "amd" in override || "tpu" in override || "intel" in override; + if (isBrandKeyed) { + const brandValue = override[brandKey]; + if (typeof brandValue === "string") pinned = brandValue; + else if (brandKey === "nvidia" && isCudaMap(brandValue)) cudaMap = brandValue; + } else if (brandKey === "nvidia" && isCudaMap(override)) { + cudaMap = override; + } } } + applyOverride(variant?.docker_image); + // A partial variant override falls back to the model image for brands it + // does not cover instead of skipping directly to the global default. + applyOverride(recipe.model?.docker_image); + const image = pinned || DEFAULT_IMAGE[brandKey]; const gpuFlags = isTpu ? "--privileged --network host \\\n -v /dev/shm:/dev/shm" @@ -537,6 +572,7 @@ export function resolveCommand(recipe, variantKey, strategyName, hwProfileId, en const singleNodeTp = resolveSingleNodeTp(recipe, variant, hwProfile, strategyName); const modelId = variant.model_id || recipe.model?.model_id || "unknown"; + const variantHardwareOverride = variant?.hardware_overrides?.[hwProfileId]; // Helper to merge args function buildArgs(roleOverride, nodeRole) { @@ -556,6 +592,9 @@ export function resolveCommand(recipe, variantKey, strategyName, hwProfileId, en // 2. Variant extra args if (variantKey !== "default" && variant.extra_args) args.push(...variant.extra_args); + if (variantKey !== "default" && variantHardwareOverride?.extra_args) { + args.push(...variantHardwareOverride.extra_args); + } // 3. Strategy args + parallel size (grouped together so -tp/-dp sits next to -ep etc.) if (strategy.deploy_type !== "pd_cluster") { @@ -766,6 +805,9 @@ export function resolveCommand(recipe, variantKey, strategyName, hwProfileId, en // Variant env if (variantKey !== "default" && variant.extra_env) Object.assign(env, variant.extra_env); + if (variantKey !== "default" && variantHardwareOverride?.extra_env) { + Object.assign(env, variantHardwareOverride.extra_env); + } // Strategy env if (strategy.deploy_type !== "pd_cluster") {