ggml-org
diff --git a/‎.github/workflows/build-self-hosted.yml‎
Lines changed: 143 additions & 0 deletions b/‎.github/workflows/build-self-hosted.yml‎
Lines changed: 143 additions & 0 deletions
diff --git a/‎CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions b/‎CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎README.md‎
Lines changed: 82 additions & 0 deletions b/‎README.md‎
Lines changed: 82 additions & 0 deletions
diff --git a/‎models/download-vitisai-model.cmd‎
Lines changed: 32 additions & 0 deletions b/‎models/download-vitisai-model.cmd‎
Lines changed: 32 additions & 0 deletions
@@ -114,3 +114,146 @@ jobs:
         run: |
           vulkaninfo --summary
           GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
+
+  npu-amd-windows:
+    runs-on: [self-hosted, Windows, X64, stx, rai300-400]
+    timeout-minutes: 60
+    continue-on-error: true   # advisory while the runner pool is new; revisit later
+
+    env:
+      FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip
+      MODEL: base
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - uses: microsoft/setup-msbuild@v2
+
+      - name: Install CMake if not available
+        shell: powershell
+        run: |
+          $installed = Get-Command cmake -ErrorAction SilentlyContinue
+          if (-not $installed) {
+            $ver = "3.28.1"
+            $url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi"
+            Invoke-WebRequest -Uri $url -OutFile cmake.msi
+            Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait
+            $p = "C:\Program Files\CMake\bin"
+            $env:PATH = "$p;$env:PATH"
+            echo $p >> $env:GITHUB_PATH
+            cmake --version
+            if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 }
+          } else { cmake --version }
+
+      - name: Download FlexML runtime
+        shell: powershell
+        run: |
+          Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip
+          if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 }
+          if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 }
+          tar xf flexmlrt.zip
+          if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 }
+          if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 }
+
+      - name: Setup FlexML, configure and build
+        shell: cmd
+        run: |
+          cd flexmlrt
+          call setup.bat
+          if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 )
+          cd ..
+          cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
+          if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 )
+          cmake --build build --config Release -j
+          if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 )
+
+      - name: Copy FlexML DLLs to build output
+        shell: powershell
+        run: |
+          foreach ($d in "flexmlrt/bin", "flexmlrt/lib") {
+            if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force }
+          }
+          if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) {
+            Write-Error "flexmlrt.dll not staged next to binaries"; exit 1
+          }
+
+      - name: Download ggml model
+        shell: cmd
+        run: |
+          call models\download-ggml-model.cmd %MODEL% models
+          if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 )
+
+      - name: Download NPU encoder cache
+        shell: cmd
+        run: |
+          .\models\download-vitisai-model.cmd %MODEL%
+          if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 )
+
+      - name: Run NPU smoke test
+        shell: cmd
+        run: |
+          build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1
+          type vitisai.log
+          findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 )
+          findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 )
+
+      - name: Upload smoke test log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vitisai-smoke-log-windows
+          path: vitisai.log
+
+  npu-amd-linux:
+    runs-on: [self-hosted, Linux, X64, stx, rai300-400]
+    timeout-minutes: 60
+    continue-on-error: true   # advisory while the runner pool is new; revisit later
+
+    env:
+      FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz
+      MODEL: base
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Verify NPU device
+        run: |
+          lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; }
+          ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; }
+
+      - name: Download FlexML runtime (Linux)
+        run: |
+          curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL"
+          tar xf flexmlrt.tar.gz
+          source flexmlrt/setup.sh
+          echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+
+      - name: Configure and build
+        run: |
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
+          cmake --build build --config Release -j $(nproc)
+
+      - name: Download ggml model
+        run: |
+          ./models/download-ggml-model.sh $MODEL
+
+      - name: Download NPU encoder cache
+        run: |
+          sh ./models/download-vitisai-model.sh $MODEL
+          [ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; }
+
+      - name: Run NPU smoke test
+        run: |
+          ./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log
+          grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; }
+          grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; }
+
+      - name: Upload smoke test log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vitisai-smoke-log-linux
+          path: vitisai.log
@@ -115,6 +115,7 @@ endif()
 option(WHISPER_COREML                "whisper: enable Core ML framework"  OFF)
 option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF)
 option(WHISPER_OPENVINO              "whisper: support for OpenVINO"      OFF)
+option(WHISPER_VITISAI               "whisper: support for AMD Vitis AI"  OFF)
 
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 
@@ -21,6 +21,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
 - Support for CPU-only inference
 - [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
 - [AMD ROCm GPU support](#amd-rocm-gpu-support)
+- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support)
 - [OpenVINO Support](#openvino-support)
 - [Ascend NPU Support](#ascend-npu-support)
 - [Moore Threads GPU Support](#moore-threads-gpu-support)
@@ -312,6 +313,87 @@ This can result in significant speedup in encoder performance. Here are the inst
 
 For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
 
+## AMD Ryzen™ AI NPU support
+
+On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference.
+
+### Prerequisites
+
+Supported Platforms
+
+- **Windows 11**
+- **Linux** (Ubuntu 24.04 LTS, Python 3.12)
+
+Install the XRT runtime and FlexML runtime for your platform:
+
+- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers)
+- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps)
+
+After installing, source the setup scripts in every shell you use to build or run whisper.cpp:
+
+```bash
+# Linux
+source /opt/xilinx/xrt/setup.sh
+source /path/to/flexmlrt/setup.sh
+```
+
+```cmd
+:: Windows
+cd /path/to/flexmlrt && call setup.bat
+```
+
+You can verify the NPU is visible with:
+
+```bash
+xrt-smi examine
+```
+
+### Download models
+
+Download the ggml model and the matching prebuilt VitisAI encoder cache:
+
+```bash
+# Linux / macOS
+sh ./models/download-ggml-model.sh base
+sh ./models/download-vitisai-model.sh base
+```
+
+```cmd
+:: Windows
+.\models\download-ggml-model.cmd base
+.\models\download-vitisai-model.cmd base
+```
+
+Use the same model name with both scripts. To see all available VitisAI encoder caches:
+
+```bash
+sh ./models/download-vitisai-model.sh --list
+```
+
+```cmd
+.\models\download-vitisai-model.cmd --list
+```
+
+The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml-<model>-encoder-vitisai.rai`.
+
+> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization.
+
+### Build
+
+```bash
+cmake -B build -DWHISPER_VITISAI=1
+cmake --build build -j --config Release
+```
+
+### Run
+
+```bash
+./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
+```
+
+For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/).
+
+
 ## NVIDIA GPU support
 
 With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels.
 
@@ -0,0 +1,32 @@
+@echo off
+setlocal
+
+set "script=%~dp0download-vitisai-model.ps1"
+
+if "%~1"=="" (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%"
+  exit /b %ERRORLEVEL%
+)
+
+if /I "%~1"=="--list" (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
+  exit /b %ERRORLEVEL%
+)
+
+if /I "%~1"=="-l" (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
+  exit /b %ERRORLEVEL%
+)
+
+if /I "%~1"=="list" (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
+  exit /b %ERRORLEVEL%
+)
+
+if "%~2"=="" (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1"
+) else (
+  PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2"
+)
+
+exit /b %ERRORLEVEL%