Benchmark #432
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| kernel: | |
| description: "Kernel name (e.g. activation)" | |
| required: true | |
| type: string | |
| workflow_run: | |
| workflows: ["Build Release"] | |
| types: [completed] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.workflow_run.id || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' | |
| runs-on: | |
| group: aws-g6-12xlarge-plus | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Get kernel name | |
| id: kernel | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "kernel=${{ inputs.kernel }}" >> $GITHUB_OUTPUT | |
| else | |
| PR_TITLE=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}" --jq '.display_title') | |
| if [[ "$PR_TITLE" != *:* ]]; then | |
| echo "Workflow run title does not contain a kernel name, skipping" | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| KERNEL=$(echo "$PR_TITLE" | cut -d: -f1 | xargs) | |
| if [ -z "$KERNEL" ]; then | |
| echo "Could not extract kernel name from workflow run title, skipping" | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| echo "kernel=$KERNEL" >> $GITHUB_OUTPUT | |
| fi | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| if: steps.kernel.outputs.skip != 'true' | |
| with: | |
| python-version: '3.12' | |
| - name: Install kernels | |
| if: steps.kernel.outputs.skip != 'true' | |
| run: pip install "kernels[benchmark] @ git+https://github.com/huggingface/kernels#subdirectory=kernels" | |
| - name: Benchmark kernel | |
| if: steps.kernel.outputs.skip != 'true' | |
| run: kernels benchmark "kernels-community/${{ steps.kernel.outputs.kernel }}" --branch main --visual benches | |
| - name: Ensure README has performance section | |
| if: steps.kernel.outputs.skip != 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| REPO="kernels-community/${{ steps.kernel.outputs.kernel }}" | |
| hf download "$REPO" README.md --local-dir /tmp/kernel-readme --revision main | |
| if ! grep -q '### Performance' /tmp/kernel-readme/README.md; then | |
| cat >> /tmp/kernel-readme/README.md << 'EOF' | |
| ### Performance | |
| <img class="dark:hidden border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_light_animation.svg" /> | |
| <img class="hidden dark:block border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_dark_animation.svg" /> | |
| <img class="dark:hidden border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_light_latency.svg" /> | |
| <img class="hidden dark:block border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_dark_latency.svg" /> | |
| <img class="dark:hidden border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_light_throughput.svg" /> | |
| <img class="hidden dark:block border border-gray-200 dark:border-gray-700 rounded-lg" src="media/benches_dark_throughput.svg" /> | |
| EOF | |
| hf upload "$REPO" /tmp/kernel-readme/README.md README.md --revision main | |
| fi | |
| - name: Upload benchmark visuals | |
| if: steps.kernel.outputs.skip != 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: hf upload "kernels-community/${{ steps.kernel.outputs.kernel }}" media media --revision main |