Feat: TEM-14 functional testing

.github/actions/smoke-test/action.yml

@@ -55,6 +55,50 @@ inputs:
       its own entrypoint and would fail the probe.
     required: false
     default: "false"
+  test-comfyui:
+    description: |
+      'true' to add `test_comfyui: true` to every group: the ComfyUI
+      reachability SMOKE check. Exposes :8188 as 8188/http and probes it
+      both in-pod (curl 127.0.0.1:8188) and via the public Runpod proxy.
+      Answers "is ComfyUI up and reachable from a browser?" — NOT whether
+      it can generate. Cheap; safe to leave on for ComfyUI images.
+    required: false
+    default: "false"
+  test-comfyui-functional:
+    description: |
+      'true' to add `test_comfyui_functional: true` to every group: the
+      ComfyUI end-to-end FUNCTIONAL check. After the pod boots, test_images.py
+      talks to ComfyUI over the public proxy URL (no SSH): provisions the
+      model(s) from tests/comfyui/models.json via the baked-in
+      ComfyUI-RunpodDirect node's /server_download/* routes, POSTs
+      tests/comfyui/workflows/*.api.json to /prompt, polls /history, and
+      asserts a real PNG came out. Heavier (pulls a ~2 GB model + uses GPU
+      time). Implies `test-comfyui` (reachability runs first). Off by
+      default — gate it behind an enabler.
+    required: false
+    default: "false"
+  save-comfyui-images:
+    description: |
+      'true' to keep the PNG(s) the ComfyUI functional check generates and
+      upload them as a workflow ARTIFACT (nothing is committed to the repo —
+      you download them from the run's Summary page). Sets COMFYUI_SAVE_DIR
+      for test_images.py, then uploads that dir after the run (even if the
+      run failed, so you can inspect a bad image). No-op unless
+      `test-comfyui-functional` actually produced an image. Off by default.
+    required: false
+    default: "false"
+  comfyui-images-artifact-name:
+    description: |
+      Name of the uploaded images artifact (only used when
+      `save-comfyui-images` is 'true'). Give each call a distinct name if the
+      same workflow run invokes this action more than once — upload-artifact
+      rejects duplicate artifact names within a run.
+    required: false
+    default: "comfyui-generated-images"
+  comfyui-images-retention-days:
+    description: "Retention (days) for the uploaded images artifact."
+    required: false
+    default: "14"
   check_all_gpu:
     description: |
       'true' to emit `check_all_gpu: true` for GPU groups. The smoke
@@ -258,6 +302,8 @@ runs:
         MIN_VRAM_GB: ${{ inputs.min-vram-gb }}
         MANUFACTURER: ${{ inputs.manufacturer }}
         TEST_JUPYTER: ${{ inputs.test-jupyter }}
+        TEST_COMFYUI: ${{ inputs.test-comfyui }}
+        TEST_COMFYUI_FUNCTIONAL: ${{ inputs.test-comfyui-functional }}
         CHECK_ALL_GPU: ${{ inputs.check_all_gpu }}
         TEST_PORTS: ${{ inputs.test_ports }}
         EXCLUDE_INSTANCES: ${{ inputs.exclude-instances }}
@@ -271,6 +317,12 @@ runs:
         if [[ "${TEST_JUPYTER,,}" == "true" ]]; then
           EXTRA_ARGS+=(--test-jupyter)
         fi
+        if [[ "${TEST_COMFYUI,,}" == "true" ]]; then
+          EXTRA_ARGS+=(--test-comfyui)
+        fi
+        if [[ "${TEST_COMFYUI_FUNCTIONAL,,}" == "true" ]]; then
+          EXTRA_ARGS+=(--test-comfyui-functional)
+        fi
         if [[ "${CHECK_ALL_GPU,,}" == "true" ]]; then
           EXTRA_ARGS+=(--check-all-gpu)
         fi
@@ -325,6 +377,29 @@ runs:
         # Forwarded to config.CREATE_TIMEOUT — bumps the SSH-readiness
         # deadline for slow pulls (mainly multi-GB ROCm base images).
         CREATE_TIMEOUT: ${{ inputs.create-timeout }}
+        SAVE_COMFYUI_IMAGES: ${{ inputs.save-comfyui-images }}
       run: |
         set -uo pipefail
+        # When asked to keep the generated PNGs, point the ComfyUI functional
+        # check at a save dir under RUNNER_TEMP (fetched over /view as a plain
+        # HTTP GET). The next step uploads it as an artifact. Left unset =
+        # validate-only, no copy kept (config.COMFYUI_SAVE_DIR default).
+        if [[ "${SAVE_COMFYUI_IMAGES,,}" == "true" ]]; then
+          export COMFYUI_SAVE_DIR="${RUNNER_TEMP}/comfy-out"
+          mkdir -p "${COMFYUI_SAVE_DIR}"
+        fi
         python3 "${GITHUB_WORKSPACE}/tests/test_images.py" "${MANIFEST_PATH}"
+
+    - name: Upload generated ComfyUI images
+      # always() so a FAILED functional run still surfaces whatever image it
+      # did manage to produce (helps debug a bad/garbled generation). Skipped
+      # entirely when image-keeping wasn't requested. if-no-files-found:ignore
+      # keeps the step green when the functional check didn't run / produced
+      # nothing (e.g. functional off, or it failed before /view).
+      if: ${{ always() && inputs.save-comfyui-images == 'true' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.comfyui-images-artifact-name }}
+        path: ${{ runner.temp }}/comfy-out
+        retention-days: ${{ inputs.comfyui-images-retention-days }}
+        if-no-files-found: ignore

.github/scripts/generate_test_manifest.py

@@ -41,6 +41,8 @@ def render_yaml(groups: dict) -> str:
             "manufacturer",
             "min_cuda_version",
             "test_jupyter",
+            "test_comfyui",
+            "test_comfyui_functional",
         ):
             if key in body:
                 val = body[key]
@@ -66,6 +68,8 @@ def build_groups(
     min_vram_gb: int,
     manufacturer: str,
     test_jupyter: bool = False,
+    test_comfyui: bool = False,
+    test_comfyui_functional: bool = False,
     check_all_gpu: bool = False,
     test_ports: list[int] | None = None,
     exclude_instances: list[str] | None = None,
@@ -84,6 +88,10 @@ def _decorate(body: dict, *, gpu_group: bool) -> dict:
             body["manufacturer"] = manufacturer
         if test_jupyter:
             body["test_jupyter"] = True
+        if test_comfyui:
+            body["test_comfyui"] = True
+        if test_comfyui_functional:
+            body["test_comfyui_functional"] = True
         if test_ports:
             body["test_ports"] = list(test_ports)
         if exclude_instances:
@@ -125,6 +133,21 @@ def main() -> int:
     ap.add_argument("--min-vram-gb", type=int, default=16)
     ap.add_argument("--manufacturer", default="Nvidia")
     ap.add_argument("--test-jupyter", action="store_true")
+    ap.add_argument(
+        "--test-comfyui",
+        action="store_true",
+        help="Add `test_comfyui: true` to every group: ComfyUI reachability "
+             "SMOKE check — expose :8188 and probe it in-pod + via the public "
+             "proxy. Answers 'is ComfyUI up and reachable?'.",
+    )
+    ap.add_argument(
+        "--test-comfyui-functional",
+        action="store_true",
+        help="Add `test_comfyui_functional: true` to every group: the ComfyUI "
+             "end-to-end FUNCTIONAL check (download model, run workflow, "
+             "validate output PNG) host-side via the public proxy (no SSH). "
+             "Implies --test-comfyui.",
+    )
     ap.add_argument("--check-all-gpu", action="store_true")
     ap.add_argument(
         "--test-port",
@@ -161,6 +184,8 @@ def main() -> int:
         min_vram_gb=args.min_vram_gb,
         manufacturer=args.manufacturer,
         test_jupyter=args.test_jupyter,
+        test_comfyui=args.test_comfyui,
+        test_comfyui_functional=args.test_comfyui_functional,
         check_all_gpu=args.check_all_gpu,
         test_ports=args.test_port,
         exclude_instances=args.exclude_instance,

.github/workflows/check-incompatibilities.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: blacksmith-8vcpu-ubuntu-2204
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
         with:
           fetch-depth: 0
 
@@ -24,13 +24,20 @@ jobs:
           # smoke-test expects image-refs as a JSON array, so wrap the single
           # input ref. inputs.image is always set here.
           image-refs: ${{ format('["{0}"]', inputs.image) }}
-          profile: base
+          profile: gpu
           runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }}
           ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }}
           test-jupyter: true
           check_all_gpu: true
           manufacturer: Nvidia
+          # Smoke: always verify ComfyUI is up + reachable on :8188
+          test-comfyui: true
+          # Run the full ComfyUI functional check on EVERY resolved GPU (that's
+          # the point of this matrix — catch GPU-specific generation failures,
+          # e.g. missing Blackwell kernels). Implies the :8188 reachability
+          # smoke, so 8188 is dropped from test_ports below (only 8080 left).
+          test-comfyui-functional: "true"
+          save-comfyui-images: "true"
           on-skip: pass
           test_ports: |
-            - 8188
             - 8080

.github/workflows/dev.yml

@@ -8,6 +8,16 @@ on:
         type: boolean
         required: true
         default: true
+      run_functional_tests:
+        description: >-
+          Run the ComfyUI functional test (download model + generate an
+          image on a real GPU). Slower and uses GPU credits; leave off for
+          the default smoke tests only (boot + CUDA + Jupyter + port checks).
+          When on, the generated PNG(s) are uploaded as the
+          "comfyui-generated-images" run artifact.
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   dev:
@@ -20,7 +30,7 @@ jobs:
     # Artifact file contents are not subject to that scrubbing.
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
 
       - name: Clear space to remove unused folders
         run: |
@@ -113,7 +123,7 @@ jobs:
     if: needs.dev.result == 'success' && inputs.push == true
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
         with:
           fetch-depth: 0
 
@@ -137,11 +147,16 @@ jobs:
         uses: ./.github/actions/smoke-test
         with:
           image-refs: ${{ steps.refs.outputs.refs }}
-          profile: base
+          profile: gpu
           runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }}
           ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }}
           test-jupyter: true
+          # Smoke: always verify ComfyUI is up + reachable on :8188
+          test-comfyui: true
+          # Functional (download model + generate image) is opt-in via the
+          # workflow_dispatch enabler. Implies test-comfyui (runs after it).
+          test-comfyui-functional: ${{ inputs.run_functional_tests }}
+          save-comfyui-images: ${{ inputs.run_functional_tests }}
           on-skip: "warn"
           test_ports: |
-            - 8188
             - 8080

.github/workflows/release.yml

@@ -16,12 +16,17 @@ on:
 permissions:
   contents: write # Required for creating GitHub releases
 
+env:
+  IMAGE_REF: runpod/comfyui
+
 jobs:
   build-and-push:
     runs-on: blacksmith-8vcpu-ubuntu-2204
+    outputs:
+      release-version: ${{ steps.vars.outputs.release-version }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
 
       - name: Clear space to remove unused folders
         run: |
@@ -43,29 +48,25 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set environment variables
+        id: vars
         run: |
-          echo "IMAGE_REF=runpod/comfyui" >> $GITHUB_ENV
-
           # Determine version based on trigger type
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             # Manual trigger: use input version
             VERSION="${{ github.event.inputs.version }}"
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=true" >> $GITHUB_ENV
           elif [[ "${{ github.event_name }}" == "release" ]]; then
             # GitHub Release published event
             VERSION="${{ github.event.release.tag_name }}"
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=false" >> $GITHUB_ENV
           else
             # Tag trigger: use tag name (remove refs/tags/ prefix)
             VERSION=${GITHUB_REF#refs/tags/}
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=false" >> $GITHUB_ENV
           fi
 
-          # Export TAG for docker-bake.hcl variable override
-          echo "TAG=${RELEASE_VERSION}" >> $GITHUB_ENV
+          # RELEASE_VERSION feeds the bake steps' TAG env below.
+          echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
+
+          # Hand the release version to the separate release-comfyui job.
+          echo "release-version=${VERSION}" >> "$GITHUB_OUTPUT"
 
       - name: Build and push CUDA 13.0 image
         id: build-cuda13
@@ -118,9 +119,9 @@ jobs:
             --argjson cuda13 "${CUDA13_REFS}" \
             --argjson regular "${REGULAR_REFS}" \
             '$cuda13 + $regular | unique')
-          # Write to a file for the artifact handoff to test-dev. (Job
-          # outputs can't be used here — they get scrubbed because the
-          # refs contain the DOCKERHUB_USERNAME secret; see job comment.)
+          # Write to a file so the refs reach the `test` job via an artifact.
+          # A job output can't be used here: the refs embed DOCKERHUB_USERNAME,
+          # so GitHub scrubs them from the output.
           mkdir -p "${RUNNER_TEMP}/refs"
           printf '%s\n' "${REFS}" > "${RUNNER_TEMP}/refs/image-refs.json"
           echo "Combined image refs:"
@@ -139,7 +140,7 @@ jobs:
     needs: build-and-push
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
         with:
           fetch-depth: 0
 
@@ -163,39 +164,54 @@ jobs:
         uses: ./.github/actions/smoke-test
         with:
           image-refs: ${{ steps.refs.outputs.refs }}
-          profile: base
+          # ComfyUI ships only GPU images; the `latest` tag carries no CUDA
+          # marker, so `base` would sort it into base_cpu and try to generate
+          # on a CPU pod. `gpu` puts every ref into a single base_gpu group.
+          profile: gpu
           runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }}
           ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }}
           test-jupyter: true
+          # Smoke: always verify ComfyUI is up + reachable on :8188
+          test-comfyui: true
+          # On a release the image MUST be able to generate, so the functional
+          # check is always on (release.yml has no run_functional_tests toggle).
+          test-comfyui-functional: "true"
+          save-comfyui-images: "true"
           on-skip: "warn"
           test_ports: |
-            - 8188
             - 8080
 
   release-comfyui:
     runs-on: blacksmith-2vcpu-ubuntu-2204
-    needs: build-and-push
+    # Gate the GitHub Release on the smoke + functional test passing — we don't
+    # want to publish a release for an image that can't generate. (Note: the
+    # Docker stable tags are still pushed by build-and-push before this runs;
+    # fully gating the image push needs the build/test/promote split tracked
+    # separately, to be reconciled with #17's release.yml rewrite.)
+    needs: [build-and-push, test]
     steps:
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v1
         if: github.event_name != 'release'
         with:
-          tag_name: ${{ env.RELEASE_VERSION }}
-          name: ${{ env.RELEASE_VERSION }}
+          tag_name: ${{ needs.build-and-push.outputs.release-version }}
+          name: ${{ needs.build-and-push.outputs.release-version }}
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Release Summary
+        env:
+          RELEASE_VERSION: ${{ needs.build-and-push.outputs.release-version }}
         run: |
           echo "🚀 Release completed!"
-          echo "Version: ${{ env.RELEASE_VERSION }}"
+          echo "Version: ${RELEASE_VERSION}"
           echo "Docker Images:"
-          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }}-cuda12.8 (CUDA 12.8)"
-          echo " - ${{ env.IMAGE_REF }}:cuda12.8"
-          echo " - ${{ env.IMAGE_REF }}:latest"
-          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }}-cuda13.0 (CUDA 13.0)"
-          echo " - ${{ env.IMAGE_REF }}:cuda13.0"
+          echo " - ${IMAGE_REF}:${RELEASE_VERSION}-cuda12.8 (CUDA 12.8)"
+          echo " - ${IMAGE_REF}:cuda12.8"
+          echo " - ${IMAGE_REF}:latest"
+          echo " - ${IMAGE_REF}:${RELEASE_VERSION}-cuda13.0 (CUDA 13.0)"
+          echo " - ${IMAGE_REF}:cuda13.0"
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             echo "Trigger: Manual workflow dispatch"
           else