Add CI workflow for self-running tests

.github/workflows/test_on_self_hosted.yml

@@ -0,0 +1,281 @@
+name: ProjectAirSim CI (Self-Hosted)
+
+on:
+  pull_request:
+    branches:
+      - main
+    # Only run when the self-hosted label is added; the job itself still
+    # rejects draft PRs and PRs without run-self-hosted.
+    types: [labeled]
+
+# Restrict GITHUB_TOKEN permissions (least privilege).
+# This workflow needs read access to checkout code and write access to
+# pull request comments to post CI results back to the PR.
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  build-and-test:
+    if: |
+      github.event.pull_request.base.ref == 'main' &&
+      !github.event.pull_request.draft &&
+      contains(github.event.pull_request.labels.*.name, 'run-self-hosted')
+    runs-on: [self-hosted]
+    timeout-minutes: 120
+
+    env:
+      UE_ROOT: ${{ secrets.UE_ROOT }}
+      AIRSIM_PY_ENV: ${{ github.workspace }}/airsimenv
+      DEBIAN_FRONTEND: noninteractive
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Sync submodules
+        run: |
+          git submodule sync
+          git submodule update --init --recursive
+
+      - name: Setup system dependencies
+        run: |
+          set -euo pipefail
+
+          sudo apt-get -o DPkg::Lock::Timeout=300 update
+          sudo apt-get -o DPkg::Lock::Timeout=300 install -y \
+            python3 \
+            python3-venv \
+            python3-pip \
+            build-essential \
+            lsof \
+            ninja-build \
+            vulkan-tools \
+            libvulkan1
+          echo "System dependencies installed"
+
+      - name: Setup AirSim development tools
+        run: |
+          chmod +x setup_linux_dev_tools.sh
+          ./setup_linux_dev_tools.sh
+
+      - name: Clean previous build
+        run: rm -rf build
+
+      - name: Build simlibs Release
+        run: ./build.sh simlibs_release
+
+      - name: Build Unreal Editor (Blocks)
+        run: |
+          "$UE_ROOT/Engine/Build/BatchFiles/Linux/Build.sh" \
+            BlocksEditor Linux Development \
+            "$(pwd)/unreal/Blocks/Blocks.uproject" \
+            -waitmutex
+
+      - name: Setup Python virtual environment and install dependencies
+        run: |
+          rm -rf "$AIRSIM_PY_ENV"
+          python3 -m venv "$AIRSIM_PY_ENV"
+          source "$AIRSIM_PY_ENV/bin/activate"
+          python -m pip install --upgrade pip
+          python -m pip install "setuptools<81" wheel
+          python -m pip install -e "client/python/projectairsim[datacollection]"
+          python -m pip install pytest pytest-cov
+
+      - name: Clean stale simulator processes and ports
+        run: |
+          pkill -f UnrealEditor || true
+          pkill -f Blocks || true
+          sleep 2
+          for p in 8989 8990 8991 41451; do
+            lsof -ti :"$p" | xargs -r kill -9 || true
+          done
+          rm -f unreal.pid unreal.log
+
+      - name: Diagnose GPU and Vulkan
+        run: |
+          echo "USER=$(whoami)"
+          echo "DISPLAY=$DISPLAY"
+          echo "XDG_SESSION_TYPE=$XDG_SESSION_TYPE"
+          echo "VK_ICD_FILENAMES=${VK_ICD_FILENAMES:-not set}"
+
+          echo "==== nvidia-smi ===="
+          nvidia-smi || true
+
+          echo "==== vulkaninfo summary ===="
+          vulkaninfo --summary || true
+
+          echo "==== Vulkan ICD files ===="
+          ls -lah /usr/share/vulkan/icd.d/ || true
+
+          echo "==== NVIDIA devices ===="
+          ls -lah /dev/nvidia* || true
+
+      - name: Launch Unreal headless with Play Mode
+        env:
+          PROJECTAIRSIM_CI: "1"
+        run: |
+          export DISPLAY=:0
+          export SDL_VIDEODRIVER=x11
+
+          "$UE_ROOT/Engine/Binaries/Linux/UnrealEditor" \
+            "$(pwd)/unreal/Blocks/Blocks.uproject" \
+            -game \
+            -enginedir="$UE_ROOT" \
+            -RenderOffScreen \
+            -vulkan \
+            -nosound -unattended -nopause -NoSplash -log \
+            -ResX=640 -ResY=480 \
+            -qualitylevel=0 \
+            > unreal.log 2>&1 &
+          echo $! > unreal.pid
+
+      - name: Wait for simulator readiness
+        run: |
+          source "$AIRSIM_PY_ENV/bin/activate"
+          echo "Waiting for ProjectAirSim sockets on ports 8989/8990..."
+          for i in {1..180}; do
+            if ps -p "$(cat unreal.pid)" > /dev/null 2>&1; then
+              if lsof -i :8989 >/dev/null 2>&1 && lsof -i :8990 >/dev/null 2>&1; then
+                echo "Ports 8989/8990 are listening, validating client handshake..."
+                if timeout 30 python -c 'from projectairsim import ProjectAirSimClient; c=ProjectAirSimClient(); c.connect(); c.get_topic_info(); c.disconnect(); print("Simulator ready")'; then
+                  echo "Simulator is ready"
+                  exit 0
+                fi
+              fi
+            else
+              echo "Unreal process has exited unexpectedly"
+              tail -n 300 unreal.log
+              exit 1
+            fi
+            sleep 1
+          done
+          echo "Simulator failed to start within timeout (180 sec)"
+          tail -n 300 unreal.log
+          exit 1
+
+      - name: Run Python tests
+        run: |
+          source "$AIRSIM_PY_ENV/bin/activate"
+          cd client/python/projectairsim/tests
+          python -m pytest -v --junitxml=pytest-results.xml
+
+      - name: Stop Unreal
+        if: always()
+        run: |
+          if [ -f unreal.pid ]; then
+            kill "$(cat unreal.pid)" || true
+          fi
+          pkill -f UnrealEditor || true
+          pkill -f Blocks || true
+
+      - name: Dump Unreal logs on failure
+        if: failure()
+        run: |
+          echo "==== Unreal Log Output ===="
+          cat unreal.log || true
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: "**/*.xml"
+
+      - name: Build PR comment body
+        if: always()
+        id: ci_summary
+        env:
+          JOB_STATUS: ${{ job.status }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+
+          ICON="❓"
+          case "$JOB_STATUS" in
+            success) ICON="✅" ;;
+            failure) ICON="❌" ;;
+            cancelled) ICON="⚠️" ;;
+          esac
+
+          XML_PATH="$(find . -name pytest-results.xml | head -n 1 || true)"
+          TESTS="N/A"
+          FAILURES="N/A"
+          ERRORS="N/A"
+          SKIPPED="N/A"
+
+          if [ -n "$XML_PATH" ] && [ -f "$XML_PATH" ]; then
+            SUITE_LINE="$(grep -m1 -E '<testsuite ' "$XML_PATH" || true)"
+            TESTS="$(echo "$SUITE_LINE" | sed -n 's/.*tests="\([0-9]\+\)".*/\1/p')"
+            FAILURES="$(echo "$SUITE_LINE" | sed -n 's/.*failures="\([0-9]\+\)".*/\1/p')"
+            ERRORS="$(echo "$SUITE_LINE" | sed -n 's/.*errors="\([0-9]\+\)".*/\1/p')"
+            SKIPPED="$(echo "$SUITE_LINE" | sed -n 's/.*skipped="\([0-9]\+\)".*/\1/p')"
+
+            TESTS="${TESTS:-N/A}"
+            FAILURES="${FAILURES:-N/A}"
+            ERRORS="${ERRORS:-N/A}"
+            SKIPPED="${SKIPPED:-N/A}"
+          fi
+
+          {
+            echo "body<<EOF"
+            echo "### ${ICON} ProjectAirSim Self-Hosted CI"
+            echo
+            echo "- Status: **${JOB_STATUS}**"
+            echo "- Pytest results: tests=${TESTS}, failures=${FAILURES}, errors=${ERRORS}, skipped=${SKIPPED}"
+            echo "- Artifacts: [test-results](${RUN_URL})"
+            echo
+            echo "[View full workflow run](${RUN_URL})"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Comment results on PR
+        if: always()
+        continue-on-error: true
+        uses: actions/github-script@v7
+        env:
+          COMMENT_BODY: ${{ steps.ci_summary.outputs.body }}
+        with:
+          script: |
+            const marker = '<!-- projectairsim-self-hosted-ci -->';
+            const issue_number = context.payload.pull_request?.number;
+            if (!issue_number) {
+              core.info('No pull_request context found; skipping comment.');
+              return;
+            }
+
+            const body = `${marker}\n${process.env.COMMENT_BODY}`;
+            const { owner, repo } = context.repo;
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+              per_page: 100,
+            });
+
+            const existing = comments.find(
+              (c) => c.user?.type === 'Bot' && typeof c.body === 'string' && c.body.includes(marker)
+            );
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body,
+              });
+            }

client/python/projectairsim/tests/test_camera_benchmarker.py

@@ -15,6 +15,10 @@
 import cv2
 
 
+BENCHMARK_IMAGE_TIMEOUT_SEC = 30
+BENCHMARK_CANCEL_TIMEOUT_SEC = 5
+
+
 class CameraBenchmarker:
     def __init__(self, image_mode: str, tgt_num_images: int):
         self.tgt_num_images = tgt_num_images
@@ -53,6 +57,26 @@ def process_image(self, image):
                 f"{self.num_images} {self.image_mode} images"
             )
 
+    async def wait_for_target_images(self, poll_interval: float = 0.01):
+        deadline = time.time() + BENCHMARK_IMAGE_TIMEOUT_SEC
+        while self.num_images < self.tgt_num_images:
+            if time.time() >= deadline:
+                raise AssertionError(
+                    "Timed out waiting for "
+                    f"{self.tgt_num_images} {self.image_mode} images; "
+                    f"received {self.num_images}"
+                )
+            await asyncio.sleep(poll_interval)
+
+    async def cancel_and_wait_for_move_task(self, move_task):
+        self.projectairsim_drone.cancel_last_task()
+        try:
+            await asyncio.wait_for(move_task, timeout=BENCHMARK_CANCEL_TIMEOUT_SEC)
+        except asyncio.TimeoutError as err:
+            raise AssertionError(
+                "Timed out waiting for cancelled benchmark move task to finish"
+            ) from err
+
     async def run_benchmark_pubsub(self):
         # Subscribe to images
         if self.image_mode == "rgb":
@@ -73,27 +97,34 @@ async def run_benchmark_pubsub(self):
             v_north=0.0, v_east=0.0, v_down=-1.0, duration=300
         )
 
-        while self.num_images < self.tgt_num_images:
-            await asyncio.sleep(0.01)
-
-        # Unsubscribe to images
-        self.projectairsim_client.unsubscribe(image_topic)
-
-        self.projectairsim_drone.cancel_last_task()
-        await move_task  # join awaited move_by_velocity_async Task now that it's cancelled
+        try:
+            await self.wait_for_target_images()
+        finally:
+            # Unsubscribe and cancel even when the benchmark times out.
+            self.projectairsim_client.unsubscribe(image_topic)
+            await self.cancel_and_wait_for_move_task(move_task)
 
     async def run_benchmark_reqrep(self):
         # Command the Drone to move "Up" in NED coordinate system for a long time
         move_task = await self.projectairsim_drone.move_by_velocity_async(
             v_north=0.0, v_east=0.0, v_down=-1.0, duration=300
         )
 
-        while self.num_images < self.tgt_num_images:
-            images = self.projectairsim_drone.get_images("DownCamera", [ImageType.SCENE])
-            self.process_image(images[ImageType.SCENE])
-
-        self.projectairsim_drone.cancel_last_task()
-        await move_task  # join awaited move_by_velocity_async Task now that it's cancelled
+        deadline = time.time() + BENCHMARK_IMAGE_TIMEOUT_SEC
+        try:
+            while self.num_images < self.tgt_num_images:
+                if time.time() >= deadline:
+                    raise AssertionError(
+                        "Timed out waiting for "
+                        f"{self.tgt_num_images} {self.image_mode} images; "
+                        f"received {self.num_images}"
+                    )
+                images = self.projectairsim_drone.get_images(
+                    "DownCamera", [ImageType.SCENE]
+                )
+                self.process_image(images[ImageType.SCENE])
+        finally:
+            await self.cancel_and_wait_for_move_task(move_task)
 
     def cleanup(self):
         self.projectairsim_drone.disarm()
@@ -108,16 +139,17 @@ async def image_benchmark_main(image_mode, transport_mode, tgt_num_images):
     )
     benchmarker.initialize()
 
-    if transport_mode == "pubsub":
-        await benchmarker.run_benchmark_pubsub()
-    elif transport_mode == "reqrep":
-        await benchmarker.run_benchmark_reqrep()
-    else:
-        warnings.warn("Invalid transport mode.")
-        return
-
-    benchmarker.cleanup()
-    return benchmarker
+    try:
+        if transport_mode == "pubsub":
+            await benchmarker.run_benchmark_pubsub()
+        elif transport_mode == "reqrep":
+            await benchmarker.run_benchmark_reqrep()
+        else:
+            warnings.warn("Invalid transport mode.")
+            return
+        return benchmarker
+    finally:
+        benchmarker.cleanup()
 
 
 def print_benchmark_result(benchmarker, image_mode, transport_mode):