ci.yml

# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license.

on:
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  # Pass additional arguments to cargo using `--config build.rustflags=["...", "..."]`.
  #
  # This ensures that we do not overwrite the parameters in `.cargo/config.toml` unnecessarily.
  RUST_CONFIG: 'build.rustflags=["-Dwarnings"]'
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always
  # The features we want to explicitly test. For example, the `flatbuffers-build` feature
  # of `diskann-quantization` requires additional setup and so must not be included by default.
  DISKANN_FEATURES: "virtual_storage,spherical-quantization,product-quantization,tracing,experimental_diversity_search,disk-index,flatbuffers,linalg,codegen"

  # Intel SDE version used for baseline and AVX-512 emulation jobs.
  SDE_VERSION: "sde-external-10.8.0-2026-03-15-lin"

defaults:
  run:
    shell: bash

permissions:
  contents: read

jobs:
  # Branch-protection gate
  #
  # Add "CI gate" as the sole required status check in branch protection.
  #
  # How it works:
  #
  # * If no upstream job fails or is cancelled, the `if` condition is `false`
  #   and this job is *skipped*. GitHub treats a skipped required check as
  #   passing.
  #
  #   Note: jobs that are conditionally skipped (e.g. `coverage` excludes
  #   Dependabot) are neither failed nor cancelled, so the gate will still
  #   pass. This is intentional -- conditional skips represent expected
  #   behavior, not failures.
  #
  #   See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks
  #
  # * If any upstream job fails or is cancelled, the `if` condition is `true`.
  #   This job runs and exits 1, blocking the merge.
  #
  # All merge-blocking jobs should be listed in `needs` below.
  ci-gate:
    name: CI gate
    runs-on: ubuntu-latest
    needs:
      - fmt
      - clippy-default-features
      - clippy-features
      - clippy-no-default-features
      - basics
      - codeql
      - baseline
      - sde
      - test-workspace
      - test-workspace-features
      - coverage
    if: ${{ always() && (cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure')) }}
    steps:
      - name: Gate failure
        run: |
          echo "::error::One or more required CI jobs failed or were cancelled."
          exit 1

  # Basic checks that must pass before we kick off more expensive tests.
  basics:
    name: basic checks
    runs-on: ubuntu-latest
    needs:
      - fmt
      - clippy-default-features
      - clippy-features
      - clippy-no-default-features
      # TODO: Re-enable docs check later
      # - docs
    steps:
      - run: exit 0

  fmt:
    name: format check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Rust
        run: rustup show && rustup component add rustfmt
      - uses: Swatinem/rust-cache@v2
      # Check fmt
      - name: "cargo fmt --check"
        run: |
          if ! cargo fmt --all --check; then
            printf "Please run \`cargo fmt --all\` to fix rustfmt errors.\n" >&2
            exit 1
          fi

  clippy-default-features:
    strategy:
      matrix:
        runner:
          - ubuntu-latest
          - ubuntu-24.04-arm
          - windows-latest

      fail-fast: false
    name: clippy-default-features (${{ matrix.runner }})
    runs-on: ${{ matrix.runner }}
    steps:
      - uses: actions/checkout@v4
      - name: Install Rust
        run: rustup show && rustup component add clippy
      - uses: Swatinem/rust-cache@v2
      - name: "clippy --workspace --all-targets"
        run: cargo clippy --locked --workspace --all-targets --no-deps --config "$RUST_CONFIG" -- -Dwarnings

  clippy-features:
    strategy:
      matrix:
        runner:
          - ubuntu-latest
          - ubuntu-24.04-arm
          - windows-latest
      fail-fast: false
    name: clippy-features (${{ matrix.runner }})
    runs-on: ${{ matrix.runner }}
    steps:
      - uses: actions/checkout@v4
      - name: Install Rust
        run: rustup show && rustup component add clippy
      - uses: Swatinem/rust-cache@v2
      - name: "clippy --workspace --all-targets"
        run: |
          set -euxo pipefail
          cargo clippy --locked --workspace \
            --all-targets \
            --no-deps \
            --features ${{ env.DISKANN_FEATURES }} \
            --config "$RUST_CONFIG" \
            -- -Dwarnings


  clippy-no-default-features:
    name: clippy (no default features)
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install Rust
        run: rustup show && rustup component add clippy

      - uses: Swatinem/rust-cache@v2

      - name: Clippy - select crates (no default features)
        run: |
          set -euxo pipefail
          crates=(
            # Core crates
            diskann
            diskann-providers
            diskann-disk
            diskann-quantization
            diskann-utils
            # Benchmark/tools crates
            diskann-benchmark-core
            diskann-benchmark-runner
            diskann-benchmark
            diskann-tools
          )
          for crate in "${crates[@]}"; do
            cargo clippy --locked --package "$crate" \
              --no-default-features \
              --all-targets \
              --profile test \
              --no-deps \
              --config "$RUST_CONFIG" \
              -- -Dwarnings
          done

  codeql:
    name: CodeQL security analysis
    runs-on: ubuntu-latest
    permissions:
      contents: read
      security-events: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: rust

      - name: Install Rust
        run: rustup show

      - uses: Swatinem/rust-cache@v2

      - name: Build workspace
        run: cargo build --workspace --locked --profile test

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:rust"

  # TODO: Re-enable docs check later
  # docs:
  #   name: docs
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Install Rust
  #       run: rustup show
  #     - uses: Swatinem/rust-cache@v2
  #     - name: "doc --workspace --no-deps"
  #       run: cargo doc --workspace --no-deps --document-private-items
  #       env:
  #         RUSTDOCFLAGS: -Dwarnings


  # Test micro-architecture detection and dispatching.
  #
  # The crate `diskann-wide` is the primary driver for this mechanism.
  #
  # Verify that code compiled for the `x86-64` baseline does not accidentally emit
  # AVX/AVX2 instructions. We compile with `-Ctarget-cpu=x86-64` and run the full
  # test suite for `diskann-wide`, `diskann-vector`, and `diskann-quantization` under
  # Intel SDE configured as a Nehalem CPU.
  #
  # SDE will abort if any unrecognised instruction (e.g. AVX) is executed.
  #
  # EXCLUDED TESTS
  #
  # * compile-tests: These don't behave well under SDE.
  # * pivots::tests::run_test_happy_path: The double-whammy of emulated SIMD and emulated
  #   architecture make this test take too long.
  baseline:
    needs: basics
    name: sde-baseline-tests
    runs-on: ubuntu-latest
    env:
      # Compile for the x86-64 baseline — no AVX, no AVX2.
      RUSTFLAGS: "-Dwarnings -Ctarget-cpu=x86-64"
      # Run every test binary through SDE emulating a Nehalem CPU.
      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "${{ github.workspace }}/intel-sde/sde64 -nhm --"

    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        run: rustup show

      - uses: Swatinem/rust-cache@v2

      - name: Cache Intel SDE
        id: cache-sde
        uses: actions/cache@v4
        with:
          path: intel-sde
          key: intel-${{ env.SDE_VERSION }}

      - name: Download Intel SDE
        if: steps.cache-sde.outputs.cache-hit != 'true'
        run: |
          set -euxo pipefail
          SDE_URL="https://downloadmirror.intel.com/915934/${SDE_VERSION}.tar.xz"
          wget -qO intel-sde.tar.xz "$SDE_URL"
          mkdir -p intel-sde
          tar xf intel-sde.tar.xz --strip-components=1 -C intel-sde
          rm intel-sde.tar.xz

      - name: Verify SDE installation
        run: ./intel-sde/sde64 --version

      - name: "SDE Baseline Tests (Nehalem)"
        run: |
          set -euxo pipefail
          cargo test --locked \
              --package diskann-wide \
              --package diskann-vector \
              --package diskann-quantization \
              -- --skip compile_tests \
                 --skip pivots::tests::run_test_happy_path


  # This test validates AVX-512 code paths using Intel SDE (Software Development Emulator).
  #
  # SDE emulates a Sapphire Rapids CPU, which supports all AVX-512 extensions required by
  # the diskann-wide V4 backend.
  #
  # The V4 functions are compiled with `#[target_feature(enable = ...)]` regardless of the
  # target CPU, and runtime dispatch detects the SDE-emulated features via `cpuid`. No
  # special RUSTFLAGS are needed beyond the defaults.
  #
  # EXCLUDED TESTS
  #
  # * compile-tests: These don't behave well under SDE.
  sde:
    needs: basics
    name: sde-avx512-tests
    runs-on: ubuntu-latest
    env:
      # Use SDE as the test runner so cargo test automatically runs binaries under emulation.
      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "${{ github.workspace }}/intel-sde/sde64 -spr --"
      # Force diskann-wide tests to require V4 architecture.
      WIDE_TEST_MIN_ARCH: "x86-64-v4"

    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        run: rustup show

      - uses: Swatinem/rust-cache@v2

      - name: Cache Intel SDE
        id: cache-sde
        uses: actions/cache@v4
        with:
          path: intel-sde
          key: intel-${{ env.SDE_VERSION }}

      - name: Download Intel SDE
        if: steps.cache-sde.outputs.cache-hit != 'true'
        run: |
          set -euxo pipefail
          SDE_URL="https://downloadmirror.intel.com/915934/${SDE_VERSION}.tar.xz"
          wget -qO intel-sde.tar.xz "$SDE_URL"
          mkdir -p intel-sde
          tar xf intel-sde.tar.xz --strip-components=1 -C intel-sde
          rm intel-sde.tar.xz

      - name: Verify SDE installation
        run: ./intel-sde/sde64 --version

      - name: "SDE Tests (AVX-512)"
        run: |
          set -euxo pipefail
          cargo test --locked \
            --package diskann-wide \
            --package diskann-vector \
            --package diskann-quantization \
            -- --skip compile_tests

  test-workspace:
    needs: basics
    name: test workspace
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - windows-latest
          - ubuntu-latest
          - ubuntu-24.04-arm

    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install Rust
        run: rustup show

      - name: Install cargo-nextest
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-nextest

      - uses: Swatinem/rust-cache@v2

      - name: test workspace with nextest
        run: |
          set -euxo pipefail
          cargo nextest run --locked --workspace --config "$RUST_CONFIG"
          cargo test --locked --doc --workspace --config "$RUST_CONFIG"

  test-workspace-features:
    needs: basics
    name: test workspace (all features)
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - windows-latest
          - ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install Rust
        run: rustup show

      - name: Install cargo-nextest
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-nextest

      - uses: Swatinem/rust-cache@v2

      - name: test workspace with nextest
        run: |
          set -euxo pipefail
          cargo nextest run --locked --workspace \
            --config "$RUST_CONFIG" \
            --features ${{ env.DISKANN_FEATURES }}

          cargo test --locked --doc --workspace --config "$RUST_CONFIG"

  coverage:
    needs: basics
    name: code coverage
    runs-on: ubuntu-latest
    if: github.actor != 'dependabot[bot]'
    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true
          fetch-depth: 0

      - name: Install Rust
        run: rustup show && rustup component add llvm-tools-preview

      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-llvm-cov

      - name: Install cargo-nextest
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-nextest

      - uses: Swatinem/rust-cache@v2

      - name: Generate code coverage
        run: |
          cargo llvm-cov nextest --locked \
            --config "$RUST_CONFIG" \
            --workspace \
            --lcov --output-path lcov.info

      - name: Generate miri code coverage
        env:
          RUSTFLAGS: "--cfg=miri"
        run: |
          cargo +nightly llvm-cov nextest --locked \
            --package diskann-quantization \
            --lcov --output-path lcov_miri.info

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          files: lcov.info
          fail_ci_if_error: true
          flags: unittests
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

      - name: Upload miri coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          files: lcov_miri.info
          fail_ci_if_error: true
          flags: miri
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  # LLVM IR bloat check: compare monomorphization cost against baseline.
  # This job is not part of the required CI gate, so it won't block PR merges.
  llvm-lines:
    if: github.event_name == 'pull_request'
    needs: basics
    name: LLVM Lines Regression Check
    runs-on: ubuntu-latest
    env:
      LLVM_LINES_GROWTH_THRESHOLD: 5
    steps:
      - name: Checkout current branch
        uses: actions/checkout@v4
        with:
          path: pr

      - name: Checkout base ref
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: baseline

      - name: Install Rust
        shell: bash
        run: rustup show
        working-directory: pr

      - name: Install cargo-llvm-lines
        run: cargo install cargo-llvm-lines --locked

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: |
            pr -> target
            baseline -> target

      - name: Generate baseline LLVM lines
        working-directory: baseline
        run: cargo llvm-lines --package diskann-benchmark --all-features --release | head -100 | tee ../baseline-llvm-lines.txt

      - name: Generate current LLVM lines
        working-directory: pr
        run: cargo llvm-lines --package diskann-benchmark --all-features --release | head -100 | tee ../current-llvm-lines.txt

      - name: Compare LLVM lines
        run: bash pr/.github/scripts/compare-llvm-lines.sh baseline-llvm-lines.txt current-llvm-lines.txt

      - name: Check LLVM lines growth threshold
        run: |
          baseline_total=$(awk '/\(TOTAL\)/{print $1}' baseline-llvm-lines.txt)
          current_total=$(awk '/\(TOTAL\)/{print $1}' current-llvm-lines.txt)
          growth=$(( (current_total - baseline_total) * 100 / baseline_total ))

          if [ "$growth" -gt "$LLVM_LINES_GROWTH_THRESHOLD" ]; then
            echo "::error::LLVM IR grew ${growth}% ($baseline_total → $current_total lines), threshold is ${LLVM_LINES_GROWTH_THRESHOLD}%"
            exit 1
          fi