api-workflow.yml

name: Clue API

on:
  push:
    branches:
      - main
      - develop
      - rc/*
      - patch/*
    paths:
      - "api/**"
      - "!api/LICENSE"
      - "!api/README.md"
      - "!api/docs/**"
      - "!api/build_scripts/generate_classes.py"
      - .github/workflows/api-workflow.yml
    tags:
      - v*
  pull_request:
    branches:
      - main
      - develop
      - rc/*
      - patch/*
    paths:
      - "api/**"
      - "!api/LICENSE"
      - "!api/README.md"
      - "!api/docs/**"
      - "!api/build_scripts/generate_classes.py"
      - .github/workflows/api-workflow.yml
      - "!api/plugin/**"
  workflow_dispatch: {}

permissions:
  contents: read
  pull-requests: write

jobs:
  formatting:
    name: Validate Clue API Code Quality

    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: api
    steps:
      - name: Checkout develop
        uses: actions/checkout@v4
        with:
          ref: develop
      - name: Checkout current ref
        uses: actions/checkout@v4
      - name: Setup Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Poetry
        run: python -m pip install poetry
      - uses: actions/cache@v3
        name: Define a cache for the virtual environment based on the dependencies lock file
        with:
          path: ./api/.venv
          key: venv-${{ hashFiles('api/poetry.lock') }}
      - name: Check for poetry config integrity
        run: |
          poetry --version
          poetry env use 3.12
          poetry check
          poetry env info
      - name: Install Dependencies
        run: |
          poetry install --verbose --with test
      - name: Run ruff formatting checks
        run: |
          poetry run ruff format clue --diff
      - name: Run ruff linter checks
        run: |
          poetry run ruff check clue --output-format=github
      - name: Run type checking
        run: |
          poetry run type_check
      - name: "Update poetry version"
        run: |
          poetry run python build_scripts/set_version.py
        env:
          BRANCH: ${{ github.ref }}
          BUILD_ID: ${{ github.run_number }}
  test:
    runs-on: ubuntu-latest

    name: Run Clue API Unit Tests

    needs: formatting

    defaults:
      run:
        working-directory: api
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
    steps:
      - name: Get base depth of PR
        if: github.event_name == 'pull_request'
        id: base-depth
        working-directory: .
        run: |
          echo "base-depth=$(expr ${{ github.event.pull_request.commits }} + 1)" >> $GITHUB_OUTPUT
      - name: Checkout current ref
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: ${{ steps.base-depth.outputs.base-depth }}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Check for valid merge base
        run: |
          git status
          git diff $BASE_SHA..$HEAD_SHA
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
      - name: Spin up test dependencies
        uses: hoverkraft-tech/compose-action@v2.2.0
        with:
          compose-file: "api/dev/docker-compose.yml"
          up-flags: "--build"
      - name: Install Poetry
        uses: abatilo/actions-poetry@v4
      - uses: actions/cache@v3
        name: Define a cache for the virtual environment based on the dependencies lock file
        with:
          path: ./api/.venv
          key: venv-${{ hashFiles('api/poetry.lock') }}
      - name: Setup Environment
        run: |
          sudo mkdir -p /etc/clue/conf/
          sudo mkdir -p /etc/clue/lookups/
          sudo mkdir -p /var/log/clue
          sudo chmod a+rw /etc/clue/conf/
          sudo chmod a+rw /etc/clue/lookups/
          sudo chmod a+rw /var/log/clue
          cp build_scripts/classification.yml /etc/clue/conf/classification.yml
          cp test/unit/config.yml /etc/clue/conf/config.yml
      - name: Install Dependencies
        run: |
          poetry install --all-extras -vvv --with test
      - name: Ensure keycloak is healthy
        run: |
          poetry run python build_scripts/keycloak_health.py
      - name: Run tests
        id: tests
        continue-on-error: true
        run: |
          poetry run test
        env:
          WRITE_MARKDOWN: "true"

      - name: Add Failed Test PR Comment
        uses: marocchino/sticky-pull-request-comment@v2
        if: github.event_name == 'pull_request' && steps.tests.outcome == 'failure'
        # https://github.com/marocchino/sticky-pull-request-comment/issues/227
        continue-on-error: true
        with:
          path: api/test-results.md
          header: api

      - name: Coverage Results
        if: steps.tests.outcome == 'success'
        run: |
          git diff --name-only $BASE_SHA..$HEAD_SHA
          git diff $BASE_SHA..$HEAD_SHA > diff.txt
          poetry run coverage_report
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          GIT_BRANCH: ${{ github.event.pull_request.head.ref }}

      - name: Add Coverage PR Comment
        uses: marocchino/sticky-pull-request-comment@v2
        if: steps.tests.outcome == 'success' && github.event_name == 'pull_request' && matrix.python-version == '3.12'
        # https://github.com/marocchino/sticky-pull-request-comment/issues/227
        continue-on-error: true
        with:
          path: api/coverage-results.md
          header: api

      - name: Force Failure if Tests Failed
        if: steps.tests.outcome == 'failure'
        run: |
          exit 1

  test_docker_build:
    if: github.event_name == 'pull_request'

    runs-on: ubuntu-latest

    name: Test Clue API Docker Image Build

    needs:
      - formatting
      - test

    defaults:
      run:
        working-directory: ./api

    steps:
      - name: Checkout current ref
        uses: actions/checkout@v4
      - name: Install Poetry
        uses: abatilo/actions-poetry@v4
      - uses: actions/cache@v3
        name: Define a cache for the virtual environment based on the dependencies lock file
        with:
          path: ./api/.venv
          key: venv-${{ hashFiles('api/poetry.lock') }}
      - name: Install Dependencies and Build
        run: |
          cp -r ../docs .
          poetry install
          poetry build
      - name: Extract Clue API version
        id: version
        run: |
          echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Test build API Docker image (no push)
        uses: docker/build-push-action@v3
        with:
          context: api
          file: api/Dockerfile
          push: false
          tags: clue-api:test
          build-args: |
            version=${{ steps.version.outputs.version }}
            branch=${{ github.ref }}
            commit=${{ github.sha }}

  push_to_pypi:
    if: github.event_name != 'pull_request'

    runs-on: ubuntu-latest

    name: Build Clue API Python Package

    needs:
      - formatting
      - test

    defaults:
      run:
        working-directory: ./api

    steps:
      - name: Checkout develop
        uses: actions/checkout@v4
        with:
          ref: develop
      - name: Checkout current ref
        uses: actions/checkout@v4
      - name: Install Poetry
        uses: abatilo/actions-poetry@v4
      - uses: actions/cache@v3
        name: Define a cache for the virtual environment based on the dependencies lock file
        with:
          path: ./api/.venv
          key: venv-${{ hashFiles('api/poetry.lock') }}
      - name: Install Dependencies
        run: |
          poetry install
      - name: "Update poetry version"
        run: |
          poetry run python build_scripts/set_version.py
        env:
          BRANCH: ${{ github.ref }}
          BUILD_ID: ${{ github.run_number }}
      - name: Build Package
        run: |
          poetry install
          poetry build
      - name: Configure Poetry for PyPI
        run: poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
      - name: Publish the package to PyPI
        if: github.event_name != 'pull_request'
        run: |
          poetry publish

  push_to_registry:
    if: github.event_name != 'pull_request'

    runs-on: ubuntu-latest

    name: Build Clue API Docker Image

    needs:
      - formatting
      - test

    defaults:
      run:
        working-directory: ./api

    permissions:
      id-token: write
      contents: read
      attestations: write

    steps:
      - name: Checkout develop
        uses: actions/checkout@v4
        with:
          ref: develop
      - name: Checkout current ref
        uses: actions/checkout@v4
      - name: Install Poetry
        uses: abatilo/actions-poetry@v4
      - uses: actions/cache@v3
        name: Define a cache for the virtual environment based on the dependencies lock file
        with:
          path: ./api/.venv
          key: venv-${{ hashFiles('api/poetry.lock') }}
      - name: Install Dependencies and Build
        run: |
          cp -r ../docs .
          poetry install
          poetry build
      - name: Extract Clue API version
        id: version
        run: |
          echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
      - name: Log in to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: cccs/clue-api
          tags: |
            type=raw,event=branch,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
            type=raw,event=branch,value=nightly,enable=${{ github.ref == format('refs/heads/{0}', 'develop') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags') }}
            type=semver,pattern={{version}}
            type=raw,value=${{steps.version.outputs.version}}_{{branch}}
            type=raw,value=${{steps.version.outputs.version}}_{{branch}}_{{sha}}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Build and push API Docker image
        id: push
        uses: docker/build-push-action@v3
        with:
          context: api
          file: api/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            version=${{ steps.meta.outputs.version }}
            branch=${{ github.ref }}
            commit=${{ github.sha }}
      - name: Generate artifact attestation
        uses: actions/attest-build-provenance@v2
        with:
          subject-name: index.docker.io/cccs/clue-api
          subject-digest: ${{ steps.push.outputs.digest }}
          push-to-registry: true