Repository navigation
fix authorize job #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - Pull Request | |
| on: | |
| push: # TEMPORARY: Remove before merging | |
| branches: [ci/split-workflows] # TEMPORARY: Remove before merging | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| branches: [main] | |
| paths-ignore: | |
| - '*.detignore' | |
| - '**/*.detignore' | |
| - '*.example' | |
| - '**/*.example' | |
| - '*.json' | |
| - '**/*.json' | |
| - '*.md' | |
| - '**/*.md' | |
| - '*.png' | |
| - '**/*.png' | |
| - '*.sample' | |
| - '**/*.sample' | |
| - '*.TAG' | |
| - '**/*.TAG' | |
| - '*.yaml' | |
| - '**/*.yaml' | |
| - '*.yml' | |
| - '**/*.yml' | |
| - '!/.github/workflows/**' | |
| - 'docs/**' | |
| - 'LICENSE' | |
| - '.gitignore' | |
| - 'changelog/**' | |
| permissions: | |
| contents: read | |
| packages: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: eval_framework_public | |
| HF_DATASET_CACHE_DIR: /tmp/huggingface_datasets | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup poetry | |
| uses: ./.github/actions/setup-poetry-on-ubuntu | |
| with: | |
| github-repo-token: ${{ secrets.TOKEN_AA_GITHUB_REPO_READER }} | |
| - name: Run Pre-Commit | |
| run: | | |
| poetry run pre-commit install | |
| poetry run pre-commit run -a | |
| - name: Run MyPy | |
| run: | | |
| poetry run mypy src tests utils | |
| build: | |
| needs: [lint] | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: docker:dind | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: eval_framework_public | |
| REPO_OWNER_LC: aleph-alpha-research | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Registry Authentication | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup Docker BuildX | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Build and Push Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER_LC }}/${{ env.IMAGE_NAME }}:latest | |
| test-cpu: | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: derskythe/github-runner-base:ubuntu-noble | |
| needs: [lint] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup poetry | |
| uses: ./.github/actions/setup-poetry-on-ubuntu | |
| - name: Huggingface datasets cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets- | |
| - name: Run tests | |
| run: | | |
| export HF_TOKEN=${{ secrets.HUGGINGFACE_API_KEY }} | |
| export HF_DATASET_CACHE_DIR=${{ env.HF_DATASET_CACHE_DIR }} | |
| export PATH=$HOME/.local/bin:$PATH | |
| poetry run pytest --durations=30 -v -m "not gpu and not cpu_slow and not external_api" | |
| test-cpu-slow: | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: derskythe/github-runner-base:ubuntu-noble | |
| needs: [lint] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup poetry | |
| uses: ./.github/actions/setup-poetry-on-ubuntu | |
| - name: Huggingface datasets cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets- | |
| - name: Run tests | |
| run: | | |
| export HF_TOKEN=${{ secrets.HUGGINGFACE_API_KEY }} | |
| export HF_DATASET_CACHE_DIR=${{ env.HF_DATASET_CACHE_DIR }} | |
| export PATH=$HOME/.local/bin:$PATH | |
| poetry run python -c "import nltk; nltk.download('punkt_tab')" | |
| poetry run pytest -n auto --max-worker-restart=0 --durations=30 -v -m "not gpu and cpu_slow and not external_api" | |
| test-gpu: | |
| needs: [build, test-cpu, test-cpu-slow] | |
| runs-on: EvalFrameworkGPURunner | |
| if: ${{ always() | |
| && needs.test-cpu.result == 'success' | |
| && needs.test-cpu-slow.result == 'success' | |
| && (needs.build.result == 'success' || needs.build.result == 'skipped') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Normalize repo owner to lowercase | |
| run: | | |
| echo "REPO_OWNER_LC=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV | |
| - name: Huggingface datasets cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets- | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull container image | |
| run: | | |
| docker pull "${{ env.REGISTRY }}/${{ env.REPO_OWNER_LC }}/${{ env.IMAGE_NAME }}:latest" | |
| - name: Run GPU tests inside container with GPU access | |
| run: | | |
| timeout 20m docker run --rm --gpus all \ | |
| -v ${{ github.workspace }}:/eval_framework \ | |
| -w /eval_framework \ | |
| -e HF_TOKEN=${{ secrets.HUGGINGFACE_API_KEY }} \ | |
| -e HF_DATASET_CACHE_DIR=${{ env.HF_DATASET_CACHE_DIR }} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER_LC }}/${{ env.IMAGE_NAME }}:latest \ | |
| bash -c 'poetry run pytest --durations=30 -v -m "gpu and not cpu_slow and not external_api and not vllm"' | |
| test-gpu-vllm: | |
| needs: [build, test-cpu, test-cpu-slow, test-gpu] | |
| runs-on: EvalFrameworkGPURunner | |
| if: ${{ always() | |
| && needs.test-cpu.result == 'success' | |
| && needs.test-cpu-slow.result == 'success' | |
| && (needs.build.result == 'success' || needs.build.result == 'skipped') | |
| && (needs.test-gpu.result == 'success' || needs.test-gpu.result == 'skipped') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Normalize repo owner to lowercase | |
| run: | | |
| echo "REPO_OWNER_LC=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV | |
| - name: Huggingface datasets cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets- | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull container image | |
| run: | | |
| docker pull "${{ env.REGISTRY }}/${{ env.REPO_OWNER_LC }}/${{ env.IMAGE_NAME }}:latest" | |
| - name: Run GPU tests inside container with GPU access | |
| run: | | |
| timeout 20m docker run --rm --gpus all \ | |
| -v ${{ github.workspace }}:/eval_framework \ | |
| -w /eval_framework \ | |
| -e HF_TOKEN=${{ secrets.HUGGINGFACE_API_KEY }} \ | |
| -e HF_DATASET_CACHE_DIR=${{ env.HF_DATASET_CACHE_DIR }} \ | |
| -e VLLM_LOGGING_LEVEL=DEBUG \ | |
| -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ | |
| -e VLLM_USE_MODELSCOPE=False \ | |
| -e VLLM_NCCL_SO_PATH="" \ | |
| -e VLLM_USE_TRITON_FLASH_ATTN=0 \ | |
| -e VLLM_DISABLE_CUSTOM_ALL_REDUCE=1 \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER_LC }}/${{ env.IMAGE_NAME }}:latest \ | |
| bash -c 'curl -sSL https://install.python-poetry.org | python3 - && export PATH="$HOME/.local/bin:$PATH" && poetry --version && poetry run pytest --log-cli-level=INFO -v -m "vllm"' |