Stop capping workspace fan-out width, narrow inspect_code bridge disp… #310
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # TODO: lock files on all platforms | |
| # TODO: test with all supported python versions | |
| name: CI | |
| on: | |
| merge_group: | |
| push: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| publish_testpypi: | |
| description: 'Publish to TestPyPI' | |
| required: true | |
| type: boolean | |
| default: false | |
| # Cancel superseded runs only for PR pushes (rapid commits to the same branch). | |
| # Never cancel push/tag/merge_group runs — a tag push drives the PyPI publish | |
| # steps below and a merge_group run backs a required check in the merge queue; | |
| # either could be silently aborted by an unrelated event sharing the ref. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| # python version for dev workspace | |
| DEV_WORKSPACE_PYTHON_VERSION: '3.14' | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| # A cold `prepare-envs` over the whole workspace plus the full check suite does | |
| # not fit in 40 minutes on the slower matrix legs. The venvs cache keeps the | |
| # happy path short; this budget only covers the cold run. | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-24.04, macos-15, windows-2022] | |
| include: | |
| - os: ubuntu-24.04 | |
| name: Linux | |
| venv_bin: bin | |
| - os: macos-15 | |
| name: macOS | |
| venv_bin: bin | |
| - os: windows-2022 | |
| name: Windows | |
| venv_bin: Scripts | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| # setuptools-scm uses tags to get the current version, fetch history and tags | |
| # to get correct version | |
| fetch-depth: 0 | |
| fetch-tags: 'true' | |
| - name: Determine FineCode log level | |
| run: | | |
| # WM/ER diagnostic logs are streamed to the job log (verbose is auto-enabled | |
| # in CI). Keep them at INFO normally; raise to DEBUG only when the job is | |
| # re-run with "Enable debug logging" (GitHub sets RUNNER_DEBUG=1). This keeps | |
| # the debug-vs-info decision in CI config — FineCode just honors --log-level. | |
| if [ "${RUNNER_DEBUG:-0}" = "1" ]; then | |
| echo "FINECODE_LOG_LEVEL=DEBUG" >> "$GITHUB_ENV" | |
| else | |
| echo "FINECODE_LOG_LEVEL=INFO" >> "$GITHUB_ENV" | |
| fi | |
| - name: Set up Python ${{ env.DEV_WORKSPACE_PYTHON_VERSION }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.DEV_WORKSPACE_PYTHON_VERSION }} | |
| # Every package in the monorepo has its own .venvs/ (root dev_workspace plus one | |
| # per project for envs created by `prepare-envs`, e.g. dev_no_runtime, runtime). | |
| # Exact-match only (no restore-keys): setup-dev-workspace.sh and prepare-envs | |
| # skip reinstalling whenever a restored venv already looks valid, so a stale | |
| # partial-match restore could mask a dependency that was added since the cache | |
| # was written. | |
| - name: Restore venvs cache | |
| id: venvs_cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: | | |
| .venvs | |
| **/.venvs | |
| key: ${{ runner.os }}-venvs-${{ hashFiles('**/pyproject.toml', '**/preset.toml') }} | |
| - name: Install dependencies | |
| id: install | |
| run: | | |
| # CI must exercise this branch's local source, so finecode and its sibling | |
| # packages need an editable install, not a released version from PyPI — see | |
| # docs/guides/developing-finecode.md#continuous-integration. | |
| sh scripts/setup-dev-workspace.sh | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| shell: bash | |
| - name: Inspect code | |
| if: ${{ !cancelled() && steps.install.outcome == 'success' }} | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" inspect_code | |
| shell: bash | |
| - name: Audit code | |
| if: ${{ !cancelled() && steps.install.outcome == 'success' }} | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" audit_code | |
| shell: bash | |
| - name: Check formatting | |
| if: ${{ !cancelled() && steps.install.outcome == 'success' }} | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" check_formatting | |
| shell: bash | |
| - name: Build artifacts | |
| id: build | |
| if: runner.os == 'Linux' && !cancelled() && steps.install.outcome == 'success' | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" build_artifact | |
| shell: bash | |
| - name: Run unit tests | |
| if: ${{ !cancelled() && steps.install.outcome == 'success' }} | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" run_tests | |
| shell: bash | |
| - name: Publish to TestPyPI and verify | |
| if: runner.os == 'Linux' && github.event_name == 'workflow_dispatch' && inputs.publish_testpypi | |
| env: | |
| FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__REPOSITORIES: '[{"name": "testpypi", "index_url": "https://test.pypi.org/simple/", "upload_url": "https://test.pypi.org/legacy/"}]' | |
| FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__CREDENTIALS_BY_REPOSITORY: '{"testpypi": {"username": "${{ secrets.TESTPYPI_USERNAME }}", "password": "${{ secrets.TESTPYPI_PASSWORD }}"}}' | |
| run: | | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run \ | |
| --log-level="$FINECODE_LOG_LEVEL" \ | |
| --map-payload-fields="src-artifact-def-path,dist-artifact-paths" \ | |
| publish_and_verify_artifact \ | |
| --src-artifact-def-path="build_artifact.src_artifact_def_path" \ | |
| --dist-artifact-paths="build_artifact.build_output_paths" | |
| shell: bash | |
| - name: Publish to PyPI and verify | |
| if: runner.os == 'Linux' && startsWith(github.ref, 'refs/tags/') | |
| env: | |
| FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__REPOSITORIES: '[{"name": "pypi", "index_url": "https://pypi.org/simple/", "upload_url": "https://upload.pypi.org/legacy/"}]' | |
| FINECODE_CONFIG_PUBLISH_AND_VERIFY_ARTIFACT__INIT_REPOSITORY_PROVIDER__CREDENTIALS_BY_REPOSITORY: '{"pypi": {"username": "${{ secrets.PYPI_USERNAME }}", "password": "${{ secrets.PYPI_PASSWORD }}"}}' | |
| run: | | |
| # TODO: make sure git tag exists (for manual trigger) | |
| source .venvs/dev_workspace/${{ matrix.venv_bin }}/activate | |
| python -m finecode run \ | |
| --log-level="$FINECODE_LOG_LEVEL" \ | |
| --map-payload-fields="src-artifact-def-path,dist-artifact-paths" \ | |
| publish_and_verify_artifact \ | |
| --src-artifact-def-path="build_artifact.src_artifact_def_path" \ | |
| --dist-artifact-paths="build_artifact.build_output_paths" | |
| shell: bash | |
| # TODO: try to replace by finecode action | |
| - name: Store the distribution packages | |
| uses: actions/upload-artifact@v6 | |
| if: runner.os == 'Linux' | |
| with: | |
| name: python-package-distributions | |
| path: dist/ | |
| # Save the venvs explicitly instead of relying on actions/cache's own post-step: | |
| # that post-step skips the save when the job fails, which throws away a perfectly | |
| # good cold install just because a later check or test failed. The only | |
| # precondition here is that the install itself succeeded; on an exact key match | |
| # there is nothing new to save. | |
| # The key is reused from the restore step, never recomputed: by now every .venvs/ | |
| # is populated, so a `**/pyproject.toml` glob would walk all of site-packages | |
| # (hashFiles times out at 120s) and could hash venv-internal files the restore | |
| # key never saw, saving under a key no later run would ever restore. | |
| - name: Save venvs cache | |
| if: ${{ always() && steps.install.outcome == 'success' && steps.venvs_cache.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: | | |
| .venvs | |
| **/.venvs | |
| key: ${{ steps.venvs_cache.outputs.cache-primary-key }} | |
| audit-private: | |
| name: Audit (private layer) | |
| runs-on: ubuntu-24.04 | |
| # This job pays a cold prepare-envs over ~74 projects and then a | |
| # workspace-wide audit_code, which is slow by design (see | |
| # docs/guides/developing-finecode.md "Running checks"), so it carries its own | |
| # budget rather than inheriting the build matrix's. | |
| timeout-minutes: 60 | |
| # `secrets` is NOT available in jobs.<job_id>.if -- only github, needs, | |
| # vars, inputs. An unavailable context evaluates to empty, so a secrets test | |
| # here would skip the job forever without ever erroring. Fork PRs are | |
| # excluded with the `github` context alone; the credentials test is the | |
| # `env` gate below, which step `if`s can read. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| env: | |
| # jobs.<job_id>.env DOES have the secrets context, and steps' `if` has | |
| # `env`. This one line is what bridges the two. | |
| HAS_PRIVATE_CLONE_APP: ${{ secrets.PRIVATE_CLONE_APP_ID != '' && secrets.PRIVATE_CLONE_APP_PRIVATE_KEY != '' }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| # setuptools-scm uses tags to get the current version, fetch history and tags | |
| # to get correct version | |
| fetch-depth: 0 | |
| fetch-tags: 'true' | |
| - name: Report skip reason | |
| if: env.HAS_PRIVATE_CLONE_APP != 'true' | |
| run: echo "::notice::Private-layer audit skipped - App credentials are not available on this run." | |
| - name: Determine FineCode log level | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| # WM/ER diagnostic logs are streamed to the job log (verbose is auto-enabled | |
| # in CI). Keep them at INFO normally; raise to DEBUG only when the job is | |
| # re-run with "Enable debug logging" (GitHub sets RUNNER_DEBUG=1). This keeps | |
| # the debug-vs-info decision in CI config — FineCode just honors --log-level. | |
| if [ "${RUNNER_DEBUG:-0}" = "1" ]; then | |
| echo "FINECODE_LOG_LEVEL=DEBUG" >> "$GITHUB_ENV" | |
| else | |
| echo "FINECODE_LOG_LEVEL=INFO" >> "$GITHUB_ENV" | |
| fi | |
| - name: Set up Python ${{ env.DEV_WORKSPACE_PYTHON_VERSION }} | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.DEV_WORKSPACE_PYTHON_VERSION }} | |
| - name: Mint private-clone token | |
| id: app_token | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| app-id: ${{ secrets.PRIVATE_CLONE_APP_ID }} | |
| private-key: ${{ secrets.PRIVATE_CLONE_APP_PRIVATE_KEY }} | |
| owner: finecode-dev | |
| repositories: fine_knowledge,finecode_internal_experiments | |
| - name: Check out fine_knowledge | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: finecode-dev/fine_knowledge | |
| path: presets/fine_knowledge | |
| token: ${{ steps.app_token.outputs.token }} | |
| # A `git clone` with the token in the URL writes it into the clone's | |
| # .git/config; checkout with persist-credentials disabled does not. | |
| persist-credentials: false | |
| - name: Check out internal experiments | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: finecode-dev/finecode_internal_experiments | |
| path: finecode_internal_experiments | |
| token: ${{ steps.app_token.outputs.token }} | |
| persist-credentials: false | |
| - name: Install the CI private-layer config | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: cp .github/ci/finecode-user.ci.toml finecode-user.toml | |
| - name: Restore venvs cache | |
| id: venvs_cache | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: | | |
| .venvs | |
| **/.venvs | |
| key: ${{ runner.os }}-private-venvs-${{ hashFiles('**/pyproject.toml', '**/preset.toml', 'finecode-user.toml') }} | |
| - name: Install dependencies | |
| id: install | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| # CI must exercise this branch's local source, so finecode and its sibling | |
| # packages need an editable install, not a released version from PyPI — see | |
| # docs/guides/developing-finecode.md#continuous-integration. | |
| sh scripts/setup-dev-workspace.sh | |
| source .venvs/dev_workspace/bin/activate | |
| shell: bash | |
| - name: Inspect code | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| source .venvs/dev_workspace/bin/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" inspect_code | |
| shell: bash | |
| - name: Extract knowledge | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| source .venvs/dev_workspace/bin/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" extract_knowledge | |
| shell: bash | |
| - name: Audit code | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| source .venvs/dev_workspace/bin/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" audit_code | |
| shell: bash | |
| - name: Run unit tests | |
| if: env.HAS_PRIVATE_CLONE_APP == 'true' | |
| run: | | |
| source .venvs/dev_workspace/bin/activate | |
| python -m finecode run --log-level="$FINECODE_LOG_LEVEL" run_tests | |
| shell: bash | |
| # See the matching step in the `build` job for the rationale: save the venvs | |
| # whenever the install succeeded, even if a later check, audit or test failed, | |
| # and reuse the restore step's key rather than recomputing it. | |
| - name: Save venvs cache | |
| if: ${{ always() && env.HAS_PRIVATE_CLONE_APP == 'true' && steps.install.outcome == 'success' && steps.venvs_cache.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: | | |
| .venvs | |
| **/.venvs | |
| key: ${{ steps.venvs_cache.outputs.cache-primary-key }} |