Nightly #330
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| # Tests plugin against formae main branch to catch SDK regressions before release. | |
| # PEL-maintained plugins only. | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' # 3 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| formae_branch: | |
| description: "Formae branch to build from (default: main)" | |
| required: false | |
| default: "main" | |
| type: string | |
| formae_version: | |
| description: | | |
| Formae version baked into the binary via -ldflags (also propagated | |
| to plugins as FORMAE_VERSION; plugins use it to resolve their PKL | |
| formae schema dep). Defaults to the canonical X.Y.Z derived from | |
| the latest tag on the formae branch — including dev tags, so a | |
| newly tagged 0.86.0-dev.N is picked up automatically. Set this | |
| explicitly only when overriding the auto-detected version. | |
| required: false | |
| default: "" | |
| type: string | |
| # Workflow-level group: holds the lock for the entire run so the matrix | |
| # entries inside conformance-tests share one claim instead of competing. | |
| # Job-level concurrency on a matrix job mass-cancels its own entries | |
| # ("higher priority waiting request" wins for each new pending matrix | |
| # entry). Shared with CI so CI and nightly serialize on the same AWS | |
| # test account. | |
| concurrency: | |
| group: aws-conformance-tests | |
| cancel-in-progress: false | |
| jobs: | |
| # Integration tests run against real AWS resources (Route53, EC2, etc.) | |
| # These are self-contained: each test creates and cleans up its own resources. | |
| test-integration: | |
| # Isolation from formae-e2e's aws-nuke is enforced in the nuke config | |
| # (region split + global allowlist), not via concurrency — GitHub | |
| # concurrency groups are per-repo and cannot serialize across repos. | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: "1.26" | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: NightlyIntegrationTests | |
| - name: Run integration tests | |
| run: make test-integration | |
| # Discover test cases from testdata/ directory for matrix strategy | |
| discover-tests: | |
| needs: [test-integration] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-cases: ${{ steps.discover.outputs.test-cases }} | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Discover test cases | |
| id: discover | |
| run: | | |
| # Find all *.pkl files in testdata/, exclude -update.pkl and -replace.pkl variants, | |
| # and extract the base test case name | |
| TEST_CASES=$(ls testdata/*.pkl 2>/dev/null \ | |
| | xargs -n1 basename \ | |
| | grep -v '\-update\.pkl$' \ | |
| | grep -v '\-replace\.pkl$' \ | |
| | sed 's/\.pkl$//' \ | |
| | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "Discovered test cases: ${TEST_CASES}" | |
| echo "test-cases=${TEST_CASES}" >> "$GITHUB_OUTPUT" | |
| # Clean up test resources before running any conformance tests | |
| pre-cleanup: | |
| needs: [discover-tests] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: NightlyPreCleanup | |
| - name: Clean test resources | |
| run: ./scripts/ci/clean-environment.sh | |
| # Conformance tests run the full CRUD lifecycle against real AWS resources. | |
| # Each test case runs in its own isolated job via matrix strategy. | |
| # Cleanup is handled by separate pre/post-cleanup jobs, NOT per test. | |
| conformance-tests: | |
| needs: [discover-tests, pre-cleanup] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| strategy: | |
| matrix: | |
| test-case: ${{ fromJson(needs.discover-tests.outputs.test-cases) }} | |
| # us-east-1 VPC quota was raised to 60 (2026-06-02) and we briefly ran at | |
| # max-parallel: 12, but that ran the AWS account into CloudControl/EC2 | |
| # throttling (explicit `ThrottlingException: Rate exceeded` errors on | |
| # Discovery ListResources during the 2026-06-03 and 2026-06-04 nightlies, | |
| # plus elevated downstream timeouts). Back off to 10 until the per-plugin | |
| # rate limiter handles shared-account contention better. | |
| max-parallel: 10 | |
| fail-fast: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: "1.26" | |
| - name: Set up Pkl | |
| uses: pkl-community/setup-pkl@v0.0.8 | |
| with: | |
| pkl-version: 0.30.0 | |
| - name: Build formae | |
| run: | | |
| BRANCH="${{ inputs.formae_branch || 'main' }}" | |
| echo "Building formae from branch: ${BRANCH}" | |
| git clone --branch "${BRANCH}" --depth 1 https://github.com/platform-engineering-labs/formae.git /tmp/formae | |
| cd /tmp/formae | |
| git fetch --tags | |
| # The version baked via -ldflags becomes formae.Version at runtime | |
| # and is propagated to plugins as FORMAE_VERSION. Plugins use it | |
| # to resolve their formae PKL dep against | |
| # `package://...formae@<version>` on S3, so it must match a | |
| # schema that's been published. The nightly's purpose is to | |
| # validate plugin main against formae main, so we pick the | |
| # latest tag on main — including dev tags — since that | |
| # represents the most recent published schema. The canonical | |
| # X.Y.Z is stripped from any -dev.N suffix to match how | |
| # formae's own release Makefile publishes schemas (a | |
| # 0.86.0-dev.N release publishes schema at formae@0.86.0). | |
| # An explicit formae_version input overrides this. | |
| VERSION="${{ inputs.formae_version }}" | |
| if [ -z "$VERSION" ]; then | |
| LATEST_TAG=$(git tag -l "[0-9]*" --sort=-version:refname | head -1) | |
| VERSION=$(echo "$LATEST_TAG" | cut -d'-' -f1) | |
| echo "No formae_version input; latest tag=${LATEST_TAG}, using canonical VERSION=${VERSION}" | |
| else | |
| echo "Using formae_version input: ${VERSION}" | |
| fi | |
| echo "Building formae with VERSION=${VERSION}" | |
| make build VERSION="${VERSION}" | |
| # formae main expects an orbital tree at the path derived from the | |
| # binary location (introduced in formae#bfeeb541 — orbital-based | |
| # plugin distribution). `make build` only produces the binary; we | |
| # have to lay out the tree skeleton ourselves. Mirrors the pattern | |
| # used in formae's own e2e-tests workflow. | |
| mkdir -p dist/bin dist/.ops | |
| cp formae dist/bin/formae | |
| echo "FORMAE_BINARY=/tmp/formae/dist/bin/formae" >> $GITHUB_ENV | |
| - name: Track formae clone | |
| if: env.POSTHOG_API_KEY != '' | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| run: | | |
| REPO=$(basename "$(git remote get-url origin)" .git 2>/dev/null || echo "unknown") | |
| curl -sf -o /dev/null https://k.platform.engineering/capture/ \ | |
| -H "Content-Type: application/json" \ | |
| -d "$(jq -n \ | |
| --arg api_key "$POSTHOG_API_KEY" \ | |
| --arg repo "$REPO" \ | |
| --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg run_id "${GITHUB_RUN_ID:-}" \ | |
| '{ | |
| api_key: $api_key, | |
| distinct_id: "formae-ci", | |
| event: "ci_repo_clone", | |
| timestamp: $ts, | |
| properties: { | |
| "$process_person_profile": false, | |
| repo: $repo, | |
| cloned_repo: "formae", | |
| ci_run_id: $run_id | |
| } | |
| }')" || echo "[telemetry] event send failed (non-critical)" >&2 | |
| # Temporary: until Orbital package manager handles plugin version | |
| # coordination, the formae Makefile is the source of truth for which | |
| # plugin branch to build against. Parse the @ref suffix and check | |
| # out that ref so the plugin is compatible with the formae binary. | |
| - name: Align plugin branch with formae Makefile | |
| run: | | |
| PLUGIN_NAME=$(basename "${{ github.repository }}") | |
| REF=$(grep "${PLUGIN_NAME}" /tmp/formae/Makefile \ | |
| | grep -o '@[^[:space:]\\]*' \ | |
| | head -1 \ | |
| | sed 's/^@//') | |
| if [ -n "$REF" ]; then | |
| echo "Formae Makefile pins ${PLUGIN_NAME} to ref: ${REF}" | |
| git fetch origin "${REF}" | |
| git checkout "${REF}" | |
| else | |
| echo "No ref override for ${PLUGIN_NAME}, staying on default branch" | |
| fi | |
| - name: Inject formae replace directives | |
| run: | | |
| FORMAE_SRC=/tmp/formae | |
| for pkg in pkg/auth pkg/model pkg/plugin pkg/plugin-conformance-tests; do | |
| if grep -q "formae/$pkg" go.mod 2>/dev/null; then | |
| echo "Injecting replace for $pkg -> $FORMAE_SRC/$pkg" | |
| go mod edit -replace "github.com/platform-engineering-labs/formae/$pkg=$FORMAE_SRC/$pkg" | |
| fi | |
| done | |
| go mod tidy | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: NightlyConformanceTests | |
| role-duration-seconds: 7200 | |
| - name: Install plugin | |
| run: make install | |
| - name: Run conformance test (${{ matrix.test-case }}) | |
| timeout-minutes: 120 | |
| env: | |
| AWS_REGION: us-east-1 | |
| FORMAE_TEST_RUN_ID: nightly-main-${{ github.run_id }} | |
| # See ci.yml: 2-min discovery default is too tight under CloudControl | |
| # ListResources throttling in the shared account. | |
| FORMAE_TEST_DISCOVERY_TIMEOUT: 5 | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| run: make conformance-test-crud-run conformance-test-discovery-run TEST=${{ matrix.test-case }} PARALLEL=1 TIMEOUT=120 | |
| - name: Dump formae client log on failure | |
| if: failure() | |
| run: | | |
| LOG="$HOME/.pel/formae/log/client.log" | |
| if [ -f "$LOG" ]; then | |
| echo "::group::client.log (last 400 lines)" | |
| tail -400 "$LOG" | |
| echo "::endgroup::" | |
| else | |
| echo "No client.log at $LOG" | |
| fi | |
| # Clean up test resources after all conformance tests (always runs) | |
| post-cleanup: | |
| needs: [conformance-tests] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: NightlyPostCleanup | |
| - name: Clean test resources | |
| run: ./scripts/ci/clean-environment.sh | |
| # Notify when nightly is fixed (was red, now green) | |
| notify-fixed: | |
| needs: [test-integration, conformance-tests, post-cleanup] | |
| runs-on: ubuntu-latest | |
| if: success() | |
| steps: | |
| - name: Check if previous run failed | |
| id: check_previous | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const runs = await github.rest.actions.listWorkflowRuns({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'nightly.yml', | |
| per_page: 2, | |
| status: 'completed' | |
| }); | |
| // Get previous run (skip current which isn't completed yet) | |
| const previousRun = runs.data.workflow_runs.find(run => run.id !== context.runId); | |
| if (previousRun && previousRun.conclusion === 'failure') { | |
| return 'true'; | |
| } | |
| return 'false'; | |
| result-encoding: string | |
| - name: Notify Slack (Fixed) | |
| if: steps.check_previous.outputs.result == 'true' | |
| uses: slackapi/slack-github-action@v3.0.1 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| { | |
| "text": ":white_check_mark: Nightly fixed on ${{ github.repository }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":white_check_mark: *Nightly fixed* on `${{ github.repository }}`\n\nPlugin conformance tests now passing against formae main branch." | |
| } | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "View Run" | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| # Notify on failure | |
| notify-failure: | |
| needs: [test-integration, conformance-tests, post-cleanup] | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| steps: | |
| - name: Notify Slack | |
| uses: slackapi/slack-github-action@v3.0.1 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| { | |
| "text": ":warning: Nightly build failed on ${{ github.repository }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":warning: *Nightly build failed* on `${{ github.repository }}`\n\nPlugin conformance tests failed against formae main branch. This may indicate an SDK regression." | |
| } | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "View Run" | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } |