Debug Conformance #184
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Debug Conformance | |
| # On-demand workflow for re-running a focused set of conformance tests. | |
| # Used to bisect regressions or iterate on a fix without paying for the | |
| # full conformance matrix. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_cases: | |
| description: "Comma-separated test cases (e.g. ses-eventdestination,route53-recordset)" | |
| required: true | |
| type: string | |
| formae_ref: | |
| description: "Formae git ref (branch/tag/SHA) to build from. Leave empty to use the latest released binary." | |
| required: false | |
| default: "" | |
| type: string | |
| formae_version: | |
| description: "Override the VERSION string used when building formae from source. Use this with formae_ref to validate against a pre-released PKL schema (e.g., 0.86.0 published via schema-prerelease workflow). Leave empty to default to the latest released tag." | |
| required: false | |
| default: "" | |
| type: string | |
| pre_cleanup: | |
| description: "Run pre-cleanup before tests" | |
| required: false | |
| default: true | |
| type: boolean | |
| # Share the conformance-tests serialization group with CI/nightly so we | |
| # never race for shared AWS state. Debug invocations queue behind any | |
| # in-flight CI/nightly run. | |
| concurrency: | |
| group: aws-conformance-tests | |
| cancel-in-progress: false | |
| jobs: | |
| parse-input: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-cases: ${{ steps.parse.outputs.test-cases }} | |
| steps: | |
| - name: Parse test_cases input | |
| id: parse | |
| run: | | |
| # Convert "a, b,c" → ["a","b","c"] | |
| TEST_CASES=$(echo '${{ inputs.test_cases }}' \ | |
| | tr ',' '\n' \ | |
| | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \ | |
| | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "Parsed test cases: ${TEST_CASES}" | |
| echo "test-cases=${TEST_CASES}" >> "$GITHUB_OUTPUT" | |
| pre-cleanup: | |
| needs: [parse-input] | |
| if: inputs.pre_cleanup | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: DebugPreCleanup | |
| - name: Clean test resources | |
| run: ./scripts/ci/clean-environment.sh | |
| conformance-tests: | |
| needs: [parse-input, pre-cleanup] | |
| # When pre_cleanup=false the pre-cleanup job is skipped; without an | |
| # explicit if-clause this job would skip too because a needed job | |
| # didn't succeed. | |
| if: ${{ !cancelled() && needs.parse-input.result == 'success' && (needs.pre-cleanup.result == 'success' || needs.pre-cleanup.result == 'skipped') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| strategy: | |
| matrix: | |
| test-case: ${{ fromJson(needs.parse-input.outputs.test-cases) }} | |
| max-parallel: 5 | |
| fail-fast: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: "1.26" | |
| - name: Set up Pkl | |
| uses: pkl-community/setup-pkl@v0.0.8 | |
| with: | |
| pkl-version: 0.30.0 | |
| # === Source-build path: only runs when formae_ref is provided === | |
| # When unset, scripts/run-conformance-tests.sh downloads the latest | |
| # released formae binary, matching CI behavior. | |
| - name: Build formae from ref ${{ inputs.formae_ref }} | |
| if: inputs.formae_ref != '' | |
| run: | | |
| REF="${{ inputs.formae_ref }}" | |
| echo "Building formae from ref: ${REF}" | |
| # Full clone — --branch doesn't accept arbitrary SHAs. | |
| git clone https://github.com/platform-engineering-labs/formae.git /tmp/formae | |
| cd /tmp/formae | |
| git checkout "${REF}" | |
| git fetch --tags | |
| # Latest release tag becomes the build's VERSION (skips pre-releases). | |
| LATEST_TAG=$(git tag -l "[0-9]*" --sort=-version:refname | grep -v -- '-' | head -1) | |
| FORMAE_BUILD_VERSION="${{ inputs.formae_version }}" | |
| if [ -z "$FORMAE_BUILD_VERSION" ]; then | |
| FORMAE_BUILD_VERSION="${LATEST_TAG}" | |
| fi | |
| echo "Building formae with VERSION=${FORMAE_BUILD_VERSION}" | |
| make build VERSION="${FORMAE_BUILD_VERSION}" | |
| # formae main expects an orbital tree at the path derived from the | |
| # binary location (introduced in formae#bfeeb541). `make build` | |
| # only produces the binary; lay out the tree skeleton ourselves. | |
| mkdir -p dist/bin dist/.ops | |
| cp formae dist/bin/formae | |
| echo "FORMAE_BINARY=/tmp/formae/dist/bin/formae" >> $GITHUB_ENV | |
| - name: Inject formae replace directives | |
| if: inputs.formae_ref != '' | |
| run: | | |
| FORMAE_SRC=/tmp/formae | |
| for pkg in pkg/auth pkg/model pkg/plugin pkg/plugin-conformance-tests; do | |
| if grep -q "formae/$pkg" go.mod 2>/dev/null; then | |
| echo "Injecting replace for $pkg -> $FORMAE_SRC/$pkg" | |
| go mod edit -replace "github.com/platform-engineering-labs/formae/$pkg=$FORMAE_SRC/$pkg" | |
| fi | |
| done | |
| go mod tidy | |
| # Note: nightly's "Align plugin branch with formae Makefile" step is | |
| # intentionally omitted. Debug runs test the plugin code at the ref | |
| # the workflow was dispatched on, regardless of formae's Makefile pin. | |
| # === End source-build path === | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: DebugConformance | |
| role-duration-seconds: 7200 | |
| - name: Install plugin | |
| run: make install | |
| - name: Run conformance test (${{ matrix.test-case }}) | |
| env: | |
| AWS_REGION: us-east-1 | |
| FORMAE_TEST_RUN_ID: debug-${{ github.run_id }}-${{ github.run_attempt }} | |
| # See ci.yml: 2-min discovery default is too tight under CloudControl | |
| # ListResources throttling in the shared account. | |
| FORMAE_TEST_DISCOVERY_TIMEOUT: 5 | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| # 90-minute timeout accommodates LB-attached fixtures (ALB provisioning | |
| # + ECS stabilization + destroy + reapply takes 65-80m). Smaller fixtures | |
| # are unaffected — they finish well under the bound. | |
| run: make conformance-test-crud-run conformance-test-discovery-run TEST=${{ matrix.test-case }} PARALLEL=1 TIMEOUT=90 | |
| - name: Dump formae client log on failure | |
| if: failure() | |
| run: | | |
| LOG="$HOME/.pel/formae/log/client.log" | |
| if [ -f "$LOG" ]; then | |
| echo "::group::client.log (last 400 lines)" | |
| tail -400 "$LOG" | |
| echo "::endgroup::" | |
| else | |
| echo "No client.log at $LOG" | |
| fi | |
| post-cleanup: | |
| needs: [conformance-tests] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout plugin | |
| uses: actions/checkout@v6 | |
| - name: Configure AWS Credentials | |
| uses: ./.github/actions/aws-credentials | |
| with: | |
| role-to-assume: arn:aws:iam::942849037363:role/admin-test-pel | |
| role-session-name: DebugPostCleanup | |
| - name: Clean test resources | |
| run: ./scripts/ci/clean-environment.sh |