Repository navigation
Container runtime alert #3294
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Container runtime alert | |
| on: | |
| schedule: | |
| - cron: "*/15 * * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| inspect: | |
| name: detect long-running ${{ matrix.stack }} containers | |
| if: github.repository == 'langgenius/mosoo' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| # Scheduled runs execute on main. The deploy-only `try` environment admits | |
| # deploy/try alone, so this job reads a main-only environment that holds a | |
| # read-only Cloudflare Containers token. | |
| environment: container-monitor | |
| strategy: | |
| # Stage has its own Workers and the same container ceilings; a failed | |
| # production read must not hide it, and vice versa. | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - stack: production | |
| wrangler_env: prod | |
| alert_title: "[Ops] Production container capacity anomaly" | |
| - stack: stage | |
| wrangler_env: stage | |
| alert_title: "[Ops] Stage container capacity anomaly" | |
| concurrency: | |
| group: container-runtime-alert-${{ matrix.stack }} | |
| cancel-in-progress: true | |
| env: | |
| ALERT_TITLE: ${{ matrix.alert_title }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| WRANGLER: wrangler@4.123.0 | |
| WRANGLER_ENV: ${{ matrix.wrangler_env }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 | |
| with: | |
| bun-version: canary | |
| - name: Read authoritative Cloudflare instance state | |
| id: inspect | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| instances_file="$RUNNER_TEMP/container-instances.json" | |
| report_file="$RUNNER_TEMP/container-runtime-report.md" | |
| # Resolves every image application wrangler deploys for this | |
| # environment and pages through all of their instances. | |
| bun scripts/collect-container-instances.ts apps/api/wrangler.toml "$WRANGLER_ENV" "$instances_file" | |
| set +e | |
| bun scripts/check-container-runtime.ts "$instances_file" > "$report_file" | |
| status=$? | |
| set -e | |
| if [[ "$status" -ne 0 && "$status" -ne 2 ]]; then | |
| exit "$status" | |
| fi | |
| echo "anomaly=$([[ "$status" -eq 2 ]] && echo true || echo false)" >> "$GITHUB_OUTPUT" | |
| echo "report_file=$report_file" >> "$GITHUB_OUTPUT" | |
| cat "$report_file" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Open or resolve incident issue | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| ANOMALY: ${{ steps.inspect.outputs.anomaly }} | |
| REPORT_FILE: ${{ steps.inspect.outputs.report_file }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| issue_number="$(gh issue list --state open --limit 100 --json number,title \ | |
| --jq '[.[] | select(.title == env.ALERT_TITLE) | .number] | first // empty')" | |
| if [[ "$ANOMALY" == "true" ]]; then | |
| if [[ -z "$issue_number" ]]; then | |
| gh issue create --assignee Yevanchen --title "$ALERT_TITLE" --body-file "$REPORT_FILE" | |
| fi | |
| exit 1 | |
| fi | |
| if [[ -n "$issue_number" ]]; then | |
| gh issue close "$issue_number" --comment "Cloudflare reports fewer than ten active containers and none running beyond two hours. Auto-resolved." | |
| fi |