refresh-raw #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: refresh-raw | |
| # Cron: every Monday 09:00 HKT = 01:00 UTC | |
| # Plus manual trigger from GitHub UI / `gh workflow run refresh-raw`. | |
| # | |
| # Architecture: | |
| # - 9 parallel matrix jobs, one per source. Each fetches its source and | |
| # commits its own slice of 01_Raw/ (with rebase-retry to handle parallel pushes). | |
| # - One aggregator job runs after all matrix jobs to write the human-readable | |
| # changelog (03_Output/Changelog/YYYY-MM-DD.md) summarizing what changed. | |
| # - fail-fast: false → one source's failure never cancels the others. | |
| # | |
| # Source list MUST match `python3 scripts/refresh_raw.py --list` output exactly, | |
| # otherwise the matrix job for that source aborts with "unknown source". | |
| on: | |
| schedule: | |
| - cron: "0 1 * * 1" | |
| workflow_dispatch: {} | |
| # Prevent two scheduled runs from racing each other on the repo. | |
| concurrency: | |
| group: refresh-raw | |
| cancel-in-progress: false | |
| jobs: | |
| refresh: | |
| name: "fetch / ${{ matrix.source }}" | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| source: | |
| - code.claude.com | |
| - platform.claude.com | |
| - anthropic.com | |
| - docs.cursor.com | |
| - ai.google.dev | |
| - openai.com | |
| - github.anthropics | |
| - github.modelcontextprotocol | |
| - github.openai | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - name: Install deps | |
| run: pip install -r scripts/requirements.txt | |
| - name: Configure git | |
| run: | | |
| git config user.name "ai-coding-runbook-bot" | |
| git config user.email "ai-coding-runbook-bot@users.noreply.github.com" | |
| - name: Refresh raw — ${{ matrix.source }} | |
| run: python3 scripts/refresh_raw.py --source "${{ matrix.source }}" | |
| - name: Commit + push (with rebase-retry for concurrent pushes) | |
| env: | |
| SOURCE: ${{ matrix.source }} | |
| run: | | |
| set -e | |
| # Defensive: remove any stale submodule index entries (gitlinks) under 01_Raw/github/ | |
| # that may exist from earlier failed runs. Without this, `git add` on a path that's | |
| # registered as a submodule keeps the gitlink instead of tracking the real files. | |
| git rm --cached -r --ignore-unmatch 01_Raw/github/ >/dev/null 2>&1 || true | |
| # Stage everything related to this source. | |
| git add 01_Raw/ || true | |
| if git diff --cached --quiet; then | |
| echo "no changes for $SOURCE" | |
| exit 0 | |
| fi | |
| DATE=$(date -u +%Y-%m-%d) | |
| MSG="raw: refresh $SOURCE ($DATE)" | |
| git commit -m "$MSG" | |
| # Up to 5 push attempts with exponential backoff. Re-stage iff rebase changed anything. | |
| for attempt in 1 2 3 4 5; do | |
| if git pull --rebase origin main && git push origin main; then | |
| echo "✓ pushed on attempt $attempt" | |
| exit 0 | |
| fi | |
| sleep $((attempt * 5 + RANDOM % 5)) | |
| done | |
| echo "✗ push failed after 5 attempts" | |
| exit 1 | |
| changelog: | |
| name: aggregate changelog | |
| needs: refresh | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout (post-matrix HEAD) | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 50 | |
| - name: Configure git | |
| run: | | |
| git config user.name "ai-coding-runbook-bot" | |
| git config user.email "ai-coding-runbook-bot@users.noreply.github.com" | |
| - name: Generate changelog | |
| id: gen | |
| run: | | |
| set -e | |
| DATE=$(date -u +%Y-%m-%d) | |
| OUT="03_Output/Changelog/${DATE}.md" | |
| mkdir -p 03_Output/Changelog | |
| # Find commits from this run (those by ai-coding-runbook-bot in the last 2 hours). | |
| COMMITS=$(git log --since="2 hours ago" --author="ai-coding-runbook-bot" --pretty=format:"%h" -- 01_Raw/ || true) | |
| if [ -z "$COMMITS" ]; then | |
| echo "no bot commits in last 2h — nothing to aggregate" | |
| echo "wrote=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| { | |
| echo "# Raw Refresh — ${DATE}" | |
| echo | |
| echo "Generated by GHA \`refresh-raw\` workflow." | |
| echo | |
| echo "## Per-source commits" | |
| echo | |
| git log --since="2 hours ago" --author="ai-coding-runbook-bot" \ | |
| --pretty=format:"- \`%h\` %s" -- 01_Raw/ || true | |
| echo | |
| echo | |
| echo "## Files changed (vs HEAD~N where N=number of bot commits)" | |
| echo | |
| N=$(echo "$COMMITS" | wc -l | tr -d ' ') | |
| echo '```' | |
| git diff --stat "HEAD~$N..HEAD" -- 01_Raw/ | head -40 | |
| echo '```' | |
| } > "$OUT" | |
| echo "wrote=true" >> "$GITHUB_OUTPUT" | |
| - name: Commit changelog | |
| if: steps.gen.outputs.wrote == 'true' | |
| run: | | |
| git add 03_Output/Changelog/ | |
| if git diff --cached --quiet; then exit 0; fi | |
| git commit -m "changelog: $(date -u +%Y-%m-%d)" | |
| for attempt in 1 2 3; do | |
| if git pull --rebase origin main && git push origin main; then | |
| exit 0 | |
| fi | |
| sleep $((attempt * 5)) | |
| done | |
| exit 1 |