CrawlerScope #48
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CrawlerScope | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: "23 */6 * * *" | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: crawler-scope-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Update crawler data | |
| env: | |
| CRAWLER_SCOPE_SNAPSHOT_RETENTION: "168" | |
| CRAWLER_SCOPE_HISTORY_RETENTION: "720" | |
| CRAWLER_SCOPE_USER_AGENT: "CrawlerScope/0.1 (+${{ github.server_url }}/${{ github.repository }}; GitHub Actions public-data collector)" | |
| run: python3 scripts/update.py | |
| - name: Commit updated data | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add data config | |
| if git diff --cached --quiet; then | |
| echo "No dataset changes to commit." | |
| else | |
| git commit -m "Update CrawlerScope data" | |
| git push | |
| fi | |
| - name: Publish dataset release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| stamp="$(python3 -c 'import json; print(json.load(open("data/current/crawlers.json"))["generatedAt"].replace(":", "").replace("-", "").replace("Z", "Z"))')" | |
| tag="dataset-${stamp}" | |
| snapshot="$(ls -1 data/snapshots/*.json | sort | tail -n 1)" | |
| python3 scripts/release_notes.py > release-notes.md | |
| if gh release view "$tag" >/dev/null 2>&1; then | |
| gh release edit "$tag" \ | |
| --title "CrawlerScope dataset ${stamp}" \ | |
| --notes-file release-notes.md \ | |
| --latest | |
| gh release upload "$tag" \ | |
| data/current/crawlers.json \ | |
| data/current/robots-ai.txt \ | |
| data/current/nginx-ai-map.conf \ | |
| data/history/summary.csv \ | |
| "$snapshot" \ | |
| --clobber | |
| else | |
| gh release create "$tag" \ | |
| data/current/crawlers.json \ | |
| data/current/robots-ai.txt \ | |
| data/current/nginx-ai-map.conf \ | |
| data/history/summary.csv \ | |
| "$snapshot" \ | |
| --title "CrawlerScope dataset ${stamp}" \ | |
| --notes-file release-notes.md \ | |
| --latest | |
| fi | |
| - name: Publish GitHub Pages branch | |
| run: | | |
| site_source="$RUNNER_TEMP/crawler-scope-site" | |
| pages_dir="$RUNNER_TEMP/crawler-scope-pages" | |
| rm -rf "$site_source" "$pages_dir" | |
| mkdir -p "$site_source/data" | |
| cp -R public/. "$site_source/" | |
| cp -R data/. "$site_source/data/" | |
| touch "$site_source/.nojekyll" | |
| git fetch origin gh-pages --depth=1 || true | |
| if git rev-parse --verify origin/gh-pages >/dev/null 2>&1; then | |
| git worktree add "$pages_dir" origin/gh-pages | |
| else | |
| git worktree add --detach "$pages_dir" | |
| git -C "$pages_dir" checkout --orphan gh-pages | |
| git -C "$pages_dir" rm -rf . || true | |
| fi | |
| find "$pages_dir" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + | |
| cp -R "$site_source"/. "$pages_dir"/ | |
| git -C "$pages_dir" add -A | |
| if git -C "$pages_dir" diff --cached --quiet; then | |
| echo "No Pages changes to publish." | |
| else | |
| git -C "$pages_dir" commit -m "Publish CrawlerScope site" | |
| git -C "$pages_dir" push origin HEAD:gh-pages | |
| fi |