Skip to content

Commit 962e1d2

Browse files
github-actions[bot]github-actions[bot]
committed
Publish CrawlerScope site
1 parent a99ad3c commit 962e1d2

3 files changed

Lines changed: 8947 additions & 44 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-06-03T11:24:07Z",
196+
"generatedAt": "2026-06-03T17:20:47Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-06-03T11:24:07Z",
230+
"lastCheckedAt": "2026-06-03T17:20:47Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -338,7 +338,7 @@
338338
"documentationUrl": null,
339339
"id": "google-special",
340340
"ipListAuthoritative": true,
341-
"lastCheckedAt": "2026-06-03T11:24:07Z",
341+
"lastCheckedAt": "2026-06-03T17:20:47Z",
342342
"note": null,
343343
"operator": "Google",
344344
"operatorCountry": "US",
@@ -424,7 +424,7 @@
424424
"documentationUrl": null,
425425
"id": "google-user-triggered",
426426
"ipListAuthoritative": true,
427-
"lastCheckedAt": "2026-06-03T11:24:07Z",
427+
"lastCheckedAt": "2026-06-03T17:20:47Z",
428428
"note": null,
429429
"operator": "Google",
430430
"operatorCountry": "US",
@@ -684,7 +684,7 @@
684684
"documentationUrl": null,
685685
"id": "bingbot",
686686
"ipListAuthoritative": true,
687-
"lastCheckedAt": "2026-06-03T11:24:07Z",
687+
"lastCheckedAt": "2026-06-03T17:20:47Z",
688688
"note": null,
689689
"operator": "Microsoft",
690690
"operatorCountry": "US",
@@ -748,7 +748,7 @@
748748
"documentationUrl": null,
749749
"id": "duckduckbot",
750750
"ipListAuthoritative": true,
751-
"lastCheckedAt": "2026-06-03T11:24:07Z",
751+
"lastCheckedAt": "2026-06-03T17:20:47Z",
752752
"note": null,
753753
"operator": "DuckDuckGo",
754754
"operatorCountry": "US",
@@ -1122,7 +1122,7 @@
11221122
"documentationUrl": null,
11231123
"id": "duckassistbot",
11241124
"ipListAuthoritative": true,
1125-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1125+
"lastCheckedAt": "2026-06-03T17:20:47Z",
11261126
"note": null,
11271127
"operator": "DuckDuckGo",
11281128
"operatorCountry": "US",
@@ -1496,7 +1496,7 @@
14961496
"documentationUrl": "https://support.apple.com/en-us/119829",
14971497
"id": "applebot",
14981498
"ipListAuthoritative": true,
1499-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1499+
"lastCheckedAt": "2026-06-03T17:20:47Z",
15001500
"note": null,
15011501
"operator": "Apple",
15021502
"operatorCountry": "US",
@@ -1543,7 +1543,7 @@
15431543
"documentationUrl": null,
15441544
"id": "mojeekbot",
15451545
"ipListAuthoritative": true,
1546-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1546+
"lastCheckedAt": "2026-06-03T17:20:47Z",
15471547
"note": null,
15481548
"operator": "Mojeek",
15491549
"operatorCountry": "GB",
@@ -1579,7 +1579,7 @@
15791579
"documentationUrl": null,
15801580
"id": "naver-yeti",
15811581
"ipListAuthoritative": true,
1582-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1582+
"lastCheckedAt": "2026-06-03T17:20:47Z",
15831583
"note": null,
15841584
"operator": "Naver",
15851585
"operatorCountry": "KR",
@@ -1650,7 +1650,7 @@
16501650
"documentationUrl": null,
16511651
"id": "yandexbot",
16521652
"ipListAuthoritative": false,
1653-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1653+
"lastCheckedAt": "2026-06-03T17:20:47Z",
16541654
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
16551655
"operator": "Yandex",
16561656
"operatorCountry": "RU",
@@ -1699,7 +1699,7 @@
16991699
"documentationUrl": null,
17001700
"id": "baiduspider",
17011701
"ipListAuthoritative": false,
1702-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1702+
"lastCheckedAt": "2026-06-03T17:20:47Z",
17031703
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
17041704
"operator": "Baidu",
17051705
"operatorCountry": "CN",
@@ -1737,7 +1737,7 @@
17371737
"documentationUrl": null,
17381738
"id": "openai-gptbot",
17391739
"ipListAuthoritative": true,
1740-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1740+
"lastCheckedAt": "2026-06-03T17:20:47Z",
17411741
"note": null,
17421742
"operator": "OpenAI",
17431743
"operatorCountry": "US",
@@ -1787,7 +1787,7 @@
17871787
"documentationUrl": null,
17881788
"id": "openai-searchbot",
17891789
"ipListAuthoritative": true,
1790-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1790+
"lastCheckedAt": "2026-06-03T17:20:47Z",
17911791
"note": null,
17921792
"operator": "OpenAI",
17931793
"operatorCountry": "US",
@@ -1852,7 +1852,7 @@
18521852
"documentationUrl": null,
18531853
"id": "openai-chatgpt-user",
18541854
"ipListAuthoritative": true,
1855-
"lastCheckedAt": "2026-06-03T11:24:07Z",
1855+
"lastCheckedAt": "2026-06-03T17:20:47Z",
18561856
"note": null,
18571857
"operator": "OpenAI",
18581858
"operatorCountry": "US",
@@ -2099,7 +2099,7 @@
20992099
"documentationUrl": null,
21002100
"id": "openai-adsbot",
21012101
"ipListAuthoritative": false,
2102-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2102+
"lastCheckedAt": "2026-06-03T17:20:47Z",
21032103
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
21042104
"operator": "OpenAI",
21052105
"operatorCountry": "US",
@@ -2131,7 +2131,7 @@
21312131
"documentationUrl": null,
21322132
"id": "perplexitybot",
21332133
"ipListAuthoritative": true,
2134-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2134+
"lastCheckedAt": "2026-06-03T17:20:47Z",
21352135
"note": null,
21362136
"operator": "Perplexity",
21372137
"operatorCountry": "US",
@@ -2172,7 +2172,7 @@
21722172
"documentationUrl": null,
21732173
"id": "perplexity-user",
21742174
"ipListAuthoritative": true,
2175-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2175+
"lastCheckedAt": "2026-06-03T17:20:47Z",
21762176
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
21772177
"operator": "Perplexity",
21782178
"operatorCountry": "US",
@@ -2209,7 +2209,7 @@
22092209
"documentationUrl": null,
22102210
"id": "claudebot",
22112211
"ipListAuthoritative": false,
2212-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2212+
"lastCheckedAt": "2026-06-03T17:20:47Z",
22132213
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
22142214
"operator": "Anthropic",
22152215
"operatorCountry": "US",
@@ -2243,7 +2243,7 @@
22432243
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
22442244
"id": "amazonbot",
22452245
"ipListAuthoritative": true,
2246-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2246+
"lastCheckedAt": "2026-06-03T17:20:47Z",
22472247
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
22482248
"operator": "Amazon",
22492249
"operatorCountry": "US",
@@ -2802,7 +2802,7 @@
28022802
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
28032803
"id": "amazon-searchbot",
28042804
"ipListAuthoritative": true,
2805-
"lastCheckedAt": "2026-06-03T11:24:07Z",
2805+
"lastCheckedAt": "2026-06-03T17:20:47Z",
28062806
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
28072807
"operator": "Amazon",
28082808
"operatorCountry": "US",
@@ -3349,7 +3349,7 @@
33493349
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
33503350
"id": "amazon-amzn-user",
33513351
"ipListAuthoritative": true,
3352-
"lastCheckedAt": "2026-06-03T11:24:07Z",
3352+
"lastCheckedAt": "2026-06-03T17:20:47Z",
33533353
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
33543354
"operator": "Amazon",
33553355
"operatorCountry": "US",
@@ -4407,7 +4407,7 @@
44074407
"documentationUrl": null,
44084408
"id": "meta-ai-crawlers",
44094409
"ipListAuthoritative": false,
4410-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4410+
"lastCheckedAt": "2026-06-03T17:20:47Z",
44114411
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
44124412
"operator": "Meta",
44134413
"operatorCountry": "US",
@@ -4446,7 +4446,7 @@
44464446
"documentationUrl": null,
44474447
"id": "bytespider",
44484448
"ipListAuthoritative": false,
4449-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4449+
"lastCheckedAt": "2026-06-03T17:20:47Z",
44504450
"note": null,
44514451
"operator": "ByteDance",
44524452
"operatorCountry": "CN",
@@ -4478,7 +4478,7 @@
44784478
"documentationUrl": null,
44794479
"id": "mistralai-user",
44804480
"ipListAuthoritative": true,
4481-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4481+
"lastCheckedAt": "2026-06-03T17:20:47Z",
44824482
"note": null,
44834483
"operator": "Mistral AI",
44844484
"operatorCountry": "FR",
@@ -4515,7 +4515,7 @@
45154515
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
45164516
"id": "ahrefsbot",
45174517
"ipListAuthoritative": true,
4518-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4518+
"lastCheckedAt": "2026-06-03T17:20:47Z",
45194519
"note": null,
45204520
"operator": "Ahrefs",
45214521
"operatorCountry": "SG",
@@ -4602,7 +4602,7 @@
46024602
"documentationUrl": null,
46034603
"id": "lumar-crawler",
46044604
"ipListAuthoritative": true,
4605-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4605+
"lastCheckedAt": "2026-06-03T17:20:47Z",
46064606
"note": null,
46074607
"operator": "Lumar",
46084608
"operatorCountry": "GB",
@@ -4701,7 +4701,7 @@
47014701
"documentationUrl": null,
47024702
"id": "semrushbot",
47034703
"ipListAuthoritative": false,
4704-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4704+
"lastCheckedAt": "2026-06-03T17:20:47Z",
47054705
"note": null,
47064706
"operator": "Semrush",
47074707
"operatorCountry": "US",
@@ -4735,7 +4735,7 @@
47354735
"documentationUrl": null,
47364736
"id": "censys",
47374737
"ipListAuthoritative": false,
4738-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4738+
"lastCheckedAt": "2026-06-03T17:20:47Z",
47394739
"note": null,
47404740
"operator": "Censys",
47414741
"operatorCountry": "US",
@@ -4772,7 +4772,7 @@
47724772
"documentationUrl": null,
47734773
"id": "shodan",
47744774
"ipListAuthoritative": false,
4775-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4775+
"lastCheckedAt": "2026-06-03T17:20:47Z",
47764776
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
47774777
"operator": "Shodan",
47784778
"operatorCountry": "US",
@@ -4817,7 +4817,7 @@
48174817
"documentationUrl": null,
48184818
"id": "datadog-synthetics",
48194819
"ipListAuthoritative": true,
4820-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4820+
"lastCheckedAt": "2026-06-03T17:20:47Z",
48214821
"note": null,
48224822
"operator": "Datadog",
48234823
"operatorCountry": "US",
@@ -4963,7 +4963,7 @@
49634963
"documentationUrl": null,
49644964
"id": "ias-crawler",
49654965
"ipListAuthoritative": true,
4966-
"lastCheckedAt": "2026-06-03T11:24:07Z",
4966+
"lastCheckedAt": "2026-06-03T17:20:47Z",
49674967
"note": null,
49684968
"operator": "Integral Ad Science",
49694969
"operatorCountry": "US",
@@ -5010,7 +5010,7 @@
50105010
"documentationUrl": null,
50115011
"id": "ttd-content",
50125012
"ipListAuthoritative": true,
5013-
"lastCheckedAt": "2026-06-03T11:24:07Z",
5013+
"lastCheckedAt": "2026-06-03T17:20:47Z",
50145014
"note": null,
50155015
"operator": "The Trade Desk",
50165016
"operatorCountry": "US",
@@ -7658,7 +7658,7 @@
76587658
"documentationUrl": "https://uptimerobot.com/locations",
76597659
"id": "uptimerobot",
76607660
"ipListAuthoritative": true,
7661-
"lastCheckedAt": "2026-06-03T11:24:07Z",
7661+
"lastCheckedAt": "2026-06-03T17:20:47Z",
76627662
"note": null,
76637663
"operator": "UptimeRobot",
76647664
"operatorCountry": "US",
@@ -7909,7 +7909,7 @@
79097909
"documentationUrl": null,
79107910
"id": "pingdom",
79117911
"ipListAuthoritative": true,
7912-
"lastCheckedAt": "2026-06-03T11:24:07Z",
7912+
"lastCheckedAt": "2026-06-03T17:20:47Z",
79137913
"note": null,
79147914
"operator": "SolarWinds Pingdom",
79157915
"operatorCountry": "US",
@@ -8102,7 +8102,7 @@
81028102
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
81038103
"id": "statuscake",
81048104
"ipListAuthoritative": true,
8105-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8105+
"lastCheckedAt": "2026-06-03T17:20:47Z",
81068106
"note": null,
81078107
"operator": "StatusCake",
81088108
"operatorCountry": "GB",
@@ -8432,7 +8432,7 @@
84328432
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
84338433
"id": "better-stack",
84348434
"ipListAuthoritative": true,
8435-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8435+
"lastCheckedAt": "2026-06-03T17:20:47Z",
84368436
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
84378437
"operator": "Better Stack",
84388438
"operatorCountry": "CZ",
@@ -8500,7 +8500,7 @@
85008500
"documentationUrl": null,
85018501
"id": "ccbot",
85028502
"ipListAuthoritative": true,
8503-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8503+
"lastCheckedAt": "2026-06-03T17:20:47Z",
85048504
"note": null,
85058505
"operator": "Common Crawl",
85068506
"operatorCountry": "US",
@@ -8540,7 +8540,7 @@
85408540
"documentationUrl": null,
85418541
"id": "flipboard-crawler",
85428542
"ipListAuthoritative": true,
8543-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8543+
"lastCheckedAt": "2026-06-03T17:20:47Z",
85448544
"note": null,
85458545
"operator": "Flipboard",
85468546
"operatorCountry": "US",
@@ -8709,7 +8709,7 @@
87098709
"documentationUrl": null,
87108710
"id": "parsely-crawler",
87118711
"ipListAuthoritative": true,
8712-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8712+
"lastCheckedAt": "2026-06-03T17:20:47Z",
87138713
"note": null,
87148714
"operator": "Parse.ly",
87158715
"operatorCountry": "US",
@@ -8752,7 +8752,7 @@
87528752
"documentationUrl": null,
87538753
"id": "pinterestbot",
87548754
"ipListAuthoritative": false,
8755-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8755+
"lastCheckedAt": "2026-06-03T17:20:47Z",
87568756
"note": null,
87578757
"operator": "Pinterest",
87588758
"operatorCountry": "US",
@@ -8786,7 +8786,7 @@
87868786
"documentationUrl": null,
87878787
"id": "linkedinbot",
87888788
"ipListAuthoritative": false,
8789-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8789+
"lastCheckedAt": "2026-06-03T17:20:47Z",
87908790
"note": null,
87918791
"operator": "LinkedIn",
87928792
"operatorCountry": "US",
@@ -8818,7 +8818,7 @@
88188818
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
88198819
"id": "telegram-link-preview",
88208820
"ipListAuthoritative": true,
8821-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8821+
"lastCheckedAt": "2026-06-03T17:20:47Z",
88228822
"note": null,
88238823
"operator": "Telegram",
88248824
"operatorCountry": "AE",
@@ -8863,7 +8863,7 @@
88638863
"documentationUrl": "https://rssapi.net/faq",
88648864
"id": "rssapi-parser",
88658865
"ipListAuthoritative": true,
8866-
"lastCheckedAt": "2026-06-03T11:24:07Z",
8866+
"lastCheckedAt": "2026-06-03T17:20:47Z",
88678867
"note": null,
88688868
"operator": "RSS API",
88698869
"operatorCountry": "DE",

0 commit comments

Comments
 (0)