Skip to content

Commit acb7bde

Browse files
github-actions[bot]github-actions[bot]
committed
Update CrawlerScope data
1 parent b35a85e commit acb7bde

3 files changed

Lines changed: 9058 additions & 44 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-06-26T04:42:30Z",
196+
"generatedAt": "2026-06-26T09:53:09Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-06-26T04:42:30Z",
230+
"lastCheckedAt": "2026-06-26T09:53:09Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -332,7 +332,7 @@
332332
"documentationUrl": null,
333333
"id": "google-special",
334334
"ipListAuthoritative": true,
335-
"lastCheckedAt": "2026-06-26T04:42:30Z",
335+
"lastCheckedAt": "2026-06-26T09:53:09Z",
336336
"note": null,
337337
"operator": "Google",
338338
"operatorCountry": "US",
@@ -410,7 +410,7 @@
410410
"documentationUrl": null,
411411
"id": "google-user-triggered",
412412
"ipListAuthoritative": true,
413-
"lastCheckedAt": "2026-06-26T04:42:30Z",
413+
"lastCheckedAt": "2026-06-26T09:53:09Z",
414414
"note": null,
415415
"operator": "Google",
416416
"operatorCountry": "US",
@@ -668,7 +668,7 @@
668668
"documentationUrl": null,
669669
"id": "bingbot",
670670
"ipListAuthoritative": true,
671-
"lastCheckedAt": "2026-06-26T04:42:30Z",
671+
"lastCheckedAt": "2026-06-26T09:53:09Z",
672672
"note": null,
673673
"operator": "Microsoft",
674674
"operatorCountry": "US",
@@ -732,7 +732,7 @@
732732
"documentationUrl": null,
733733
"id": "duckduckbot",
734734
"ipListAuthoritative": true,
735-
"lastCheckedAt": "2026-06-26T04:42:30Z",
735+
"lastCheckedAt": "2026-06-26T09:53:09Z",
736736
"note": null,
737737
"operator": "DuckDuckGo",
738738
"operatorCountry": "US",
@@ -1174,7 +1174,7 @@
11741174
"documentationUrl": null,
11751175
"id": "duckassistbot",
11761176
"ipListAuthoritative": true,
1177-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1177+
"lastCheckedAt": "2026-06-26T09:53:09Z",
11781178
"note": null,
11791179
"operator": "DuckDuckGo",
11801180
"operatorCountry": "US",
@@ -1616,7 +1616,7 @@
16161616
"documentationUrl": "https://support.apple.com/en-us/119829",
16171617
"id": "applebot",
16181618
"ipListAuthoritative": true,
1619-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1619+
"lastCheckedAt": "2026-06-26T09:53:09Z",
16201620
"note": null,
16211621
"operator": "Apple",
16221622
"operatorCountry": "US",
@@ -1663,7 +1663,7 @@
16631663
"documentationUrl": null,
16641664
"id": "mojeekbot",
16651665
"ipListAuthoritative": true,
1666-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1666+
"lastCheckedAt": "2026-06-26T09:53:09Z",
16671667
"note": null,
16681668
"operator": "Mojeek",
16691669
"operatorCountry": "GB",
@@ -1699,7 +1699,7 @@
16991699
"documentationUrl": null,
17001700
"id": "naver-yeti",
17011701
"ipListAuthoritative": true,
1702-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1702+
"lastCheckedAt": "2026-06-26T09:53:09Z",
17031703
"note": null,
17041704
"operator": "Naver",
17051705
"operatorCountry": "KR",
@@ -1770,7 +1770,7 @@
17701770
"documentationUrl": null,
17711771
"id": "yandexbot",
17721772
"ipListAuthoritative": false,
1773-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1773+
"lastCheckedAt": "2026-06-26T09:53:09Z",
17741774
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
17751775
"operator": "Yandex",
17761776
"operatorCountry": "RU",
@@ -1819,7 +1819,7 @@
18191819
"documentationUrl": null,
18201820
"id": "baiduspider",
18211821
"ipListAuthoritative": false,
1822-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1822+
"lastCheckedAt": "2026-06-26T09:53:09Z",
18231823
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
18241824
"operator": "Baidu",
18251825
"operatorCountry": "CN",
@@ -1857,7 +1857,7 @@
18571857
"documentationUrl": null,
18581858
"id": "openai-gptbot",
18591859
"ipListAuthoritative": true,
1860-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1860+
"lastCheckedAt": "2026-06-26T09:53:09Z",
18611861
"note": null,
18621862
"operator": "OpenAI",
18631863
"operatorCountry": "US",
@@ -1907,7 +1907,7 @@
19071907
"documentationUrl": null,
19081908
"id": "openai-searchbot",
19091909
"ipListAuthoritative": true,
1910-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1910+
"lastCheckedAt": "2026-06-26T09:53:09Z",
19111911
"note": null,
19121912
"operator": "OpenAI",
19131913
"operatorCountry": "US",
@@ -1972,7 +1972,7 @@
19721972
"documentationUrl": null,
19731973
"id": "openai-chatgpt-user",
19741974
"ipListAuthoritative": true,
1975-
"lastCheckedAt": "2026-06-26T04:42:30Z",
1975+
"lastCheckedAt": "2026-06-26T09:53:09Z",
19761976
"note": null,
19771977
"operator": "OpenAI",
19781978
"operatorCountry": "US",
@@ -2226,7 +2226,7 @@
22262226
"documentationUrl": null,
22272227
"id": "openai-adsbot",
22282228
"ipListAuthoritative": false,
2229-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2229+
"lastCheckedAt": "2026-06-26T09:53:09Z",
22302230
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
22312231
"operator": "OpenAI",
22322232
"operatorCountry": "US",
@@ -2258,7 +2258,7 @@
22582258
"documentationUrl": null,
22592259
"id": "perplexitybot",
22602260
"ipListAuthoritative": true,
2261-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2261+
"lastCheckedAt": "2026-06-26T09:53:09Z",
22622262
"note": null,
22632263
"operator": "Perplexity",
22642264
"operatorCountry": "US",
@@ -2299,7 +2299,7 @@
22992299
"documentationUrl": null,
23002300
"id": "perplexity-user",
23012301
"ipListAuthoritative": true,
2302-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2302+
"lastCheckedAt": "2026-06-26T09:53:09Z",
23032303
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
23042304
"operator": "Perplexity",
23052305
"operatorCountry": "US",
@@ -2336,7 +2336,7 @@
23362336
"documentationUrl": null,
23372337
"id": "claudebot",
23382338
"ipListAuthoritative": false,
2339-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2339+
"lastCheckedAt": "2026-06-26T09:53:09Z",
23402340
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
23412341
"operator": "Anthropic",
23422342
"operatorCountry": "US",
@@ -2370,7 +2370,7 @@
23702370
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
23712371
"id": "amazonbot",
23722372
"ipListAuthoritative": true,
2373-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2373+
"lastCheckedAt": "2026-06-26T09:53:09Z",
23742374
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
23752375
"operator": "Amazon",
23762376
"operatorCountry": "US",
@@ -2929,7 +2929,7 @@
29292929
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
29302930
"id": "amazon-searchbot",
29312931
"ipListAuthoritative": true,
2932-
"lastCheckedAt": "2026-06-26T04:42:30Z",
2932+
"lastCheckedAt": "2026-06-26T09:53:09Z",
29332933
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
29342934
"operator": "Amazon",
29352935
"operatorCountry": "US",
@@ -3476,7 +3476,7 @@
34763476
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
34773477
"id": "amazon-amzn-user",
34783478
"ipListAuthoritative": true,
3479-
"lastCheckedAt": "2026-06-26T04:42:30Z",
3479+
"lastCheckedAt": "2026-06-26T09:53:09Z",
34803480
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
34813481
"operator": "Amazon",
34823482
"operatorCountry": "US",
@@ -4534,7 +4534,7 @@
45344534
"documentationUrl": null,
45354535
"id": "meta-ai-crawlers",
45364536
"ipListAuthoritative": false,
4537-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4537+
"lastCheckedAt": "2026-06-26T09:53:09Z",
45384538
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
45394539
"operator": "Meta",
45404540
"operatorCountry": "US",
@@ -4573,7 +4573,7 @@
45734573
"documentationUrl": null,
45744574
"id": "bytespider",
45754575
"ipListAuthoritative": false,
4576-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4576+
"lastCheckedAt": "2026-06-26T09:53:09Z",
45774577
"note": null,
45784578
"operator": "ByteDance",
45794579
"operatorCountry": "CN",
@@ -4605,7 +4605,7 @@
46054605
"documentationUrl": null,
46064606
"id": "mistralai-user",
46074607
"ipListAuthoritative": true,
4608-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4608+
"lastCheckedAt": "2026-06-26T09:53:09Z",
46094609
"note": null,
46104610
"operator": "Mistral AI",
46114611
"operatorCountry": "FR",
@@ -4642,7 +4642,7 @@
46424642
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
46434643
"id": "ahrefsbot",
46444644
"ipListAuthoritative": true,
4645-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4645+
"lastCheckedAt": "2026-06-26T09:53:09Z",
46464646
"note": null,
46474647
"operator": "Ahrefs",
46484648
"operatorCountry": "SG",
@@ -4729,7 +4729,7 @@
47294729
"documentationUrl": null,
47304730
"id": "lumar-crawler",
47314731
"ipListAuthoritative": true,
4732-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4732+
"lastCheckedAt": "2026-06-26T09:53:09Z",
47334733
"note": null,
47344734
"operator": "Lumar",
47354735
"operatorCountry": "GB",
@@ -4828,7 +4828,7 @@
48284828
"documentationUrl": null,
48294829
"id": "semrushbot",
48304830
"ipListAuthoritative": false,
4831-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4831+
"lastCheckedAt": "2026-06-26T09:53:09Z",
48324832
"note": null,
48334833
"operator": "Semrush",
48344834
"operatorCountry": "US",
@@ -4862,7 +4862,7 @@
48624862
"documentationUrl": null,
48634863
"id": "censys",
48644864
"ipListAuthoritative": false,
4865-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4865+
"lastCheckedAt": "2026-06-26T09:53:09Z",
48664866
"note": null,
48674867
"operator": "Censys",
48684868
"operatorCountry": "US",
@@ -4899,7 +4899,7 @@
48994899
"documentationUrl": null,
49004900
"id": "shodan",
49014901
"ipListAuthoritative": false,
4902-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4902+
"lastCheckedAt": "2026-06-26T09:53:09Z",
49034903
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
49044904
"operator": "Shodan",
49054905
"operatorCountry": "US",
@@ -4944,7 +4944,7 @@
49444944
"documentationUrl": null,
49454945
"id": "datadog-synthetics",
49464946
"ipListAuthoritative": true,
4947-
"lastCheckedAt": "2026-06-26T04:42:30Z",
4947+
"lastCheckedAt": "2026-06-26T09:53:09Z",
49484948
"note": null,
49494949
"operator": "Datadog",
49504950
"operatorCountry": "US",
@@ -5090,7 +5090,7 @@
50905090
"documentationUrl": null,
50915091
"id": "ias-crawler",
50925092
"ipListAuthoritative": true,
5093-
"lastCheckedAt": "2026-06-26T04:42:30Z",
5093+
"lastCheckedAt": "2026-06-26T09:53:09Z",
50945094
"note": null,
50955095
"operator": "Integral Ad Science",
50965096
"operatorCountry": "US",
@@ -5137,7 +5137,7 @@
51375137
"documentationUrl": null,
51385138
"id": "ttd-content",
51395139
"ipListAuthoritative": true,
5140-
"lastCheckedAt": "2026-06-26T04:42:30Z",
5140+
"lastCheckedAt": "2026-06-26T09:53:09Z",
51415141
"note": null,
51425142
"operator": "The Trade Desk",
51435143
"operatorCountry": "US",
@@ -7785,7 +7785,7 @@
77857785
"documentationUrl": "https://uptimerobot.com/locations",
77867786
"id": "uptimerobot",
77877787
"ipListAuthoritative": true,
7788-
"lastCheckedAt": "2026-06-26T04:42:30Z",
7788+
"lastCheckedAt": "2026-06-26T09:53:09Z",
77897789
"note": null,
77907790
"operator": "UptimeRobot",
77917791
"operatorCountry": "US",
@@ -8024,7 +8024,7 @@
80248024
"documentationUrl": null,
80258025
"id": "pingdom",
80268026
"ipListAuthoritative": true,
8027-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8027+
"lastCheckedAt": "2026-06-26T09:53:09Z",
80288028
"note": null,
80298029
"operator": "SolarWinds Pingdom",
80308030
"operatorCountry": "US",
@@ -8213,7 +8213,7 @@
82138213
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
82148214
"id": "statuscake",
82158215
"ipListAuthoritative": true,
8216-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8216+
"lastCheckedAt": "2026-06-26T09:53:09Z",
82178217
"note": null,
82188218
"operator": "StatusCake",
82198219
"operatorCountry": "GB",
@@ -8543,7 +8543,7 @@
85438543
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
85448544
"id": "better-stack",
85458545
"ipListAuthoritative": true,
8546-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8546+
"lastCheckedAt": "2026-06-26T09:53:09Z",
85478547
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
85488548
"operator": "Better Stack",
85498549
"operatorCountry": "CZ",
@@ -8611,7 +8611,7 @@
86118611
"documentationUrl": null,
86128612
"id": "ccbot",
86138613
"ipListAuthoritative": true,
8614-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8614+
"lastCheckedAt": "2026-06-26T09:53:09Z",
86158615
"note": null,
86168616
"operator": "Common Crawl",
86178617
"operatorCountry": "US",
@@ -8651,7 +8651,7 @@
86518651
"documentationUrl": null,
86528652
"id": "flipboard-crawler",
86538653
"ipListAuthoritative": true,
8654-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8654+
"lastCheckedAt": "2026-06-26T09:53:09Z",
86558655
"note": null,
86568656
"operator": "Flipboard",
86578657
"operatorCountry": "US",
@@ -8820,7 +8820,7 @@
88208820
"documentationUrl": null,
88218821
"id": "parsely-crawler",
88228822
"ipListAuthoritative": true,
8823-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8823+
"lastCheckedAt": "2026-06-26T09:53:09Z",
88248824
"note": null,
88258825
"operator": "Parse.ly",
88268826
"operatorCountry": "US",
@@ -8863,7 +8863,7 @@
88638863
"documentationUrl": null,
88648864
"id": "pinterestbot",
88658865
"ipListAuthoritative": false,
8866-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8866+
"lastCheckedAt": "2026-06-26T09:53:09Z",
88678867
"note": null,
88688868
"operator": "Pinterest",
88698869
"operatorCountry": "US",
@@ -8897,7 +8897,7 @@
88978897
"documentationUrl": null,
88988898
"id": "linkedinbot",
88998899
"ipListAuthoritative": false,
8900-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8900+
"lastCheckedAt": "2026-06-26T09:53:09Z",
89018901
"note": null,
89028902
"operator": "LinkedIn",
89038903
"operatorCountry": "US",
@@ -8929,7 +8929,7 @@
89298929
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
89308930
"id": "telegram-link-preview",
89318931
"ipListAuthoritative": true,
8932-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8932+
"lastCheckedAt": "2026-06-26T09:53:09Z",
89338933
"note": null,
89348934
"operator": "Telegram",
89358935
"operatorCountry": "AE",
@@ -8974,7 +8974,7 @@
89748974
"documentationUrl": "https://rssapi.net/faq",
89758975
"id": "rssapi-parser",
89768976
"ipListAuthoritative": true,
8977-
"lastCheckedAt": "2026-06-26T04:42:30Z",
8977+
"lastCheckedAt": "2026-06-26T09:53:09Z",
89788978
"note": null,
89798979
"operator": "RSS API",
89808980
"operatorCountry": "DE",

0 commit comments

Comments
 (0)