Skip to content

Commit 44c6fbb

Browse files
github-actions[bot]github-actions[bot]
committed
Publish CrawlerScope site
1 parent bb03921 commit 44c6fbb

3 files changed

Lines changed: 9069 additions & 44 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-06-14T14:32:46Z",
196+
"generatedAt": "2026-06-14T20:00:07Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-06-14T14:32:46Z",
230+
"lastCheckedAt": "2026-06-14T20:00:07Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -332,7 +332,7 @@
332332
"documentationUrl": null,
333333
"id": "google-special",
334334
"ipListAuthoritative": true,
335-
"lastCheckedAt": "2026-06-14T14:32:46Z",
335+
"lastCheckedAt": "2026-06-14T20:00:07Z",
336336
"note": null,
337337
"operator": "Google",
338338
"operatorCountry": "US",
@@ -410,7 +410,7 @@
410410
"documentationUrl": null,
411411
"id": "google-user-triggered",
412412
"ipListAuthoritative": true,
413-
"lastCheckedAt": "2026-06-14T14:32:46Z",
413+
"lastCheckedAt": "2026-06-14T20:00:07Z",
414414
"note": null,
415415
"operator": "Google",
416416
"operatorCountry": "US",
@@ -668,7 +668,7 @@
668668
"documentationUrl": null,
669669
"id": "bingbot",
670670
"ipListAuthoritative": true,
671-
"lastCheckedAt": "2026-06-14T14:32:46Z",
671+
"lastCheckedAt": "2026-06-14T20:00:07Z",
672672
"note": null,
673673
"operator": "Microsoft",
674674
"operatorCountry": "US",
@@ -732,7 +732,7 @@
732732
"documentationUrl": null,
733733
"id": "duckduckbot",
734734
"ipListAuthoritative": true,
735-
"lastCheckedAt": "2026-06-14T14:32:46Z",
735+
"lastCheckedAt": "2026-06-14T20:00:07Z",
736736
"note": null,
737737
"operator": "DuckDuckGo",
738738
"operatorCountry": "US",
@@ -1174,7 +1174,7 @@
11741174
"documentationUrl": null,
11751175
"id": "duckassistbot",
11761176
"ipListAuthoritative": true,
1177-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1177+
"lastCheckedAt": "2026-06-14T20:00:07Z",
11781178
"note": null,
11791179
"operator": "DuckDuckGo",
11801180
"operatorCountry": "US",
@@ -1616,7 +1616,7 @@
16161616
"documentationUrl": "https://support.apple.com/en-us/119829",
16171617
"id": "applebot",
16181618
"ipListAuthoritative": true,
1619-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1619+
"lastCheckedAt": "2026-06-14T20:00:07Z",
16201620
"note": null,
16211621
"operator": "Apple",
16221622
"operatorCountry": "US",
@@ -1663,7 +1663,7 @@
16631663
"documentationUrl": null,
16641664
"id": "mojeekbot",
16651665
"ipListAuthoritative": true,
1666-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1666+
"lastCheckedAt": "2026-06-14T20:00:07Z",
16671667
"note": null,
16681668
"operator": "Mojeek",
16691669
"operatorCountry": "GB",
@@ -1699,7 +1699,7 @@
16991699
"documentationUrl": null,
17001700
"id": "naver-yeti",
17011701
"ipListAuthoritative": true,
1702-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1702+
"lastCheckedAt": "2026-06-14T20:00:07Z",
17031703
"note": null,
17041704
"operator": "Naver",
17051705
"operatorCountry": "KR",
@@ -1770,7 +1770,7 @@
17701770
"documentationUrl": null,
17711771
"id": "yandexbot",
17721772
"ipListAuthoritative": false,
1773-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1773+
"lastCheckedAt": "2026-06-14T20:00:07Z",
17741774
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
17751775
"operator": "Yandex",
17761776
"operatorCountry": "RU",
@@ -1819,7 +1819,7 @@
18191819
"documentationUrl": null,
18201820
"id": "baiduspider",
18211821
"ipListAuthoritative": false,
1822-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1822+
"lastCheckedAt": "2026-06-14T20:00:07Z",
18231823
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
18241824
"operator": "Baidu",
18251825
"operatorCountry": "CN",
@@ -1857,7 +1857,7 @@
18571857
"documentationUrl": null,
18581858
"id": "openai-gptbot",
18591859
"ipListAuthoritative": true,
1860-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1860+
"lastCheckedAt": "2026-06-14T20:00:07Z",
18611861
"note": null,
18621862
"operator": "OpenAI",
18631863
"operatorCountry": "US",
@@ -1907,7 +1907,7 @@
19071907
"documentationUrl": null,
19081908
"id": "openai-searchbot",
19091909
"ipListAuthoritative": true,
1910-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1910+
"lastCheckedAt": "2026-06-14T20:00:07Z",
19111911
"note": null,
19121912
"operator": "OpenAI",
19131913
"operatorCountry": "US",
@@ -1972,7 +1972,7 @@
19721972
"documentationUrl": null,
19731973
"id": "openai-chatgpt-user",
19741974
"ipListAuthoritative": true,
1975-
"lastCheckedAt": "2026-06-14T14:32:46Z",
1975+
"lastCheckedAt": "2026-06-14T20:00:07Z",
19761976
"note": null,
19771977
"operator": "OpenAI",
19781978
"operatorCountry": "US",
@@ -2225,7 +2225,7 @@
22252225
"documentationUrl": null,
22262226
"id": "openai-adsbot",
22272227
"ipListAuthoritative": false,
2228-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2228+
"lastCheckedAt": "2026-06-14T20:00:07Z",
22292229
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
22302230
"operator": "OpenAI",
22312231
"operatorCountry": "US",
@@ -2257,7 +2257,7 @@
22572257
"documentationUrl": null,
22582258
"id": "perplexitybot",
22592259
"ipListAuthoritative": true,
2260-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2260+
"lastCheckedAt": "2026-06-14T20:00:07Z",
22612261
"note": null,
22622262
"operator": "Perplexity",
22632263
"operatorCountry": "US",
@@ -2298,7 +2298,7 @@
22982298
"documentationUrl": null,
22992299
"id": "perplexity-user",
23002300
"ipListAuthoritative": true,
2301-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2301+
"lastCheckedAt": "2026-06-14T20:00:07Z",
23022302
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
23032303
"operator": "Perplexity",
23042304
"operatorCountry": "US",
@@ -2335,7 +2335,7 @@
23352335
"documentationUrl": null,
23362336
"id": "claudebot",
23372337
"ipListAuthoritative": false,
2338-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2338+
"lastCheckedAt": "2026-06-14T20:00:07Z",
23392339
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
23402340
"operator": "Anthropic",
23412341
"operatorCountry": "US",
@@ -2369,7 +2369,7 @@
23692369
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
23702370
"id": "amazonbot",
23712371
"ipListAuthoritative": true,
2372-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2372+
"lastCheckedAt": "2026-06-14T20:00:07Z",
23732373
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
23742374
"operator": "Amazon",
23752375
"operatorCountry": "US",
@@ -2928,7 +2928,7 @@
29282928
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
29292929
"id": "amazon-searchbot",
29302930
"ipListAuthoritative": true,
2931-
"lastCheckedAt": "2026-06-14T14:32:46Z",
2931+
"lastCheckedAt": "2026-06-14T20:00:07Z",
29322932
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
29332933
"operator": "Amazon",
29342934
"operatorCountry": "US",
@@ -3475,7 +3475,7 @@
34753475
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
34763476
"id": "amazon-amzn-user",
34773477
"ipListAuthoritative": true,
3478-
"lastCheckedAt": "2026-06-14T14:32:46Z",
3478+
"lastCheckedAt": "2026-06-14T20:00:07Z",
34793479
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
34803480
"operator": "Amazon",
34813481
"operatorCountry": "US",
@@ -4533,7 +4533,7 @@
45334533
"documentationUrl": null,
45344534
"id": "meta-ai-crawlers",
45354535
"ipListAuthoritative": false,
4536-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4536+
"lastCheckedAt": "2026-06-14T20:00:07Z",
45374537
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
45384538
"operator": "Meta",
45394539
"operatorCountry": "US",
@@ -4572,7 +4572,7 @@
45724572
"documentationUrl": null,
45734573
"id": "bytespider",
45744574
"ipListAuthoritative": false,
4575-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4575+
"lastCheckedAt": "2026-06-14T20:00:07Z",
45764576
"note": null,
45774577
"operator": "ByteDance",
45784578
"operatorCountry": "CN",
@@ -4604,7 +4604,7 @@
46044604
"documentationUrl": null,
46054605
"id": "mistralai-user",
46064606
"ipListAuthoritative": true,
4607-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4607+
"lastCheckedAt": "2026-06-14T20:00:07Z",
46084608
"note": null,
46094609
"operator": "Mistral AI",
46104610
"operatorCountry": "FR",
@@ -4641,7 +4641,7 @@
46414641
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
46424642
"id": "ahrefsbot",
46434643
"ipListAuthoritative": true,
4644-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4644+
"lastCheckedAt": "2026-06-14T20:00:07Z",
46454645
"note": null,
46464646
"operator": "Ahrefs",
46474647
"operatorCountry": "SG",
@@ -4728,7 +4728,7 @@
47284728
"documentationUrl": null,
47294729
"id": "lumar-crawler",
47304730
"ipListAuthoritative": true,
4731-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4731+
"lastCheckedAt": "2026-06-14T20:00:07Z",
47324732
"note": null,
47334733
"operator": "Lumar",
47344734
"operatorCountry": "GB",
@@ -4827,7 +4827,7 @@
48274827
"documentationUrl": null,
48284828
"id": "semrushbot",
48294829
"ipListAuthoritative": false,
4830-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4830+
"lastCheckedAt": "2026-06-14T20:00:07Z",
48314831
"note": null,
48324832
"operator": "Semrush",
48334833
"operatorCountry": "US",
@@ -4861,7 +4861,7 @@
48614861
"documentationUrl": null,
48624862
"id": "censys",
48634863
"ipListAuthoritative": false,
4864-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4864+
"lastCheckedAt": "2026-06-14T20:00:07Z",
48654865
"note": null,
48664866
"operator": "Censys",
48674867
"operatorCountry": "US",
@@ -4898,7 +4898,7 @@
48984898
"documentationUrl": null,
48994899
"id": "shodan",
49004900
"ipListAuthoritative": false,
4901-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4901+
"lastCheckedAt": "2026-06-14T20:00:07Z",
49024902
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
49034903
"operator": "Shodan",
49044904
"operatorCountry": "US",
@@ -4943,7 +4943,7 @@
49434943
"documentationUrl": null,
49444944
"id": "datadog-synthetics",
49454945
"ipListAuthoritative": true,
4946-
"lastCheckedAt": "2026-06-14T14:32:46Z",
4946+
"lastCheckedAt": "2026-06-14T20:00:07Z",
49474947
"note": null,
49484948
"operator": "Datadog",
49494949
"operatorCountry": "US",
@@ -5089,7 +5089,7 @@
50895089
"documentationUrl": null,
50905090
"id": "ias-crawler",
50915091
"ipListAuthoritative": true,
5092-
"lastCheckedAt": "2026-06-14T14:32:46Z",
5092+
"lastCheckedAt": "2026-06-14T20:00:07Z",
50935093
"note": null,
50945094
"operator": "Integral Ad Science",
50955095
"operatorCountry": "US",
@@ -5136,7 +5136,7 @@
51365136
"documentationUrl": null,
51375137
"id": "ttd-content",
51385138
"ipListAuthoritative": true,
5139-
"lastCheckedAt": "2026-06-14T14:32:46Z",
5139+
"lastCheckedAt": "2026-06-14T20:00:07Z",
51405140
"note": null,
51415141
"operator": "The Trade Desk",
51425142
"operatorCountry": "US",
@@ -7784,7 +7784,7 @@
77847784
"documentationUrl": "https://uptimerobot.com/locations",
77857785
"id": "uptimerobot",
77867786
"ipListAuthoritative": true,
7787-
"lastCheckedAt": "2026-06-14T14:32:46Z",
7787+
"lastCheckedAt": "2026-06-14T20:00:07Z",
77887788
"note": null,
77897789
"operator": "UptimeRobot",
77907790
"operatorCountry": "US",
@@ -8035,7 +8035,7 @@
80358035
"documentationUrl": null,
80368036
"id": "pingdom",
80378037
"ipListAuthoritative": true,
8038-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8038+
"lastCheckedAt": "2026-06-14T20:00:07Z",
80398039
"note": null,
80408040
"operator": "SolarWinds Pingdom",
80418041
"operatorCountry": "US",
@@ -8224,7 +8224,7 @@
82248224
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
82258225
"id": "statuscake",
82268226
"ipListAuthoritative": true,
8227-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8227+
"lastCheckedAt": "2026-06-14T20:00:07Z",
82288228
"note": null,
82298229
"operator": "StatusCake",
82308230
"operatorCountry": "GB",
@@ -8554,7 +8554,7 @@
85548554
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
85558555
"id": "better-stack",
85568556
"ipListAuthoritative": true,
8557-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8557+
"lastCheckedAt": "2026-06-14T20:00:07Z",
85588558
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
85598559
"operator": "Better Stack",
85608560
"operatorCountry": "CZ",
@@ -8622,7 +8622,7 @@
86228622
"documentationUrl": null,
86238623
"id": "ccbot",
86248624
"ipListAuthoritative": true,
8625-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8625+
"lastCheckedAt": "2026-06-14T20:00:07Z",
86268626
"note": null,
86278627
"operator": "Common Crawl",
86288628
"operatorCountry": "US",
@@ -8662,7 +8662,7 @@
86628662
"documentationUrl": null,
86638663
"id": "flipboard-crawler",
86648664
"ipListAuthoritative": true,
8665-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8665+
"lastCheckedAt": "2026-06-14T20:00:07Z",
86668666
"note": null,
86678667
"operator": "Flipboard",
86688668
"operatorCountry": "US",
@@ -8831,7 +8831,7 @@
88318831
"documentationUrl": null,
88328832
"id": "parsely-crawler",
88338833
"ipListAuthoritative": true,
8834-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8834+
"lastCheckedAt": "2026-06-14T20:00:07Z",
88358835
"note": null,
88368836
"operator": "Parse.ly",
88378837
"operatorCountry": "US",
@@ -8874,7 +8874,7 @@
88748874
"documentationUrl": null,
88758875
"id": "pinterestbot",
88768876
"ipListAuthoritative": false,
8877-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8877+
"lastCheckedAt": "2026-06-14T20:00:07Z",
88788878
"note": null,
88798879
"operator": "Pinterest",
88808880
"operatorCountry": "US",
@@ -8908,7 +8908,7 @@
89088908
"documentationUrl": null,
89098909
"id": "linkedinbot",
89108910
"ipListAuthoritative": false,
8911-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8911+
"lastCheckedAt": "2026-06-14T20:00:07Z",
89128912
"note": null,
89138913
"operator": "LinkedIn",
89148914
"operatorCountry": "US",
@@ -8940,7 +8940,7 @@
89408940
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
89418941
"id": "telegram-link-preview",
89428942
"ipListAuthoritative": true,
8943-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8943+
"lastCheckedAt": "2026-06-14T20:00:07Z",
89448944
"note": null,
89458945
"operator": "Telegram",
89468946
"operatorCountry": "AE",
@@ -8985,7 +8985,7 @@
89858985
"documentationUrl": "https://rssapi.net/faq",
89868986
"id": "rssapi-parser",
89878987
"ipListAuthoritative": true,
8988-
"lastCheckedAt": "2026-06-14T14:32:46Z",
8988+
"lastCheckedAt": "2026-06-14T20:00:07Z",
89898989
"note": null,
89908990
"operator": "RSS API",
89918991
"operatorCountry": "DE",

0 commit comments

Comments
 (0)