Skip to content

Commit f80176e

Browse files
github-actions[bot]github-actions[bot]
committed
Update CrawlerScope data
1 parent caa7105 commit f80176e

3 files changed

Lines changed: 232 additions & 232 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-08-22T13:03:14Z",
196+
"generatedAt": "2026-08-22T18:50:46Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-08-22T13:03:14Z",
230+
"lastCheckedAt": "2026-08-22T18:50:46Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -332,7 +332,7 @@
332332
"documentationUrl": null,
333333
"id": "google-special",
334334
"ipListAuthoritative": true,
335-
"lastCheckedAt": "2026-08-22T13:03:14Z",
335+
"lastCheckedAt": "2026-08-22T18:50:46Z",
336336
"note": null,
337337
"operator": "Google",
338338
"operatorCountry": "US",
@@ -410,7 +410,7 @@
410410
"documentationUrl": null,
411411
"id": "google-user-triggered",
412412
"ipListAuthoritative": true,
413-
"lastCheckedAt": "2026-08-22T13:03:14Z",
413+
"lastCheckedAt": "2026-08-22T18:50:46Z",
414414
"note": null,
415415
"operator": "Google",
416416
"operatorCountry": "US",
@@ -670,7 +670,7 @@
670670
"documentationUrl": null,
671671
"id": "bingbot",
672672
"ipListAuthoritative": true,
673-
"lastCheckedAt": "2026-08-22T13:03:14Z",
673+
"lastCheckedAt": "2026-08-22T18:50:46Z",
674674
"note": null,
675675
"operator": "Microsoft",
676676
"operatorCountry": "US",
@@ -734,7 +734,7 @@
734734
"documentationUrl": null,
735735
"id": "duckduckbot",
736736
"ipListAuthoritative": true,
737-
"lastCheckedAt": "2026-08-22T13:03:14Z",
737+
"lastCheckedAt": "2026-08-22T18:50:46Z",
738738
"note": null,
739739
"operator": "DuckDuckGo",
740740
"operatorCountry": "US",
@@ -1248,7 +1248,7 @@
12481248
"documentationUrl": null,
12491249
"id": "duckassistbot",
12501250
"ipListAuthoritative": true,
1251-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1251+
"lastCheckedAt": "2026-08-22T18:50:46Z",
12521252
"note": null,
12531253
"operator": "DuckDuckGo",
12541254
"operatorCountry": "US",
@@ -1762,7 +1762,7 @@
17621762
"documentationUrl": "https://support.apple.com/en-us/119829",
17631763
"id": "applebot",
17641764
"ipListAuthoritative": true,
1765-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1765+
"lastCheckedAt": "2026-08-22T18:50:46Z",
17661766
"note": null,
17671767
"operator": "Apple",
17681768
"operatorCountry": "US",
@@ -1818,7 +1818,7 @@
18181818
"documentationUrl": null,
18191819
"id": "mojeekbot",
18201820
"ipListAuthoritative": true,
1821-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1821+
"lastCheckedAt": "2026-08-22T18:50:46Z",
18221822
"note": null,
18231823
"operator": "Mojeek",
18241824
"operatorCountry": "GB",
@@ -1854,7 +1854,7 @@
18541854
"documentationUrl": null,
18551855
"id": "naver-yeti",
18561856
"ipListAuthoritative": true,
1857-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1857+
"lastCheckedAt": "2026-08-22T18:50:46Z",
18581858
"note": null,
18591859
"operator": "Naver",
18601860
"operatorCountry": "KR",
@@ -1925,7 +1925,7 @@
19251925
"documentationUrl": null,
19261926
"id": "yandexbot",
19271927
"ipListAuthoritative": false,
1928-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1928+
"lastCheckedAt": "2026-08-22T18:50:46Z",
19291929
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
19301930
"operator": "Yandex",
19311931
"operatorCountry": "RU",
@@ -1974,7 +1974,7 @@
19741974
"documentationUrl": null,
19751975
"id": "baiduspider",
19761976
"ipListAuthoritative": false,
1977-
"lastCheckedAt": "2026-08-22T13:03:14Z",
1977+
"lastCheckedAt": "2026-08-22T18:50:46Z",
19781978
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
19791979
"operator": "Baidu",
19801980
"operatorCountry": "CN",
@@ -2012,7 +2012,7 @@
20122012
"documentationUrl": null,
20132013
"id": "openai-gptbot",
20142014
"ipListAuthoritative": true,
2015-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2015+
"lastCheckedAt": "2026-08-22T18:50:46Z",
20162016
"note": null,
20172017
"operator": "OpenAI",
20182018
"operatorCountry": "US",
@@ -2062,7 +2062,7 @@
20622062
"documentationUrl": null,
20632063
"id": "openai-searchbot",
20642064
"ipListAuthoritative": true,
2065-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2065+
"lastCheckedAt": "2026-08-22T18:50:46Z",
20662066
"note": null,
20672067
"operator": "OpenAI",
20682068
"operatorCountry": "US",
@@ -2127,7 +2127,7 @@
21272127
"documentationUrl": null,
21282128
"id": "openai-chatgpt-user",
21292129
"ipListAuthoritative": true,
2130-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2130+
"lastCheckedAt": "2026-08-22T18:50:46Z",
21312131
"note": null,
21322132
"operator": "OpenAI",
21332133
"operatorCountry": "US",
@@ -2353,7 +2353,7 @@
23532353
"documentationUrl": null,
23542354
"id": "openai-adsbot",
23552355
"ipListAuthoritative": false,
2356-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2356+
"lastCheckedAt": "2026-08-22T18:50:46Z",
23572357
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
23582358
"operator": "OpenAI",
23592359
"operatorCountry": "US",
@@ -2385,7 +2385,7 @@
23852385
"documentationUrl": null,
23862386
"id": "perplexitybot",
23872387
"ipListAuthoritative": true,
2388-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2388+
"lastCheckedAt": "2026-08-22T18:50:46Z",
23892389
"note": null,
23902390
"operator": "Perplexity",
23912391
"operatorCountry": "US",
@@ -2426,7 +2426,7 @@
24262426
"documentationUrl": null,
24272427
"id": "perplexity-user",
24282428
"ipListAuthoritative": true,
2429-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2429+
"lastCheckedAt": "2026-08-22T18:50:46Z",
24302430
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
24312431
"operator": "Perplexity",
24322432
"operatorCountry": "US",
@@ -2463,7 +2463,7 @@
24632463
"documentationUrl": null,
24642464
"id": "claudebot",
24652465
"ipListAuthoritative": false,
2466-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2466+
"lastCheckedAt": "2026-08-22T18:50:46Z",
24672467
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
24682468
"operator": "Anthropic",
24692469
"operatorCountry": "US",
@@ -2497,7 +2497,7 @@
24972497
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
24982498
"id": "amazonbot",
24992499
"ipListAuthoritative": true,
2500-
"lastCheckedAt": "2026-08-22T13:03:14Z",
2500+
"lastCheckedAt": "2026-08-22T18:50:46Z",
25012501
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
25022502
"operator": "Amazon",
25032503
"operatorCountry": "US",
@@ -3056,7 +3056,7 @@
30563056
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
30573057
"id": "amazon-searchbot",
30583058
"ipListAuthoritative": true,
3059-
"lastCheckedAt": "2026-08-22T13:03:14Z",
3059+
"lastCheckedAt": "2026-08-22T18:50:46Z",
30603060
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
30613061
"operator": "Amazon",
30623062
"operatorCountry": "US",
@@ -3603,7 +3603,7 @@
36033603
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
36043604
"id": "amazon-amzn-user",
36053605
"ipListAuthoritative": true,
3606-
"lastCheckedAt": "2026-08-22T13:03:14Z",
3606+
"lastCheckedAt": "2026-08-22T18:50:46Z",
36073607
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
36083608
"operator": "Amazon",
36093609
"operatorCountry": "US",
@@ -4661,7 +4661,7 @@
46614661
"documentationUrl": null,
46624662
"id": "meta-ai-crawlers",
46634663
"ipListAuthoritative": false,
4664-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4664+
"lastCheckedAt": "2026-08-22T18:50:46Z",
46654665
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
46664666
"operator": "Meta",
46674667
"operatorCountry": "US",
@@ -4700,7 +4700,7 @@
47004700
"documentationUrl": null,
47014701
"id": "bytespider",
47024702
"ipListAuthoritative": false,
4703-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4703+
"lastCheckedAt": "2026-08-22T18:50:46Z",
47044704
"note": null,
47054705
"operator": "ByteDance",
47064706
"operatorCountry": "CN",
@@ -4732,7 +4732,7 @@
47324732
"documentationUrl": null,
47334733
"id": "mistralai-user",
47344734
"ipListAuthoritative": true,
4735-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4735+
"lastCheckedAt": "2026-08-22T18:50:46Z",
47364736
"note": null,
47374737
"operator": "Mistral AI",
47384738
"operatorCountry": "FR",
@@ -4769,7 +4769,7 @@
47694769
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
47704770
"id": "ahrefsbot",
47714771
"ipListAuthoritative": true,
4772-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4772+
"lastCheckedAt": "2026-08-22T18:50:46Z",
47734773
"note": null,
47744774
"operator": "Ahrefs",
47754775
"operatorCountry": "SG",
@@ -4886,7 +4886,7 @@
48864886
"documentationUrl": null,
48874887
"id": "lumar-crawler",
48884888
"ipListAuthoritative": true,
4889-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4889+
"lastCheckedAt": "2026-08-22T18:50:46Z",
48904890
"note": null,
48914891
"operator": "Lumar",
48924892
"operatorCountry": "GB",
@@ -4985,7 +4985,7 @@
49854985
"documentationUrl": null,
49864986
"id": "semrushbot",
49874987
"ipListAuthoritative": false,
4988-
"lastCheckedAt": "2026-08-22T13:03:14Z",
4988+
"lastCheckedAt": "2026-08-22T18:50:46Z",
49894989
"note": null,
49904990
"operator": "Semrush",
49914991
"operatorCountry": "US",
@@ -5019,7 +5019,7 @@
50195019
"documentationUrl": null,
50205020
"id": "censys",
50215021
"ipListAuthoritative": false,
5022-
"lastCheckedAt": "2026-08-22T13:03:14Z",
5022+
"lastCheckedAt": "2026-08-22T18:50:46Z",
50235023
"note": null,
50245024
"operator": "Censys",
50255025
"operatorCountry": "US",
@@ -5056,7 +5056,7 @@
50565056
"documentationUrl": null,
50575057
"id": "shodan",
50585058
"ipListAuthoritative": false,
5059-
"lastCheckedAt": "2026-08-22T13:03:14Z",
5059+
"lastCheckedAt": "2026-08-22T18:50:46Z",
50605060
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
50615061
"operator": "Shodan",
50625062
"operatorCountry": "US",
@@ -5101,7 +5101,7 @@
51015101
"documentationUrl": null,
51025102
"id": "datadog-synthetics",
51035103
"ipListAuthoritative": true,
5104-
"lastCheckedAt": "2026-08-22T13:03:14Z",
5104+
"lastCheckedAt": "2026-08-22T18:50:46Z",
51055105
"note": null,
51065106
"operator": "Datadog",
51075107
"operatorCountry": "US",
@@ -5247,7 +5247,7 @@
52475247
"documentationUrl": null,
52485248
"id": "ias-crawler",
52495249
"ipListAuthoritative": true,
5250-
"lastCheckedAt": "2026-08-22T13:03:14Z",
5250+
"lastCheckedAt": "2026-08-22T18:50:46Z",
52515251
"note": null,
52525252
"operator": "Integral Ad Science",
52535253
"operatorCountry": "US",
@@ -5294,7 +5294,7 @@
52945294
"documentationUrl": null,
52955295
"id": "ttd-content",
52965296
"ipListAuthoritative": true,
5297-
"lastCheckedAt": "2026-08-22T13:03:14Z",
5297+
"lastCheckedAt": "2026-08-22T18:50:46Z",
52985298
"note": null,
52995299
"operator": "The Trade Desk",
53005300
"operatorCountry": "US",
@@ -7942,7 +7942,7 @@
79427942
"documentationUrl": "https://uptimerobot.com/locations",
79437943
"id": "uptimerobot",
79447944
"ipListAuthoritative": true,
7945-
"lastCheckedAt": "2026-08-22T13:03:14Z",
7945+
"lastCheckedAt": "2026-08-22T18:50:46Z",
79467946
"note": null,
79477947
"operator": "UptimeRobot",
79487948
"operatorCountry": "US",
@@ -8181,7 +8181,7 @@
81818181
"documentationUrl": null,
81828182
"id": "pingdom",
81838183
"ipListAuthoritative": true,
8184-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8184+
"lastCheckedAt": "2026-08-22T18:50:46Z",
81858185
"note": null,
81868186
"operator": "SolarWinds Pingdom",
81878187
"operatorCountry": "US",
@@ -8370,7 +8370,7 @@
83708370
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
83718371
"id": "statuscake",
83728372
"ipListAuthoritative": true,
8373-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8373+
"lastCheckedAt": "2026-08-22T18:50:46Z",
83748374
"note": null,
83758375
"operator": "StatusCake",
83768376
"operatorCountry": "GB",
@@ -8700,7 +8700,7 @@
87008700
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
87018701
"id": "better-stack",
87028702
"ipListAuthoritative": true,
8703-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8703+
"lastCheckedAt": "2026-08-22T18:50:46Z",
87048704
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
87058705
"operator": "Better Stack",
87068706
"operatorCountry": "CZ",
@@ -8768,7 +8768,7 @@
87688768
"documentationUrl": null,
87698769
"id": "ccbot",
87708770
"ipListAuthoritative": true,
8771-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8771+
"lastCheckedAt": "2026-08-22T18:50:46Z",
87728772
"note": null,
87738773
"operator": "Common Crawl",
87748774
"operatorCountry": "US",
@@ -8808,7 +8808,7 @@
88088808
"documentationUrl": null,
88098809
"id": "flipboard-crawler",
88108810
"ipListAuthoritative": true,
8811-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8811+
"lastCheckedAt": "2026-08-22T18:50:46Z",
88128812
"note": null,
88138813
"operator": "Flipboard",
88148814
"operatorCountry": "US",
@@ -8977,7 +8977,7 @@
89778977
"documentationUrl": null,
89788978
"id": "parsely-crawler",
89798979
"ipListAuthoritative": true,
8980-
"lastCheckedAt": "2026-08-22T13:03:14Z",
8980+
"lastCheckedAt": "2026-08-22T18:50:46Z",
89818981
"note": null,
89828982
"operator": "Parse.ly",
89838983
"operatorCountry": "US",
@@ -9020,7 +9020,7 @@
90209020
"documentationUrl": null,
90219021
"id": "pinterestbot",
90229022
"ipListAuthoritative": false,
9023-
"lastCheckedAt": "2026-08-22T13:03:14Z",
9023+
"lastCheckedAt": "2026-08-22T18:50:46Z",
90249024
"note": null,
90259025
"operator": "Pinterest",
90269026
"operatorCountry": "US",
@@ -9054,7 +9054,7 @@
90549054
"documentationUrl": null,
90559055
"id": "linkedinbot",
90569056
"ipListAuthoritative": false,
9057-
"lastCheckedAt": "2026-08-22T13:03:14Z",
9057+
"lastCheckedAt": "2026-08-22T18:50:46Z",
90589058
"note": null,
90599059
"operator": "LinkedIn",
90609060
"operatorCountry": "US",
@@ -9086,7 +9086,7 @@
90869086
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
90879087
"id": "telegram-link-preview",
90889088
"ipListAuthoritative": true,
9089-
"lastCheckedAt": "2026-08-22T13:03:14Z",
9089+
"lastCheckedAt": "2026-08-22T18:50:46Z",
90909090
"note": null,
90919091
"operator": "Telegram",
90929092
"operatorCountry": "AE",
@@ -9131,7 +9131,7 @@
91319131
"documentationUrl": "https://rssapi.net/faq",
91329132
"id": "rssapi-parser",
91339133
"ipListAuthoritative": true,
9134-
"lastCheckedAt": "2026-08-22T13:03:14Z",
9134+
"lastCheckedAt": "2026-08-22T18:50:46Z",
91359135
"note": null,
91369136
"operator": "RSS API",
91379137
"operatorCountry": "DE",

0 commit comments

Comments
 (0)