Update robots.txt

public/robots.txt

@@ -5,7 +5,7 @@ Disallow: /links/
 
 
 User-agent: 360Monitoring
-User-agent: AI Search
+User-agent: AI2Bot
 User-agent: AI2Bot-DeepResearchEval
 User-agent: ActiveComply Bot
 User-agent: AddSearchBot
@@ -17,159 +17,171 @@ User-agent: Amazonbot
 User-agent: Amzn-SearchBot
 User-agent: Amzn-User
 User-agent: Anomura
+User-agent: Apple App Site Association
 User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Audisto Crawler
-User-agent: Automaton
 User-agent: AwarioSmartBot
 User-agent: AzureAI-SearchBot
 User-agent: BestChange Bot
 User-agent: Better Uptime
+User-agent: BigScootsMonitor
 User-agent: BinaryCanary
 User-agent: Bing Ads
 User-agent: BlogVault
 User-agent: Bravebot
 User-agent: BrightEdge Bot
-User-agent: BugsNag
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: Channel3Bot
 User-agent: ChatGLM-Spider
-User-agent: ChatGPT agent
 User-agent: ChatGPT-User
+User-agent: Chathive crawler
 User-agent: Checkly Bot
 User-agent: Claude-SearchBot
 User-agent: Claude-User
 User-agent: ClaudeBot
 User-agent: CloudVertexBot
+User-agent: Cloudflare CustomerSLO Prober
+User-agent: Cloudflare CustomerSLO Prober (Staging)
+User-agent: Cloudflare Observatory Curly
 User-agent: Cloudflare-AutoRAG
 User-agent: Cludo
+User-agent: CohereBot
 User-agent: ContentKing
+User-agent: ContentSquare Static Resource Scraper
 User-agent: ContextualBot
 User-agent: Cotoyogi
 User-agent: Crazy Egg Bot
 User-agent: CriteoBot
 User-agent: Datenbank Crawler
 User-agent: Deepcrawl
-User-agent: Devin
-User-agent: Diffbot
+User-agent: Direqt Anomura
 User-agent: Doctom Monitor
 User-agent: DuckAssistBot
 User-agent: DuckAssistbot
-User-agent: Echobot Bot
 User-agent: FacebookBot
 User-agent: Factset_spyderbot
-User-agent: Fedicabot
 User-agent: Foregenix ThreatView/WebScan
 User-agent: FullStory
 User-agent: GPTBot
 User-agent: GTmetrix
+User-agent: GeistHaus-PageFetcher
 User-agent: Gemini-Deep-Research
 User-agent: Ghost Inspector
 User-agent: Google AdsBot
 User-agent: Google Adsense: Mediapartners-Google
 User-agent: Google Firebase
-User-agent: Google NotebookLM
 User-agent: Google Schema Markup Testing Tool
 User-agent: Google-CloudVertexBot
 User-agent: Google-Extended
 User-agent: Google-NotebookLM
+User-agent: Google-TravelAds-Live
+User-agent: GoogleAgent-URLContext
 User-agent: GoogleOther
+User-agent: Grafana Synthetic Monitoring
 User-agent: Grafana's Synthetic Monitoring
 User-agent: HetrixTools
 User-agent: HostTracker
 User-agent: Hotjar
+User-agent: HubSpot Crawler
 User-agent: IAS crawler
 User-agent: ICC Crawler
 User-agent: ICC-Crawler
+User-agent: Idealo
 User-agent: IndeedJobBot
+User-agent: InstatusBot
 User-agent: InternetArchiveBot
-User-agent: Jagged Pixel UptimeBot
-User-agent: Jetpack
 User-agent: Kangaroo Bot
-User-agent: Kinsta
+User-agent: Kimi-SearchBot
+User-agent: Kimi-User
+User-agent: KimiBot
 User-agent: KlaviyoAIBot
 User-agent: LCC
 User-agent: LeikiBot
 User-agent: LinerBot
 User-agent: LinkupBot
 User-agent: MagnetmeBot
-User-agent: Manus Bot
-User-agent: MediaMonitoringBot
-User-agent: Medialogia Bot
+User-agent: Marfeel
 User-agent: Mediatoolkitbot
-User-agent: Mediavine Medatada Parser
 User-agent: Meta-ExternalAds
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
 User-agent: MirrorWebCrawler
+User-agent: MistralAI-Index
 User-agent: MistralAI-User
+User-agent: MistralBot
 User-agent: MonSpark
-User-agent: Monitage
 User-agent: MontasticMonitor
 User-agent: NETVIGIE
+User-agent: Navu
 User-agent: New Relic
-User-agent: NixStatsMonitoringBot
 User-agent: Nodeping
 User-agent: NostoCrawlerBot
+User-agent: OAI-AdsBot
 User-agent: OAI-SearchBot
 User-agent: Oh Dear
 User-agent: Onetrust CMP Scanner
 User-agent: Outbrain
 User-agent: PanguBot
 User-agent: Panopta
 User-agent: Parse.ly
+User-agent: PerformlineCrawler
 User-agent: Perplexity-User
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: PhindBot
+User-agent: Ping-Admin
 User-agent: PingPing Bot
 User-agent: Pingdom
 User-agent: PinterestBot
 User-agent: Poggio-Citations
-User-agent: Productsup Crawler
 User-agent: Proximic
 User-agent: Pulsetic
 User-agent: QualifiedBot
 User-agent: Quantcastbot
 User-agent: Rackspace
 User-agent: Reelevant
+User-agent: Retool
 User-agent: SBIntuitionsBot
 User-agent: SE Ranking Bot
 User-agent: SalesViewerBot
+User-agent: Selectika AI
 User-agent: SemrushBot-OCOB
+User-agent: SentiBot
 User-agent: Sentry Uptime Monitoring
+User-agent: Shap-User
+User-agent: ShapBot
+User-agent: SirdataBot
 User-agent: Site24x7
 User-agent: SiteUpTimeBot
+User-agent: Skroutz ImageBot
 User-agent: Slickstream
 User-agent: SolarWinds Observability
 User-agent: Spider
 User-agent: Splunk Synthetics
-User-agent: StatsDroneBot
+User-agent: Stape Scanner
 User-agent: StatusCake
 User-agent: Synthetic Bot
 User-agent: TTD Content
 User-agent: Taboola
-User-agent: TavilyBot
 User-agent: Thousand Eyes Cloud Agent
 User-agent: Timpibot
 User-agent: Trendiction Bot
 User-agent: UpDownBot
 User-agent: Uptime LLC
+User-agent: Uptime Monitoring
 User-agent: UptimeBot
 User-agent: UptimeRobot
 User-agent: Uptimia
 User-agent: UptrendsBot
+User-agent: UseAI
 User-agent: VelenPublicWebCrawler
-User-agent: WPMUDEV Uptime Monitor 5.0
-User-agent: Watchbot
 User-agent: WebsitePulseBot
 User-agent: Webstatus247
 User-agent: WormlyBot
-User-agent: Yahoo Ad Monitoring
 User-agent: Yahoo Japan
 User-agent: Yext Inc
-User-agent: YouBot
 User-agent: ZanistaBot
 User-agent: Zvelo
 User-agent: atlassian-bot
@@ -185,11 +197,13 @@ User-agent: kagi-fetcher
 User-agent: klaviyo
 User-agent: laion-huggingface-processor
 User-agent: logicmonitor
+User-agent: mediaboardbot
 User-agent: meta-externalagent
 User-agent: meta-externalfetcher
 User-agent: meta-webindexer
 User-agent: netEstate Imprint Crawler
 User-agent: omgili
-User-agent: webpagetest
+User-agent: qcbot
 User-agent: webzio-extended
+User-agent: xAI-SearchBot
 Disallow: /