Add more bots to robots.txt and apply it to blog

Author: Nakaner

ansible/group_vars/blog.yml

@@ -26,6 +26,7 @@ apache:
         DocumentRoot /var/www/blog
         ServerName blog.openrailwaymap.org
 
+        Alias /robots.txt /var/www/common/ban_bad_robots.txt
         <Directory /var/www/blog>
             AllowOverride None
         </Directory>

ansible/group_vars/webservers.yml

@@ -33,6 +33,9 @@ robots_txt: |-
   User-agent: OAI-SearchBot
   User-agent: GPTBot
   User-agent: ChatGPT-User
+  User-agent: ClaudeBot
+  User-agent: YaK
+  User-agent: Flyriverbot
   User-agent: Amazonbot
   User-agent: MistralAI-User
   User-agent: AhrefsBot