Merge branch 'add-robots-txt' into 'v2.9.x'

Adding robots.txt to fix random 404 message

See merge request allianceauth/allianceauth!1302

allianceauth/static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

docs/installation/apache.md

@@ -47,15 +47,23 @@ Place your virtual host configuration in the appropriate section within `/etc/ht
         ServerName auth.example.com
     
         ProxyPassMatch ^/static !
+        ProxyPassMatch ^/robots.txt !
+
         ProxyPass / http://127.0.0.1:8000/
         ProxyPassReverse / http://127.0.0.1:8000/
         ProxyPreserveHost On
 
         Alias "/static" "/var/www/myauth/static"
+        Alias "/robots.txt" "/var/www/myauth/static/robots.txt"
+
         <Directory "/var/www/myauth/static">
             Require all granted
         </Directory>
 
+        <Location "/robots.txt">
+            SetHandler None
+            Require all granted
+        </Location>
 </VirtualHost>
 ```
 

docs/installation/nginx.md

@@ -69,6 +69,10 @@ server {
         autoindex off;
     }
 
+    location /robots.txt {
+        alias /var/www/myauth/static/robots.txt;
+    }
+
     # Gunicorn config goes below
     location / {
         include proxy_params;