robots.txt added

This fixes the randomly appearing 404 error in Auth  (issue #1232)

allianceauth/static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

docs/installation/apache.md

@@ -45,17 +45,25 @@ Place your virtual host configuration in the appropriate section within `/etc/ht
 ```
 <VirtualHost *:80>
         ServerName auth.example.com
-
+    
         ProxyPassMatch ^/static !
+        ProxyPassMatch ^/robots.txt !
+
         ProxyPass / http://127.0.0.1:8000/
         ProxyPassReverse / http://127.0.0.1:8000/
         ProxyPreserveHost On
 
         Alias "/static" "/var/www/myauth/static"
+        Alias "/robots.txt" "/var/www/myauth/static/robots.txt"
+
         <Directory "/var/www/myauth/static">
             Require all granted
         </Directory>
 
+        <Location "/robots.txt">
+            SetHandler None
+            Require all granted
+        </Location>
 </VirtualHost>
 ```
 

docs/installation/nginx.md

@@ -69,6 +69,10 @@ server {
         autoindex off;
     }
 
+    location /robots.txt {
+        alias /var/www/myauth/static/robots.txt;
+    }
+
     # Gunicorn config goes below
     location / {
         include proxy_params;