Added robots.txt rule.

[skip ci]
Eraldo · Dec 22, 2014 · a0aa679 · a0aa679
1 parent d4298fc
commit a0aa679
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/colegend/configuration/urls.py b/colegend/configuration/urls.py
@@ -4,6 +4,8 @@
 
 # Uncomment the next two lines to enable the admin:
 from django.contrib import admin
+from django.http import HttpResponse
+
 admin.autodiscover()
 
 
@@ -51,6 +53,8 @@
     # django-autocomplet-light
     url(r'^autocomplete/', include('autocomplete_light.urls')),
 
+    # robots.txt file for crawlers
+    (r'^robots.txt$', lambda r: HttpResponse("User-agent: *\nDisallow: /", mimetype="text/plain")),
 ) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
 # remove group model from admin