Permalink
Browse files

added robots.txt

  • Loading branch information...
1 parent 92b972a commit 3938039710a44e6117305730b0df9d4145ce12d1 @amyth committed Mar 8, 2016
Showing with 18 additions and 0 deletions.
  1. +4 −0 config/urls.py
  2. +14 −0 templates/robots.txt
View
@@ -16,7 +16,11 @@
from django.conf.urls import include, url
from django.contrib import admin
+from django.views.generic import TemplateView
+
+
urlpatterns = [
url(r'^', include('apps.home.urls', namespace='home')),
#url(r'^admin/', admin.site.urls),
+ url(r'^robots.txt$', TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), name="robots_file")
]
View
@@ -0,0 +1,14 @@
+User-agent: *
+Disallow:
+
+# too many repeated hits, too quick
+User-agent: litefinder
+Disallow: /
+
+# Yahoo. too many repeated hits, too quick
+User-agent: Slurp
+Disallow: /
+
+# too many repeated hits, too quick
+User-agent: Baidu
+Disallow: /

0 comments on commit 3938039

Please sign in to comment.