Adding robots.txt file to keep bots from crawling certain parts of the site. Bug 665231.

Author: Fred Wenzel

ffdemo/settings.py

@@ -39,7 +39,8 @@
 USE_L10N = True
 
 # Paths that don't require a locale prefix.
-SUPPORTED_NONLOCALES = ('media', 'admin', 'requests', 'accounts', 'moderate')
+SUPPORTED_NONLOCALES = ('media', 'admin', 'requests', 'accounts', 'moderate',
+                        'robots.txt')
 
 # Gettext text domain
 TEXT_DOMAIN = 'django'

ffdemo/templates_orig/robots.txt

@@ -0,0 +1,12 @@
+User-agent: *
+Disallow: /localeurl/
+Disallow: /i18n/
+Disallow: /linear_sammy.html
+Disallow: /makemark_sammy.html
+Disallow: /mark/
+Disallow: /gml/
+Disallow: /accounts/
+Disallow: /list_invites/
+Disallow: /moderate/
+Disallow: /requests/
+Disallow: /newsletter/subscribe/

ffdemo/urls.py

@@ -1,6 +1,7 @@
 import django
 from django.conf.urls.defaults import *
 from django.conf import settings
+from django.views.generic.simple import direct_to_template
 
 # Uncomment the next two lines to enable the admin:
 #from django.contrib import admin
@@ -9,6 +10,8 @@
 urlpatterns = patterns('',
     (r'^localeurl/', include('localeurl.urls')),
     (r'^i18n/', include('django.conf.urls.i18n')),
+    (r'^robots\.txt$', direct_to_template, {'template': 'robots.txt',
+                                            'mimetype': 'text/plain'}),
 )
 
 if settings.DEBUG: