robots.txt

settings.py

@@ -185,6 +185,7 @@
     'opst_plugins',
 #    'haystack',
 #    'cms_search',
+#    'fixture_magic',
 )
 
 THUMBNAIL_PROCESSORS = (

templates/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /search/

urls.py

@@ -2,6 +2,7 @@
 from django.contrib import admin
 from django.conf import settings
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
+from django.views.generic.simple import direct_to_template
 
 # from cms.sitemaps import CMSSitemap
 
@@ -11,6 +12,7 @@
     url(r'^admin/', include(admin.site.urls)),
     url(r'^admin/tagging/autocomplete', include('pagetags.urls')),
 	#url(r'^', include('opst_plugins.urls')),
+    (r'^robots\.txt$', direct_to_template, {'template': 'robots.txt', 'mimetype': 'text/plain'}),
     url(r'^', include('cms.urls')),
 )
 urlpatterns += staticfiles_urlpatterns()