[iar#178] Allow customization of robots.txt.

Serve `/robots.txt` from a template instead of from a static file to
allow for customization by extensions and site owners. See
ckan/ideas#178.

ckan/config/routing.py

@@ -428,6 +428,9 @@ def make_map():
         m.connect('/testing/primer', action='primer')
         m.connect('/testing/markup', action='markup')
 
+    # robots.txt
+    map.connect('/(robots.txt)', controller='template', action='view')
+
     # Mark all unmarked routes added up until now as core routes
     for route in map.matchlist:
         if not hasattr(route, '_ckan_core'):

ckan/controllers/template.py

@@ -2,12 +2,13 @@
 
 import ckan.lib.base as base
 import ckan.lib.render
+from ckan.common import response
 
 
 class TemplateController(base.BaseController):
 
     def view(self, url):
-        """By default, the final controller tried to fulfill the request
+        u"""By default, the final controller tried to fulfill the request
         when no other routes match. It may be used to display a template
         when all else fails, e.g.::
 
@@ -28,12 +29,16 @@ def view(self, url):
         By default this controller aborts the request with a 404 (Not
         Found)
         """
+        if url.endswith(u'.txt'):
+            response.headers[u'Content-Type'] = u'text/plain; charset=utf-8'
+        # Default content-type is text/html
         try:
             return base.render(url)
         except ckan.lib.render.TemplateNotFound:
-            if url.endswith('.html'):
+            if url.endswith(u'.html'):
                 base.abort(404)
-            url += '.html'
+            url += u'.html'
+            response.headers[u'Content-Type'] = u'text/html; charset=utf-8'
             try:
                 return base.render(url)
             except ckan.lib.render.TemplateNotFound:

ckan/public/robots.txt → ckan/templates/robots.txt

@@ -1,9 +1,11 @@
 User-agent: *
+{% block all_user_agents -%}
 Disallow: /dataset/rate/
 Disallow: /revision/
 Disallow: /dataset/*/history
 Disallow: /api/
-
-User-Agent: *
 Crawl-Delay: 10
+{%- endblock %}
 
+{% block additional_user_agents -%}
+{%- endblock %}

ckan/tests/controllers/test_template.py

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+from nose.tools import assert_equal
+
+import ckan.tests.helpers as helpers
+
+
+class TestTemplateController(helpers.FunctionalTestBase):
+
+    def test_content_type(self):
+        cases = {
+            u'/robots.txt': u'text/plain; charset=utf-8',
+            u'/page': u'text/html; charset=utf-8',
+            u'/page.html': u'text/html; charset=utf-8',
+        }
+        app = self._get_test_app()
+        for url, expected in cases.iteritems():
+            response = app.get(url, status=200)
+            assert_equal(response.headers.get(u'Content-Type'), expected)

ckan/tests/test_coding_standards.py

@@ -252,7 +252,6 @@ def find_unprefixed_string_literals(filename):
     u'ckan/controllers/revision.py',
     u'ckan/controllers/storage.py',
     u'ckan/controllers/tag.py',
-    u'ckan/controllers/template.py',
     u'ckan/controllers/user.py',
     u'ckan/controllers/util.py',
     u'ckan/exceptions.py',

ckan/tests/test_robots_txt.py

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+from nose.tools import assert_equal, ok_
+
+import ckan.tests.helpers as helpers
+
+
+class TestRobotsTxt(helpers.FunctionalTestBase):
+
+    def test_robots_txt(self):
+        app = self._get_test_app()
+        response = app.get(u'/robots.txt', status=200)
+        assert_equal(response.headers.get(u'Content-Type'), u'text/plain; charset=utf-8')
+        ok_(u'User-agent' in response)