Merge pull request #7 from davidbgk/6-robotstxt-url

 Add a robots.txt URL, fixes #6

udata/core/site/__init__.py

@@ -1,3 +1,2 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-

udata/core/site/views.py

@@ -1,12 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from flask import g, request, current_app
+from flask import g, request, current_app, send_from_directory, Blueprint
 from werkzeug.contrib.atom import AtomFeed
 from werkzeug.local import LocalProxy
 
 from udata import search, theme
-from udata.app import cache
 from udata.frontend import csv
 from udata.frontend.views import DetailView
 from udata.i18n import I18nBlueprint, lazy_gettext as _
@@ -19,6 +18,7 @@
 from udata.core.organization.csv import OrganizationCsvAdapter
 from udata.core.reuse.csv import ReuseCsvAdapter
 
+noI18n = Blueprint('noI18n', __name__)
 blueprint = I18nBlueprint('site', __name__)
 
 
@@ -84,6 +84,11 @@ def home():
     return theme.render('home.html', **processor(context))
 
 
+@noI18n.route('/robots.txt')
+def static_from_root():
+    return send_from_directory(current_app.static_folder, request.path[1:])
+
+
 @blueprint.route('/map/')
 def map():
     return theme.render('site/map.html')

udata/frontend/__init__.py

@@ -64,7 +64,7 @@ def init_app(app):
 
     from udata.core.storages.views import blueprint as storages_blueprint
     from udata.core.user.views import blueprint as user_blueprint
-    from udata.core.site.views import blueprint as site_blueprint
+    from udata.core.site.views import noI18n, blueprint as site_blueprint
     from udata.core.dataset.views import blueprint as dataset_blueprint
     from udata.core.reuse.views import blueprint as reuse_blueprint
     from udata.core.organization.views import blueprint as org_blueprint
@@ -75,6 +75,7 @@ def init_app(app):
 
     app.register_blueprint(storages_blueprint)
     app.register_blueprint(user_blueprint)
+    app.register_blueprint(noI18n)
     app.register_blueprint(site_blueprint)
     app.register_blueprint(dataset_blueprint)
     app.register_blueprint(reuse_blueprint)

udata/static/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /fr/users/
+Disallow: /en/users/
+Disallow: /es/users/

udata/tests/site/test_site_views.py

@@ -5,7 +5,7 @@
 
 from datetime import datetime
 
-from flask import url_for, g
+from flask import url_for
 
 from udata.frontend import csv
 from udata.models import Site
@@ -24,6 +24,17 @@ def test_site_global(self):
             self.assertIsInstance(current_site._get_current_object(), Site)
             self.assertEqual(current_site.id, self.app.config['SITE_ID'])
 
+    def test_render_robotstxt(self):
+        '''It should render the robots.txt with all pages allowed.'''
+        response = self.get('/robots.txt')
+        self.assertEqual(response.data.split('\n'), [
+            u'User-agent: *',
+            u'Disallow: /fr/users/',
+            u'Disallow: /en/users/',
+            u'Disallow: /es/users/',
+            u''
+        ])
+
     def test_render_home(self):
         '''It should render the home page'''
         with self.autoindex():