Merge pull request #9 from davidbgk/8-add-dataset

Add a sitemap for datasets, reuses, organizations and topics, fixes #8

requirements/install.pip

@@ -17,6 +17,7 @@ Flask-Navigation==0.2.0
 Flask-OAuthlib==0.9.1
 flask-restplus==0.7.1
 Flask-Script==2.0.5
+Flask-Sitemap==0.1.0
 Flask-Security==1.7.4
 Flask-Themes2==0.1.4
 Flask-Uploads==0.1.3

udata/app.py

@@ -97,7 +97,7 @@ def default(self, obj):
             return obj.isoformat()
         elif hasattr(obj, 'serialize'):
             return obj.serialize()
-        elif hasattr(obj, '_data'): # Serialize Raw data for Document and EmbeddedDocument
+        elif hasattr(obj, '_data'):  # Serialize Raw data for Document and EmbeddedDocument
             return obj._data
         return super(UDataJsonEncoder, self).default(obj)
 
@@ -156,7 +156,7 @@ def init_logging(app):
 
 
 def register_extensions(app):
-    from udata import models, routing, tasks, mail, i18n, auth, theme
+    from udata import models, routing, tasks, mail, i18n, auth, theme, search, sitemap
     i18n.init_app(app)
     models.init_app(app)
     routing.init_app(app)
@@ -167,7 +167,6 @@ def register_extensions(app):
     nav.init_app(app)
     theme.init_app(app)
     mail.init_app(app)
-
-    from udata import search
     search.init_app(app)
+    sitemap.init_app(app)
     return app

udata/sitemap.py

@@ -0,0 +1,42 @@
+from functools import wraps
+from flask.ext.sitemap import Sitemap, sitemap_page_needed
+from flask.ext.cache import Cache
+
+from core.dataset.models import Dataset
+from core.reuse.models import Reuse
+from core.organization.models import Organization
+from core.topic.models import Topic
+
+
+def init_app(app):
+    cache = Cache()
+    sitemap = Sitemap()
+
+    @sitemap_page_needed.connect
+    def create_page(app, page, urlset):
+        cache[page] = sitemap.render_page(urlset=urlset)
+
+    def load_page(fn):
+        @wraps(fn)
+        def loader(*args, **kwargs):
+            page = kwargs.get('page')
+            try:
+                data = cache.get(page)
+            except KeyError:
+                data = None
+            return data if data else fn(*args, **kwargs)
+        return loader
+
+    @sitemap.register_generator
+    def collect_urls():
+        for item in Dataset.objects.visible():
+            yield 'datasets.show_redirect', {'dataset': item.id}, None, "weekly", 1
+        for item in Reuse.objects.visible():
+            yield 'reuses.show_redirect', {'reuse': item}, None, "weekly", 0.8
+        for item in Organization.objects.visible():
+            yield 'organizations.show_redirect', {'org': item}, None, "weekly", 0.8
+        for item in Topic.objects.all():
+            yield 'topics.display_redirect', {'topic': item}, None, "weekly", 0.8
+
+    app.config['SITEMAP_VIEW_DECORATORS'] = [load_page]
+    sitemap.init_app(app)

udata/tests/factories.py

@@ -143,9 +143,6 @@ class Meta:
 
 
 class VisibleDatasetFactory(DatasetFactory):
-    # class Meta:
-    #     model = models.Dataset
-
     @factory.lazy_attribute
     def resources(self):
         return [ResourceFactory()]
@@ -175,7 +172,6 @@ def datasets(self):
         return [DatasetFactory()]
 
 
-
 class LicenseFactory(MongoEngineFactory):
     class Meta:
         model = models.License

udata/tests/test_sitemap.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from udata import frontend
+from udata.tests import TestCase, WebTestMixin, SearchTestMixin
+
+from .factories import TopicFactory, OrganizationFactory, VisibleReuseFactory, VisibleDatasetFactory
+
+
+class SitemapTestCase(WebTestMixin, SearchTestMixin, TestCase):
+
+    def create_app(self):
+        app = super(SitemapTestCase, self).create_app()
+        frontend.init_app(app)
+        return app
+
+
+class SitemapTest(SitemapTestCase):
+
+    def test_topics_within_sitemap(self):
+        '''It should return a topic list from the sitemap.'''
+        topics = TopicFactory.create_batch(3)
+        response = self.get('sitemap.xml')
+        self.assert200(response)
+        self.assertEqual(response.data.count('<loc>'), 3, response.data)
+        self.assertIn('<priority>0.8</priority>', response.data)
+        self.assertIn('<changefreq>weekly</changefreq>', response.data)
+        self.assertIn(
+            '<loc>http://localhost/topics/{topic}/</loc>'.format(topic=topics[0].slug),
+            response.data)
+
+    def test_organizations_within_sitemap(self):
+        '''It should return an organization list from the sitemap.'''
+        organizations = OrganizationFactory.create_batch(3)
+        response = self.get('sitemap.xml')
+        self.assert200(response)
+        self.assertEqual(response.data.count('<loc>'), 3, response.data)
+        self.assertIn('<priority>0.8</priority>', response.data)
+        self.assertIn('<changefreq>weekly</changefreq>', response.data)
+        self.assertIn(
+            '<loc>http://localhost/organizations/{organization}/</loc>'.format(organization=organizations[0].slug),
+            response.data)
+
+    def test_reuses_within_sitemap(self):
+        '''It should return a reuse list from the sitemap.'''
+        reuses = VisibleReuseFactory.create_batch(3)
+        response = self.get('sitemap.xml')
+        self.assert200(response)
+        self.assertEqual(response.data.count('<loc>'), 3, response.data)
+        self.assertIn('<priority>0.8</priority>', response.data)
+        self.assertIn('<changefreq>weekly</changefreq>', response.data)
+        self.assertIn(
+            '<loc>http://localhost/reuses/{reuse}/</loc>'.format(reuse=reuses[0].slug),
+            response.data)
+
+    def test_datasets_within_sitemap(self):
+        '''It should return a dataset list from the sitemap.'''
+        datasets = VisibleDatasetFactory.create_batch(3)
+        response = self.get('sitemap.xml')
+        self.assert200(response)
+        self.assertEqual(response.data.count('<loc>'), 3, response.data)
+        self.assertIn('<priority>1</priority>', response.data)
+        self.assertIn('<changefreq>weekly</changefreq>', response.data)
+        self.assertIn(
+            '<loc>http://localhost/datasets/{dataset}/</loc>'.format(dataset=datasets[0].id),
+            response.data)