Improve wordpress blog support. Add all blogs to new system.

Push this to prod first so that the DB can be full of blog
posts and ready when the updated views land.

* Support multiple blogs in custom filter and view
* Strip all tags from titles and excerpts
* Support custom template context variable name
* Support limiting blog posts returned by view
* Add a featured tag helper
* Switch from strip_tags to bleach for blog posts
* Add featured tag helper

bedrock/settings/base.py

@@ -555,7 +555,25 @@ def set_whitenoise_headers(headers, path, url):
     'firefox': {
         'url': 'https://blog.mozilla.org/firefox/',
         'name': 'The Firefox Frontier',
-        'num_posts': 20,
+        # default num_posts is 20
+        # uncomment and change this to get more
+        # 'num_posts': 20,
+    },
+    'hacks': {
+        'url': 'https://hacks.mozilla.org/',
+        'name': 'Hacks',
+    },
+    'cd': {
+        'url': 'https://connected.mozilla.org/',
+        'name': 'Connected Devices',
+    },
+    'futurereleases': {
+        'url': 'https://blog.mozilla.org/futurereleases/',
+        'name': 'Future Releases',
+    },
+    'internetcitizen': {
+        'url': 'https://blog.mozilla.org/internetcitizen/',
+        'name': 'Internet Citizen',
     },
 }
 

bedrock/wordpress/models.py

@@ -1,11 +1,14 @@
+# -*- coding: utf-8 -*-
+
 from __future__ import print_function, unicode_literals
 
 import operator
+import random
+
 from django.conf import settings
 from django.db import models, transaction
 from django.db.models import Q
 from django.db.utils import DatabaseError
-from django.utils.html import strip_tags
 
 import bleach
 from django_extensions.db.fields.json import JSONField
@@ -15,9 +18,30 @@
 from bedrock.wordpress.api import get_posts_data
 
 
+def strip_tags(text):
+    return bleach.clean(text, tags=[], strip=True).strip()
+
+
+def process_excerpt(excerpt):
+    summary = strip_tags(excerpt)
+    if summary.lower().endswith('continue reading'):
+        summary = summary[:-16]
+
+    if summary.lower().endswith('read more'):
+        summary = summary[:-9]
+
+    if summary.lower().endswith('[…]'):
+        summary = summary[:-10] + '…'
+
+    if summary.endswith('[…]'):
+        summary = summary[:-3] + '…'
+
+    return summary
+
+
 class BlogPostQuerySet(models.QuerySet):
-    def filter_by_blog(self, blog_slug):
-        return self.filter(wp_blog_slug=blog_slug)
+    def filter_by_blogs(self, *blog_slugs):
+        return self.filter(wp_blog_slug__in=blog_slugs)
 
     def filter_by_tags(self, *tags):
         tag_qs = [Q(tags__contains='"{}"'.format(t)) for t in tags]
@@ -28,8 +52,8 @@ class BlogPostManager(models.Manager):
     def get_queryset(self):
         return BlogPostQuerySet(self.model, using=self._db)
 
-    def filter_by_blog(self, blog_slug):
-        return self.get_queryset().filter_by_blog(blog_slug)
+    def filter_by_blogs(self, *blog_slugs):
+        return self.get_queryset().filter_by_blogs(*blog_slugs)
 
     def filter_by_tags(self, *tags):
         return self.get_queryset().filter_by_tags(*tags)
@@ -38,16 +62,16 @@ def update_posts(self, data, database='default'):
         with transaction.atomic(using=database):
             count = 0
             posts = data['posts']
-            self.filter_by_blog(data['wp_blog_slug']).delete()
+            self.filter_by_blogs(data['wp_blog_slug']).delete()
             for post in posts:
                 try:
                     self.create(
                         wp_id=post['id'],
                         wp_blog_slug=data['wp_blog_slug'],
                         date=post['date_gmt'],
                         modified=post['modified_gmt'],
-                        title=bleach.clean(post['title']['rendered']),
-                        excerpt=bleach.clean(post['excerpt']['rendered']),
+                        title=strip_tags(post['title']['rendered']),
+                        excerpt=process_excerpt(post['excerpt']['rendered']),
                         link=post['link'],
                         featured_media=post['featured_media'],
                         tags=post['tags'],
@@ -92,16 +116,11 @@ def get_absolute_url(self):
         return self.link
 
     def htmlify(self):
-        summary = strip_tags(self.excerpt).strip()
-        if summary.lower().endswith('continue reading'):
-            summary = summary[:-16]
-
-        return Markup(summary)
+        return Markup(self.excerpt)
 
     @property
     def blog_title(self):
-        title = strip_tags(self.title).strip()
-        return Markup(title).unescape()
+        return Markup(self.title).unescape()
 
     @property
     def blog_link(self):
@@ -111,6 +130,17 @@ def blog_link(self):
     def blog_name(self):
         return settings.WP_BLOGS[self.wp_blog_slug]['name']
 
+    def get_featured_tag(self, tags):
+        """Return a tag present both in the post and the passed in list.
+
+        If no tag matches something odd has happened, so just return blank.
+        """
+        matching_tags = list(set(self.tags) & set(tags))
+        if matching_tags:
+            return random.choice(matching_tags)
+        else:
+            return ''
+
     def get_featured_image_url(self, size='large'):
         try:
             return self.featured_media['media_details']['sizes'][size]['source_url']

bedrock/wordpress/tests/test_models.py

@@ -1,7 +1,7 @@
 from django.test import override_settings
 
 from bedrock.wordpress import api
-from bedrock.wordpress.models import BlogPost
+from bedrock.wordpress import models
 
 import pytest
 import responses
@@ -15,6 +15,10 @@
         'name': 'The Firefox Frontier',
         'num_posts': 10,
     },
+    'hacks': {
+        'url': 'https://hacks.mozilla.org/',
+        'name': 'Hacks',
+    },
 }
 
 
@@ -23,10 +27,10 @@ def get_test_file_content(filename):
         return fh.read()
 
 
-def setup_responses():
-    posts_url = api._api_url(TEST_WP_BLOGS['firefox']['url'], 'posts', None)
-    tags_url = api._api_url(TEST_WP_BLOGS['firefox']['url'], 'tags', None)
-    media_url = api._api_url(TEST_WP_BLOGS['firefox']['url'], 'media', 75)
+def setup_responses(blog='firefox'):
+    posts_url = api._api_url(TEST_WP_BLOGS[blog]['url'], 'posts', None)
+    tags_url = api._api_url(TEST_WP_BLOGS[blog]['url'], 'tags', None)
+    media_url = api._api_url(TEST_WP_BLOGS[blog]['url'], 'media', 75)
     responses.add(responses.GET, posts_url, body=get_test_file_content('posts.json'))
     responses.add(responses.GET, tags_url, body=get_test_file_content('tags.json'))
     responses.add(responses.GET, media_url, body=get_test_file_content('media_75.json'))
@@ -49,9 +53,11 @@ def test_get_posts_data():
 @override_settings(WP_BLOGS=TEST_WP_BLOGS)
 @pytest.mark.django_db
 def test_refresh_posts():
-    setup_responses()
-    BlogPost.objects.refresh('firefox')
-    blog = BlogPost.objects.filter_by_blog('firefox')
+    setup_responses('firefox')
+    models.BlogPost.objects.refresh('firefox')
+    setup_responses('hacks')
+    models.BlogPost.objects.refresh('hacks')
+    blog = models.BlogPost.objects.filter_by_blogs('firefox')
     assert len(blog) == 3
     bp = blog.get(wp_id=10)
     assert bp.tags == ['browser', 'fastest']
@@ -60,15 +66,31 @@ def test_refresh_posts():
     assert bp.tags == ['fastest', 'privacy', 'programming', 'rust', 'security']
     assert bp.featured_media['id'] == 75
     assert bp.get_featured_image_url('large').endswith('Put-Your-Trust-in-Rust-600x315.png')
+    blog = models.BlogPost.objects.filter_by_blogs('firefox', 'hacks')
+    assert len(blog) == 6
 
 
 @responses.activate
 @override_settings(WP_BLOGS=TEST_WP_BLOGS)
 @pytest.mark.django_db
 def test_filter_by_tags():
     setup_responses()
-    BlogPost.objects.refresh('firefox')
-    blog = BlogPost.objects.filter_by_blog('firefox')
+    models.BlogPost.objects.refresh('firefox')
+    blog = models.BlogPost.objects.filter_by_blogs('firefox')
     assert len(blog.filter_by_tags('browser')) == 1
     assert len(blog.filter_by_tags('fastest')) == 3
     assert len(blog.filter_by_tags('browser', 'jank')) == 2
+
+
+@responses.activate
+@override_settings(WP_BLOGS=TEST_WP_BLOGS)
+@pytest.mark.django_db
+def test_featured_tag():
+    setup_responses()
+    models.BlogPost.objects.refresh('firefox')
+    blog = models.BlogPost.objects.filter_by_blogs('firefox')
+    p = blog.filter_by_tags('browser')[0]
+    assert p.get_featured_tag(['browser', 'dude']) == 'browser'
+    p = blog.filter_by_tags('browser', 'jank')[0]
+    assert p.get_featured_tag(['browser', 'jank', 'dude']) in ['browser', 'jank']
+    assert p.get_featured_tag(['dude']) == ''

bedrock/wordpress/tests/test_views.py

@@ -12,11 +12,14 @@
 @override_settings(WP_BLOGS=TEST_WP_BLOGS)
 @pytest.mark.django_db
 def test_blog_posts_view():
-    setup_responses()
+    setup_responses('firefox')
     BlogPost.objects.refresh('firefox')
+    setup_responses('hacks')
+    BlogPost.objects.refresh('hacks')
 
     req = RequestFactory().get('/')
-    view = BlogPostsView(blog_slug='firefox', blog_tags=['browser', 'jank'],
+    view = BlogPostsView(blog_slugs='firefox',
+                         blog_tags=['browser', 'jank'],
                          template_name='mozorg/book.html')
     view.request = req
     blog_posts = view.get_context_data()['blog_posts']
@@ -25,19 +28,38 @@ def test_blog_posts_view():
     assert blog_posts[1].wp_id == 69
 
     # no tags defined
-    view = BlogPostsView(blog_slug='firefox', template_name='mozorg/book.html')
+    view = BlogPostsView(blog_slugs='firefox', template_name='mozorg/book.html')
     view.request = req
     blog_posts = view.get_context_data()['blog_posts']
     assert len(blog_posts) == 3
 
-    # incorrect blog slug
-    view = BlogPostsView(blog_slug='not-a-blog', template_name='mozorg/book.html')
+    # multiple blogs
+    view = BlogPostsView(blog_slugs=['firefox', 'hacks'],
+                         blog_posts_limit=8,
+                         template_name='mozorg/book.html')
     view.request = req
     blog_posts = view.get_context_data()['blog_posts']
-    assert len(blog_posts) == 0
+    assert len(blog_posts) == 6
+
+    # multiple blogs with tags
+    view = BlogPostsView(blog_slugs=['firefox', 'hacks'],
+                         blog_tags=['browser', 'jank'],
+                         blog_posts_limit=8,
+                         template_name='mozorg/book.html')
+    view.request = req
+    blog_posts = view.get_context_data()['blog_posts']
+    assert len(blog_posts) == 4
 
-    # no blog defined
-    view = BlogPostsView(template_name='mozorg/book.html')
+    # incorrect blog slug
+    view = BlogPostsView(blog_slugs='not-a-blog', template_name='mozorg/book.html')
     view.request = req
     blog_posts = view.get_context_data()['blog_posts']
     assert len(blog_posts) == 0
+
+    # no blog defined, but limited
+    view = BlogPostsView(blog_posts_limit=2,
+                         blog_posts_template_variable='articles',
+                         template_name='mozorg/book.html')
+    view.request = req
+    blog_posts = view.get_context_data()['articles']
+    assert len(blog_posts) == 2

bedrock/wordpress/views.py

@@ -3,21 +3,41 @@
 from bedrock.wordpress.models import BlogPost
 from lib.l10n_utils import LangFilesMixin
 
+from raven.contrib.django.raven_compat.models import client as sentry_client
+
 
 class BlogPostsMixin(object):
     blog_tags = None
-    blog_slug = None
+    blog_slugs = None
+    blog_posts_limit = 4
+    blog_posts_template_variable = 'blog_posts'
 
     def get_context_data(self, **kwargs):
         ctx = super(BlogPostsMixin, self).get_context_data(**kwargs)
-        if self.blog_slug:
-            blog = BlogPost.objects.filter_by_blog(self.blog_slug)
-            if self.blog_tags:
-                blog = blog.filter_by_tags(*self.blog_tags)
-
-            ctx['blog_posts'] = blog
-        else:
-            ctx['blog_posts'] = []
+        blog = BlogPost.objects.all()
+        if self.blog_slugs:
+            if isinstance(self.blog_slugs, basestring):
+                blog_slugs = [self.blog_slugs]
+            else:
+                blog_slugs = self.blog_slugs
+
+            blog = blog.filter_by_blogs(*blog_slugs)
+            ctx['blog_slugs'] = blog_slugs
+
+        if self.blog_tags:
+            blog = blog.filter_by_tags(*self.blog_tags)
+            ctx['blog_tags'] = self.blog_tags
+
+        if self.blog_posts_limit:
+            blog = blog[:self.blog_posts_limit]
+
+        try:
+            # run the query here so that we can catch exceptions
+            # and render the page without blog posts
+            ctx[self.blog_posts_template_variable] = list(blog)
+        except Exception:
+            sentry_client.captureException()
+            ctx[self.blog_posts_template_variable] = []
 
         return ctx