Fix bug 1303794: Add blog feeds for about/technology page

* Add command for updating blog feeds
* Add settings for first set of blogs
* Add model for caching blog articles

bedrock/mozorg/cron.py

@@ -3,24 +3,13 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from django.conf import settings
-from django.core.cache import cache
 
 import cronjobs
-import feedparser
 
 from bedrock.mozorg.models import TwitterCache
 from bedrock.mozorg.util import get_tweets
 
 
-@cronjobs.register
-def update_feeds():
-    for name, url in settings.FEEDS.items():
-        feed_info = feedparser.parse(url)
-        # Cache for a year (it will be set by the cron job no matter
-        # what on a set interval)
-        cache.set('feeds-%s' % name, feed_info, 60 * 60 * 24 * 365)
-
-
 @cronjobs.register
 def update_tweets():
     for account in settings.TWITTER_ACCOUNTS:

bedrock/mozorg/management/commands/update_blog_feeds.py

@@ -0,0 +1,48 @@
+from __future__ import print_function
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import DatabaseError
+from django.db import transaction
+
+from feedparser import parse
+
+from bedrock.mozorg.models import BlogArticle
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
+                            help='If no error occurs, swallow all output.'),
+        parser.add_argument('--database', default='default',
+                            help=('Specifies the database to use, if using a db. '
+                                  'Defaults to "default".')),
+        parser.add_argument('--articles', default=5, type=int,
+                            help='Number of articles to store from each feed. Defaults to 5.')
+
+    def handle(self, *args, **options):
+        for feed_id, feed_options in settings.BLOG_FEEDS.items():
+            feed_url = feed_options.get('feed_url', None)
+            if feed_url is None:
+                feed_url = '%s/feed/atom/' % feed_options['url'].rstrip('/')
+            feed = parse(feed_url)
+            if feed.entries:
+                with transaction.atomic(using=options['database']):
+                    count = 0
+                    BlogArticle.objects.filter(blog_slug=feed_id).delete()
+                    for article in feed.entries:
+                        try:
+                            BlogArticle.objects.create(
+                                blog_slug=feed_id,
+                                blog_name=feed_options['name'],
+                                published=article.published,
+                                updated=article.updated,
+                                title=article.title,
+                                summary=article.summary,
+                                link=article.link,
+                            )
+                        except DatabaseError:
+                            continue
+                        count += 1
+                        if count >= options['articles']:
+                            break

bedrock/mozorg/migrations/0002_blogarticle.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mozorg', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BlogArticle',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('blog_slug', models.CharField(max_length=30)),
+                ('blog_name', models.CharField(max_length=50)),
+                ('published', models.DateTimeField()),
+                ('updated', models.DateTimeField()),
+                ('title', models.CharField(max_length=255)),
+                ('summary', models.TextField()),
+                ('link', models.URLField()),
+            ],
+            options={
+                'ordering': ['-published'],
+                'get_latest_by': 'published',
+            },
+        ),
+    ]

bedrock/mozorg/models.py

@@ -1,8 +1,12 @@
+from django.conf import settings
 from django.core.cache import cache
 from django.db import models
 from django.db.models.signals import post_save
 from django.db.utils import DatabaseError
 from django.dispatch import receiver
+from django.utils.html import strip_tags
+
+from jinja2 import Markup
 
 from picklefield import PickledObjectField
 from django_extensions.db.fields import ModificationDateTimeField
@@ -89,3 +93,34 @@ class Meta:
         unique_together = ('date', 'source_name', 'team_name')
         get_latest_by = 'date'
         ordering = ['-date']
+
+
+class BlogArticle(models.Model):
+    blog_slug = models.CharField(max_length=30)
+    blog_name = models.CharField(max_length=50)
+    published = models.DateTimeField()
+    updated = models.DateTimeField()
+    title = models.CharField(max_length=255)
+    summary = models.TextField()
+    link = models.URLField()
+
+    class Meta:
+        get_latest_by = 'published'
+        ordering = ['-published']
+
+    def __unicode__(self):
+        return '%s: %s' % (self.blog_name, self.title)
+
+    def get_absolute_url(self):
+        return self.link
+
+    def htmlify(self):
+        summary = strip_tags(self.summary).strip()
+        if summary.lower().endswith('continue reading'):
+            summary = summary[:-16]
+
+        return Markup(summary)
+
+    @property
+    def blog_link(self):
+        return settings.BLOG_FEEDS[self.blog_slug]['url']

bedrock/mozorg/templates/mozorg/technology/index.html

@@ -224,32 +224,28 @@ <h3>{{ _('Gaming') }}</h3>
   </section>
   {% endblock %}
 
+  {% if articles %}
   <section id="blogs">
     <div class="container">
       <header>
         <h2>{{ _('Blogs') }}</h2>
         <p>{{ _('Read the latest from Mozilla’s technology blogs.') }}</p>
       </header>
       <ul>
-        <li>
-          <article>
-            <h3><a rel="external" href="#">The Project SensorWeb Poster Experiment</a></h3>
-            <p class="meta">Connected Devices <time datetime="2016-08-26">August 26, 2016</time></p>
-            <p>The Project SensorWeb team is on a mission help people leverage the Web to bring open transparency to the physical world around us. Open data, Open platform, Open web...</p>
-            <a rel="external" class="continue" href="#">{{ _('Continue reading') }}</a>
-          </article>
-        </li>
-        <li>
-          <article>
-            <h3><a rel="external" href="#">View Source Conference Berlin 2016</a></h3>
-            <p class="meta">Hacks <time datetime="2016-08-26">August 26, 2016</time></p>
-            <p>View Source is an intimate, single-track conference for web developers, now in its second year. View Source 2016 takes place in Berlin, Germany, September 12-14, beginning with Ignite lightning...</p>
-            <a rel="external" class="continue" href="#">{{ _('Continue reading') }}</a>
-          </article>
-        </li>
+        {% for article in articles %}
+          <li>
+            <article>
+              <h3><a rel="external" href="{{ article.link }}">{{ article.title }}</a></h3>
+              <p class="meta"><a href="{{ article.blog_link }}">{{ article.blog_name }}</a> <time datetime="{{ article.published }}">{{ article.published|l10n_format_date }}</time></p>
+              <p>{{ article.htmlify() }}</p>
+              <a rel="external" class="continue" href="{{ article.link }}">{{ _('Continue reading') }}</a>
+            </article>
+          </li>
+        {% endfor %}
       </ul>
     </div>
   </section>
+  {% endif %}
 
 </main>
 {% endblock %}

bedrock/mozorg/tests/test_commands.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from django.core.management import call_command
+from django.db import IntegrityError
+from django.test import override_settings
+
+from mock import patch
+from pathlib import Path
+
+from bedrock.mozorg.tests import TestCase
+from bedrock.mozorg.models import BlogArticle
+
+
+HACKS_FILE = Path(__file__).parent.joinpath('test_files', 'data', 'hacks-blog.xml')
+TEST_BLOG_FEEDS = {
+    'hacks': {
+        'name': 'Hacks',
+        'url': 'https://hacks.mozilla.org',
+        'feed_url': str(HACKS_FILE),
+    }
+}
+
+
+@override_settings(BLOG_FEEDS=TEST_BLOG_FEEDS)
+class TestUpdateBlogFeeds(TestCase):
+    def test_load_feed(self):
+        call_command('update_blog_feeds', articles=4)
+        self.assertEqual(BlogArticle.objects.count(), 4)
+
+    @patch('bedrock.mozorg.management.commands.update_blog_feeds.BlogArticle')
+    def test_error_loading_feed(self, mock_model):
+        mock_model.objects.create.side_effect = [IntegrityError] + [None] * 4
+        call_command('update_blog_feeds', articles=4)
+        # 5 calls since first fails and we want 4 articles
+        self.assertEqual(mock_model.objects.create.call_count, 5)