Serialize data with json instead of shelve

bigbuild/management/commands/cachepages.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
-import shelve
+import json
 from bigbuild.models import PageList
 from bigbuild import get_retired_directory
 from django.core.management.base import BaseCommand
@@ -22,6 +22,8 @@ def handle(self, *args, **options):
         page_list = PageList()
 
         # Save the retired pages out to a new cache
-        d = shelve.open(retired_cache_path)
-        d['retired_pages'] = page_list.retired_pages
-        d.close()
+        with open(retired_cache_path, 'wb') as f:
+            json.dump(
+                dict(retired_pages=[p.to_json() for p in page_list.retired_pages]),
+                f
+            )

bigbuild/models/base.py

@@ -28,6 +28,7 @@ def __init__(
         headline="",
         byline="",
         description="",
+        image_url="",
         pub_date=None,
         published=False,
         show_in_feeds=True,
@@ -40,7 +41,7 @@ def __init__(
         self.headline = headline
         self.byline = byline
         self.description = description
-        self.image_url = ""
+        self.image_url = image_url
         self.pub_date = pub_date or datetime.now().replace(
             second=0,
             microsecond=0

bigbuild/models/pagelists.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 import os
 import six
-import shelve
+import json
 import logging
 from django.conf import settings
 from collections import Sequence
@@ -11,6 +11,7 @@
     MissingRecommendedMetadataWarning
 )
 from bigbuild.models import Page, RetiredPage
+from dateutil.parser import parse as dateparse
 from bigbuild import get_page_directory, get_retired_directory
 logger = logging.getLogger(__name__)
 
@@ -129,7 +130,13 @@ def get_retired_pages(self):
         # Pull the cached data if it exists
         if os.path.exists(self.retired_cache_path):
             logger.debug("Loading cached retired page list")
-            page_list = shelve.open(self.retired_cache_path)['retired_pages']
+            with open(self.retired_cache_path, 'rb') as f:
+                page_list = []
+                dict_list = json.load(f)['retired_pages']
+                for d in dict_list:
+                    d['pub_date'] = dateparse(d['pub_date'])
+                    p = RetiredPage(**d)
+                    page_list.append(p)
         # Otherwise get them from the YAML
         else:
             logger.debug("Retrieving YAML retired page list")

requirements.txt

@@ -5,6 +5,7 @@ PyYAML==3.11
 validictory==1.0.1
 django-compressor==2.0
 greeking==2.1.3
+python-dateutil>=2.6.0
 GitPython
 pytz
 flake8

setup.py

@@ -116,6 +116,7 @@ def run(self):
         'django-compressor>=2.0',
         'greeking>=2.1.3',
         'GitPython>=2.0.8',
+        'python-dateutil>=2.6.0',
         'pytz',
         'six',
     ],