Finished new api work

README.md

@@ -58,11 +58,12 @@ Use the Django management command `import_wordpress` and provide the slug of the
 
 `./manage.py import_wordpress blog --url=https://public-api.wordpress.com/wp/v2/sites/davidmburke.com`
 
-Notice the special wordpress.com url schema. For a private wordpress instance it would typically look like `https://example.com/wp-json` instead.
+Notice the special wordpress.com url schema. For a private wordpress instance it would typically look like `https://example.com/wp-json/wp/v2` instead.
 
 **Optional Arguments**
 
 - --convert-images (False) set to True to attempt converting images to Wagtail Image objects. Requires `beautifulsoup4`.
+- --create-users (False) set to True to create new users out of authors.
 
 **Extending**
 

blog/abstract.py

@@ -169,7 +169,7 @@ class Meta:
         FieldPanel('title', classname="full title"),
         MultiFieldPanel([
             FieldPanel('tags'),
-            InlinePanel('categories', label=_("Categories")),
+            FieldPanel('blog_categories'),
         ], heading="Tags and Categories"),
         ImageChooserPanel('header_image'),
         FieldPanel('body', classname="full"),

blog/management/commands/import_wordpress.py

@@ -6,10 +6,15 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('blog_index', help="Title of blog index page to attach blogs")
         parser.add_argument('--url', help="Base url of wordpress instance")
-        parser.add_argument('--convert-images', default=False, help="Find and convert images to Wagtail Images")
+        parser.add_argument('--convert-images', default=False, type=bool, help="Find and convert images to Wagtail Images")
+        parser.add_argument('--create-users', default=False, type=bool, help="Create users out of found Authors")
 
     def handle(self, *args, **options):
         url = options.get('url')
-        convert_images = options.get('convert-images')
-        wordpress_import = WordpressImport(url, convert_images=convert_images)
+        url = url.rstrip('/')
+        wordpress_import = WordpressImport(
+            url,
+            convert_images=options.get('convert_images'),
+            create_users=options.get('create_users')
+        )
         wordpress_import.get_posts()

blog/tests.py

@@ -5,8 +5,8 @@
 from django.contrib.auth.models import User
 from django.test import TestCase
 from django_comments_xtd.models import XtdComment
-
 from wagtail.core.models import Page
+import responses
 
 from .models import (BlogPage, BlogTag, BlogPageTag, BlogIndexPage,
                      BlogCategory, BlogCategoryBlogPage)
@@ -239,15 +239,27 @@ def test_unique_category_slug(self):
 
 
 class BlogAPIImportTests(TestCase):
+    @responses.activate
     def test_import(self):
+        url = "https://public-api.wordpress.com/wp/v2/sites/davidmburke.com"
+        with open("test_v2_resp.json") as json_file:
+            data = json.load(json_file)
+            responses.add(
+                responses.GET,
+                url + '/posts?per_page=50&_embed=1',
+                json=data,
+                status=404,
+                headers={'X-WP-TotalPages': '1'}
+            )
+
         home = Page.objects.get(slug='home')
         self.user = User.objects.create_user('test', 'test@test.test', 'pass')
         blog_index = home.add_child(instance=BlogIndexPage(
             title='Blog Index', slug='blog', search_description="x",
             owner=self.user))
 
-        url = "https://public-api.wordpress.com/wp/v2/sites/davidmburke.com"
         importer = WordpressImport(url)
-        importer.first_page_only = True
         importer.convert_images = True
         importer.get_posts()
+        posts = BlogPage.objects.all()
+        self.assertEqual(len(posts), 1)

blog/wordpress_import.py

@@ -3,14 +3,15 @@
 import html
 import os
 from io import BytesIO
+from typing import List
 
 import requests
 from django.core.files import File
 from django.contrib.auth import get_user_model
 from wagtail.core.models import Page
 from wagtail.images.models import Image
 
-from .models import BlogPage
+from .models import BlogPage, BlogCategory, BlogTag
 
 User = get_user_model()
 
@@ -21,10 +22,10 @@ class WordpressImport():
     blog_index = None
     per_page = 50  # Number of items per page for wordpress rest api
     convert_images = False
-    create_users = True  # Create users from Author data, if they don't exist
+    create_users = False  # Create users from Author data, if they don't exist
     first_page_only = False # Only process one page. Useful in testing and previewing.
 
-    def __init__(self, url: str, blog_index_slug='', convert_images=False):
+    def __init__(self, url: str, blog_index_slug='', convert_images=False, create_users=False):
         """
         Set optional configuration
 
@@ -35,6 +36,7 @@ def __init__(self, url: str, blog_index_slug='', convert_images=False):
         if blog_index_slug:
             self.blog_index_slug = blog_index_slug
         self.convert_images = convert_images
+        self.create_users = create_users
         self.blog_index = Page.objects.filter(slug=self.blog_index_slug).first()
 
     def get_headers(self):
@@ -78,10 +80,14 @@ def process_post(self, post):
         page.search_description = self.convert_html_entities(post['excerpt']['rendered'])
         page.date = post['date'][:10]
         self.set_blog_authors(page, post)
+        if self.convert_images:
+            self.set_featured_media(page, post)
         if page.id:
             page.save()
         else:
             self.blog_index.add_child(instance=page)
+        self.set_categories(page, post)
+        self.set_tags(page, post)
 
     def convert_html_entities(self, text):
         """converts html symbols so they show up correctly in wagtail"""
@@ -104,6 +110,7 @@ def set_blog_authors(self, page: BlogPage, post):
         wag_author = User.objects.filter(username=wp_author['slug']).first()
         if wag_author:
             page.owner = wag_author
+            page.author = wag_author
         elif self.create_users:
             name = wp_author['name']
             last_name = ''
@@ -112,9 +119,66 @@ def set_blog_authors(self, page: BlogPage, post):
             wag_author = User.objects.create(
                 username=wp_author['slug'],
                 first_name=name.split()[0],
-                last_name=last_name
+                last_name=last_name,
+                is_staff=True
             )
             page.owner = wag_author
+            page.author = wag_author
+
+    def set_categories(self, page: BlogPage, post):
+        categories: List[int] = post['categories']
+        embed_terms = post['_embedded'].get('wp:term')
+        for category in categories:
+            for embed_category_list in embed_terms:
+                for embed_category in embed_category_list:
+                    if embed_category['id'] == category and embed_category['taxonomy'] == 'category':
+                        try:
+                            blog_category = BlogCategory.objects.get(slug=embed_category['slug'])
+                        except BlogCategory.DoesNotExist:
+                            blog_category = BlogCategory.objects.create(
+                                slug=embed_category['slug'],
+                                name=embed_category['name']
+                            )
+                        page.blog_categories.add(blog_category)
+
+    def set_tags(self, page: BlogPage, post):
+        tags: List[int] = post['tags']
+        embed_terms = post['_embedded'].get('wp:term')
+        for tag in tags:
+            for embed_tag_list in embed_terms:
+                for embed_tag in embed_tag_list:
+                    if embed_tag['id'] == tag and embed_tag['taxonomy'] == 'post_tag':
+                        try:
+                            blog_tag = BlogTag.objects.get(slug=embed_tag['slug'])
+                        except BlogTag.DoesNotExist:
+                            blog_tag = BlogTag.objects.create(
+                                slug=embed_tag['slug'],
+                                name=embed_tag['name']
+                            )
+                        page.tags.add(blog_tag)
+
+    def set_featured_media(self, page: BlogPage, post):
+        featured_media_id: int = post.get('featured_media')
+        if not featured_media_id:
+            return
+        featured_medias: list = post['_embedded'].get('wp:featuredmedia')
+        if featured_medias is None:
+            return
+        for feature_media in featured_medias:
+            if feature_media['id'] == featured_media_id:
+                source_url = feature_media['source_url']
+                title = feature_media['title']
+                details = feature_media['media_details']
+                resp = requests.get(source_url, stream=True)
+                if resp.status_code != requests.codes.ok:
+                    print("Unable to import " + source_url)
+                    continue
+                fp = BytesIO()
+                fp.write(resp.content)
+                image = Image(title=title, width=details['width'], height=details['height'])
+                image.file.save(details['file'], File(fp))
+                image.save()
+                page.header_image = image
 
     def create_images_from_urls_in_content(self, body):
         """create Image objects and transfer image files to media root"""

requirements.txt

@@ -8,3 +8,4 @@ django-comments-xtd==2.4.1
 requests
 lxml
 ipdb
+responses