Tool for downloading and uploading content from different instances

pages/api.py

@@ -1,7 +1,7 @@
-from pages.serializers import PageSerializer
+from pages.serializers import PageSerializer, ContentSerializer
 from rest_framework import generics
 from rest_framework.permissions import IsAdminUser
-from pages.models import Page
+from pages.models import Page, Content
 
 
 class PageList(generics.ListCreateAPIView):
@@ -16,3 +16,18 @@ class PageEdit(generics.RetrieveUpdateDestroyAPIView):
     serializer_class = PageSerializer
     permission_classes = (IsAdminUser,)
     paginate_by = 100
+
+
+class ContentList(generics.ListCreateAPIView):
+    queryset = Content.objects.all()
+    serializer_class = ContentSerializer
+    permission_classes = (IsAdminUser,)
+    paginate_by = 200
+
+
+class ContentEdit(generics.RetrieveUpdateDestroyAPIView):
+    queryset = Content.objects.all()
+    serializer_class = ContentSerializer
+    permission_classes = (IsAdminUser,)
+    paginate_by = 200
+

pages/management/commands/pages_pull.py

@@ -3,16 +3,25 @@
 import os
 
 class Command(BaseCommand):
-    help = 'Pull data from an API'
+    help = 'Pull data from a Django Page CMS API'
 
     def add_arguments(self, parser):
-        parser.add_argument('host', type=str)
-        parser.add_argument('auth', type=str)
-        parser.add_argument('--filename', type=str,  default="data/download.json")
+        parser.add_argument('auth', type=str,
+            help='authentication in the form user:password')
+        parser.add_argument('--host', type=str,
+            help='server to pull from', 
+            default='http://127.0.0.1:8000/api/')
+        parser.add_argument('--filename', type=str,
+            default="data/download.json")
 
     def handle(self, *args, **options):
         auth = options['auth'].split(':')
-        host = options['host'] + 'api/?format=json'
+        host = options['host']
+        if not host.endswith('/'):
+            host = host + '/'
+
+        print("Fetching page data on " + host)
+        host = host + '?format=json'
         filename = options['filename']
 
         page_list = requests.get(host, auth=(auth[0], auth[1]))

pages/management/commands/pages_push.py

@@ -2,35 +2,110 @@
 import requests
 import os
 import json
+import sys
+
+def http_error(response):
+    with open('error.html', "w") as f:
+        f.write(response.text)
+    raise ValueError("Error type " + str(response.status_code) + " file written: error.html")
 
 class Command(BaseCommand):
-    help = 'Pull data from an API'
+    help = 'Push data to a Django Page CMS API'
 
     def add_arguments(self, parser):
-        parser.add_argument('host', type=str)
-        parser.add_argument('auth', type=str)
-        parser.add_argument('--filename', type=str,  default="data/download.json")
+        parser.add_argument('auth', type=str,
+            help='authentication in the form user:password')
+        parser.add_argument('--host', type=str,
+            help='server to pull from', 
+            default='http://127.0.0.1:8000/api/')
+        parser.add_argument('--filename', type=str,
+            default="data/download.json")
+
+    def push_content(self, page):
+        page_id = str(page['id'])
+        auth = self.auth
+        headers = {'Content-Type': 'application/json'}
+        for content in page['content_set']:
+            content['page'] = page_id
+            data = json.dumps(content)
+            url = self.host + 'contents/' + str(content['id']) + '/'
+            response = requests.put(url, data=data, auth=self.auth, headers=headers)
+            if response.status_code == 404:
+                url = self.host + 'contents/'
+                response = requests.post(url, data=data, auth=self.auth, headers=headers)
+            if response.status_code != 200 and response.status_code != 201:
+                http_error(response)
+            sys.stdout.write('.')
+
+    def push_page(self, page):
+        page_id = str(page['id'])
+        auth = self.auth
+
+        headers = {'Content-Type': 'application/json'}
+
+        server_page = self.datetime_mapping.get(page['creation_date'], None)
+
+        # we don't change the parent if for a reason or another it is
+        # not present on the server
+        if self.server_id_mapping.get(page['parent']):
+            page['parent'] = self.server_id_mapping[page['parent']]
+        else:
+            del page['parent']
+
+        if server_page:
+            self.server_id_mapping[page['id']] = server_page['id']
+            page['id'] = server_page['id']
+            sys.stdout.write("Update page " + str(page['id']))
+            url = self.host + 'pages/' + str(page['id']) + '/'
+            data = json.dumps(page)
+            response = requests.put(url, data=data, auth=self.auth, headers=headers)
+        else:
+            sys.stdout.write("Create page " + str(page['id']))
+            url = self.host
+            data = json.dumps(page)
+            response = requests.post(url, data=data, auth=self.auth, headers=headers)
+            if response.status_code == 201:
+                new_page = json.loads(response.text)
+                new_page['content_set'] = page['content_set']
+                self.server_id_mapping[page['id']] = new_page['id']
+                self.datetime_mapping[new_page['creation_date']] = new_page
+                self.id_mapping[new_page['id']] = new_page
+                page = new_page
+
+        if response.status_code != 200 and response.status_code != 201:
+            http_error(response)
+
+        sys.stdout.write(' .')
+        self.push_content(page)
+
+        print('')
 
     def handle(self, *args, **options):
         auth = options['auth'].split(':')
+        self.auth = (auth[0], auth[1])
         host = options['host']
+        if not host.endswith('/'):
+            host = host + '/'
+        self.host = host
         filename = options['filename']
+        self.datetime_mapping = {}
+        self.id_mapping = {}
+        self.server_id_mapping = {}
+
+        print("Fetching the state of the pages on the server " + self.host)
+        host = self.host + '?format=json'
+        response = requests.get(host, auth=self.auth)
+        if response.status_code != 200:
+            http_error(response)
+        self.current_page_list = json.loads(response.text)['results']
+
+        for page in self.current_page_list:
+            self.datetime_mapping[page['creation_date']] = page
+            self.id_mapping[page['id']] = page
 
         with open(filename, "r") as f:
             data = f.read()
             pages = json.loads(data)
             for page in pages['results']:
-                page_id = str(page['id'])
-                print("Pushing page " + page_id)
-                data = json.dumps(page)
-                url = host + 'api/' + page_id + '/'
-                headers = {'Content-Type': 'application/json'}
-                # try an update first
-                response = requests.put(url, data=data, auth=(auth[0], auth[1]), headers=headers)
-                if response.status_code == 404:# or response.status_code == 400:
-                    print("Page " + page_id + " doesn't exist on the target. Skiped.")
-                    continue
-                if response.status_code != 200 and response.status_code != 201:
-                    raise ValueError(response.status_code, response.text)
-
+                self.push_page(page)
 

pages/models.py

@@ -495,7 +495,7 @@ class Content(models.Model):
 
     # languages could have five characters : Brazilian Portuguese is pt-br
     language = models.CharField(_('language'), max_length=5, blank=False)
-    body = models.TextField(_('body'))
+    body = models.TextField(_('body'), blank=True)
     type = models.CharField(_('type'), max_length=100, blank=False,
         db_index=True)
     page = models.ForeignKey(Page, verbose_name=_('page'))

pages/serializers.py

@@ -1,12 +1,35 @@
 from pages.models import Page, Content
 from rest_framework import serializers
+from django.contrib.auth.models import User
 
 class ContentSerializer(serializers.ModelSerializer):
     class Meta:
         model = Content
 
 class PageSerializer(serializers.ModelSerializer):
     content_set = ContentSerializer(many=True, read_only=True)
+    creation_date = serializers.DateTimeField()
 
     class Meta:
-        model = Page
+        model = Page
+
+    def create(self, validated_data):
+
+        attributes = ('status', 'delegate_to', 'freeze_date', 'creation_date',
+            'publication_end_date', 'template', 'redirect_to_url',
+            'last_modification_date', 'publication_date')
+
+        admin = User.objects.filter(is_superuser=True)[0]
+
+        cleaned_data = {}
+        for attribute in attributes:
+            cleaned_data[attribute] = validated_data.get(attribute)
+
+        if validated_data.get('parent', False):
+            cleaned_data['parent'] = validated_data.get('parent')
+
+        cleaned_data['author'] = admin
+
+        page = Page.objects.create(**cleaned_data)
+        page.invalidate()
+        return page

pages/urls.py

@@ -14,7 +14,10 @@
 else:
     urlpatterns += [
         url(r'^api/$', api.PageList.as_view()),
-        url(r'^api/(?P<pk>[0-9]+)/$', api.PageEdit.as_view())
+        url(r'^api/pages/$', api.PageList.as_view()),
+        url(r'^api/pages/(?P<pk>[0-9]+)/$', api.PageEdit.as_view()),
+        url(r'^api/contents/$', api.ContentList.as_view()),
+        url(r'^api/contents/(?P<pk>[0-9]+)/$', api.ContentEdit.as_view())
     ]
 
 if settings.PAGE_USE_LANGUAGE_PREFIX: