Speed up API: bulk what you can

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 4b78a9ff2001133ab819ca3aa32375c8
+config: ab73e5c7fe477f1043dabf7652cb5660
 tags: 33eac41acc08762151beb8f3b7b86c8f

docs/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '0.30.1',
+    VERSION: '0.31.0',
     LANGUAGE: 'None',
     COLLAPSE_INDEX: false,
     FILE_SUFFIX: '.html',

docs/index.html

@@ -8,7 +8,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>TSC 0.30.1 documentation</title>
+  <title>TSC 0.31.0 documentation</title>
 
 
 
@@ -428,7 +428,7 @@
 
       <li><a href="index.html#document-index">Docs</a> &raquo;</li>
 
-      <li>TSC 0.30.1 documentation</li>
+      <li>TSC 0.31.0 documentation</li>
 
 
       <li class="wy-breadcrumbs-aside">

shared/api.py

@@ -157,8 +157,22 @@ def create_one(self, data):
                      '{0} with unique fields {1}'.format(
                          self.model._meta.verbose_name, str(unique_data)))
 
-        # Without the QA status deciding whether to update existing data:
-        obj, created = self.model.objects.update_or_create(defaults=update_data, **unique_data)
+
+        obj, created = self.model.objects.get_or_create(defaults=update_data, **unique_data)
+        verb = "Created" if created else "Updated"
+
+        # Early exit 2: retain locally changed data (status not NEW)
+        if (not created and obj.status != QualityControlMixin.STATUS_NEW):
+            msg = ('[API][create_one] Not overwriting locally changed data '
+                   'with QA status: {0}'.format(obj.get_status_display()))
+            logger.info(msg)
+            content = {"msg": msg}
+            return RestResponse(content, status=status.HTTP_200_OK)
+
+        else:
+            # Continue on happy trail: update if new or existing but unchanged
+            self.model.objects.filter(**unique_data).update(**update_data)
+
         obj.refresh_from_db()
         obj.save()  # to update cached fields
 
@@ -210,8 +224,8 @@ def create(self, request):
                 # Bucket "retain": existing_objects with status > STATUS_NEW
                 to_retain = [t for t in existing_records if t["status"] > QualityControlMixin.STATUS_NEW]
                 # TODO Log PKs (debug) or total number (info) of skipped records
-                logger.info("[API][create] Skipping {0} unchanged records".format(len(to_retain)))
-                logger.debug("[API][create] Skipping unchanged records: {0}".format(str(to_retain)))
+                logger.info("[API][create] Skipping {0} locally changed records".format(len(to_retain)))
+                logger.debug("[API][create] Skipping locally changed records: {0}".format(str(to_retain)))
 
                 # With QA: update if existing but unchanged (QA status "NEW")                
                 to_update = [t for t in existing_records if t["status"] == QualityControlMixin.STATUS_NEW]
@@ -225,17 +239,39 @@ def create(self, request):
             # Bucket "bulk_create": List of new records without match in existing records
             records_to_create = [d for d in new_records if (d['source'], d['source_id']) not in to_update]
 
+            updated = []
+            created = []
+
             # Hammertime
             with transaction.atomic():
                 if records_to_update:
                     # updated = self.model.objects.bulk_update(
                     #     [self.model(**x) for x in records_to_update], records_to_update[0].keys())
-                    updated = [self.create_one(x) for x in records_to_update]
+                    # updated = [self.create_one(x) for x in records_to_update]
+                    for data in records_to_update:
+                        unique_data, update_data = self.split_data(data)
+                        self.model.objects.filter(**unique_data).update(**update_data)
+
+                        # to update cached fields
+                        obj.refresh_from_db()
+                        obj.save()  
+
                 if records_to_create:
                     # created = self.model.objects.bulk_create([self.model(**x) for x in records_to_create])
-                    created = [self.create_one(x) for x in records_to_create]
+                    # created = [self.create_one(x) for x in records_to_create]
+                    for data in records_to_create:
+                        unique_data, update_data = self.split_data(data)
+                        obj, created = self.model.objects.get_or_create(defaults=update_data, **unique_data)
+
+                        # to update cached fields
+                        obj.refresh_from_db()
+                        obj.save()  
+
+            msg = "Retained {0}, updated {1}, created {2} records.".format(
+                len(to_retain), len(records_to_update), len(records_to_create)
+            )
 
-            return RestResponse([to_retain, updated, created], status=status.HTTP_200_OK)
+            return RestResponse(msg, status=status.HTTP_200_OK)
 
         # Create none --------------------------------------------------------#
         else:

wastd/api.py

@@ -1772,12 +1772,12 @@ class OccurrenceTaxonAreaEncounterPolyViewSet(BatchUpsertViewSet):
     model = occ_models.TaxonAreaEncounter
     uid_fields = ("source", "source_id")
 
-    # def split_data(self, data):
-    #     """Custom split data: resolve taxon."""
-    #     unique_fields, update_data = super(OccurrenceTaxonAreaEncounterPolyViewSet, self).split_data(data)
-    #     # update_data["taxon"] = Taxon.objects.get(name_id=data["taxon"])
-    #     # update_data["encountered_by"] = User.objects.get(pk=data["encountered_by"])
-    #     return (unique_fields, update_data)
+    def split_data(self, data):
+        """Custom split data: resolve taxon."""
+        unique_fields, update_data = super(OccurrenceTaxonAreaEncounterPolyViewSet, self).split_data(data)
+        update_data["taxon"] = Taxon.objects.get(name_id=data["taxon"])
+        update_data["encountered_by"] = User.objects.get(pk=data["encountered_by"])
+        return (unique_fields, update_data)
 
 
 class OccurrenceTaxonAreaEncounterPointViewSet(OccurrenceTaxonAreaEncounterPolyViewSet):