Merge pull request #1353 from vakila/perf-data-bulk

Bug 1217520 - Import perf data using bulk_create if possible
mozilla · Mar 18, 2016 · 39d51ca · 39d51ca
2 parents 2a9dbef + 5fd5523
commit 39d51ca
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 7 deletions.
diff --git a/treeherder/perf/management/commands/import_perf_data.py b/treeherder/perf/management/commands/import_perf_data.py
@@ -5,7 +5,8 @@
 import concurrent.futures
 from django.core.management.base import (BaseCommand,
                                          CommandError)
-from django.db import transaction
+from django.db import (IntegrityError,
+                       transaction)
 
 from treeherder.client import (PerfherderClient,
                                PerformanceTimeInterval)
@@ -56,22 +57,43 @@ def _add_series(pc, project_name, signature_hash, signature_props, verbosity):
             'option_collection': option_collection,
             'platform': platform,
             'framework': framework,
-            'extra_properties': extra_properties
+            'extra_properties': extra_properties,
+            'last_updated': datetime.datetime.fromtimestamp(0)
         })
 
     series = pc.get_performance_data(
         project_name, signatures=signature_hash,
         time_interval=PerformanceTimeInterval.ONE_YEAR)[signature_hash]
 
-    with transaction.atomic():
+    try:
+        new_series = []
+        latest_timestamp = datetime.datetime.fromtimestamp(0)
         for datum in series:
-            PerformanceDatum.objects.get_or_create(
+            timestamp = datetime.datetime.fromtimestamp(datum['push_timestamp'])
+            new_series.append(PerformanceDatum(
                 repository=repository,
                 result_set_id=datum['result_set_id'],
                 job_id=datum['job_id'],
                 signature=signature,
                 value=datum['value'],
-                push_timestamp=datetime.datetime.fromtimestamp(datum['push_timestamp']))
+                push_timestamp=timestamp))
+            if timestamp > latest_timestamp:
+                latest_timestamp = timestamp
+        PerformanceDatum.objects.bulk_create(new_series)
+        signature.last_updated = latest_timestamp
+        signature.save()
+    except IntegrityError:
+        with transaction.atomic():
+            # bulk_create fails if data to import overlaps with existing data
+            # so we fall back to creating objects one at a time
+            for datum in series:
+                PerformanceDatum.objects.get_or_create(
+                    repository=repository,
+                    result_set_id=datum['result_set_id'],
+                    job_id=datum['job_id'],
+                    signature=signature,
+                    value=datum['value'],
+                    push_timestamp=datetime.datetime.fromtimestamp(datum['push_timestamp']))
 
 
 class Command(BaseCommand):

diff --git a/treeherder/perf/models.py b/treeherder/perf/models.py
@@ -87,8 +87,7 @@ class Meta:
 
     def save(self, *args, **kwargs):
         super(PerformanceDatum, self).save(*args, **kwargs)  # Call the "real" save() method.
-        if not self.signature.last_updated or (self.signature.last_updated <
-                                               self.push_timestamp):
+        if self.signature.last_updated < self.push_timestamp:
             self.signature.last_updated = self.push_timestamp
             self.signature.save()