Workaround for Misfit API bug where behavior of end dates is inconsis…

…tent

misfitapp/extras.py

@@ -18,14 +18,25 @@ def cc_to_underscore_keys(dictionary):
 def chunkify_dates(start, end, days_in_chunk=DAYS_IN_CHUNK):
     """
     Return a list of tuples that chunks the date range into ranges
-    of length days_in_chunk.
+    of length days_in_chunk, exclusive of the end date. So the end
+    date of one chunk is equal to the start date of the chunk after.
     """
     chunks = []
     s = start
     e = start + datetime.timedelta(days=days_in_chunk)
     while e - datetime.timedelta(days=days_in_chunk) < end:
         e = min(e, end)
         chunks.append((s, e))
-        s = e + datetime.timedelta(days=1)
+        s = e
         e = s + datetime.timedelta(days=days_in_chunk)
     return chunks
+
+
+def dedupe_by_field(l, field):
+    """
+    Returns a new list with duplicate objects removed. Objects are equal
+    iff the have the same value for 'field'.
+    """
+    d = {getattr(obj, field): obj for obj in l}
+    return list(d.values())
+

misfitapp/models.py

@@ -1,10 +1,13 @@
 from django.conf import settings
-from django.db import models
+from django.db import models, IntegrityError
 from django.utils.encoding import python_2_unicode_compatible
 from math import pow
 import datetime
 
-from .extras import cc_to_underscore_keys, chunkify_dates
+from .extras import (
+    cc_to_underscore_keys,
+    chunkify_dates,
+    dedupe_by_field)
 
 MAX_KEY_LEN = 24
 UserModel = getattr(settings, 'AUTH_USER_MODEL', 'auth.User')
@@ -59,7 +62,7 @@ def create_from_misfit(cls, misfit, uid, start_date=datetime.date(2014,1,1), end
                     data = cc_to_underscore_keys(summary.data)
                     data['user_id'] = uid
                     obj_list.append(cls(**data))
-        cls.objects.bulk_create(obj_list)
+        cls.objects.bulk_create(dedupe_by_field(obj_list, 'date'))
 
 
 @python_2_unicode_compatible
@@ -142,7 +145,7 @@ def create_from_misfit(cls, misfit, uid, start_date=datetime.date(2014,1,1), end
                     data = cc_to_underscore_keys(goal.data)
                     data['user_id'] = uid
                     obj_list.append(cls(**data))
-        cls.objects.bulk_create(obj_list)
+        cls.objects.bulk_create(dedupe_by_field(obj_list, 'date'))
 
 
 @python_2_unicode_compatible
@@ -189,7 +192,7 @@ def create_from_misfit(cls, misfit, uid, start_date=datetime.date(2014,1,1), end
                     data = cc_to_underscore_keys(session.data)
                     data['user_id'] = uid
                     obj_list.append(cls(**data))
-        cls.objects.bulk_create(obj_list)
+        cls.objects.bulk_create(dedupe_by_field(obj_list, 'start_time'))
 
 
 @python_2_unicode_compatible
@@ -226,13 +229,16 @@ def create_from_misfit(cls, misfit, uid, start_date=datetime.date(2014,1,1), end
                     data['user_id'] = uid
                     segments = data.pop('sleep_details')
                     s = cls(**data)
-                    s.save()
-                    for seg in segments:
-                        seg_list.append(SleepSegment(sleep=s,
-                                                     time=seg['datetime'],
-                                                     sleep_type=seg['value']))
-
-
+                    try:
+                        s.save()
+                        for seg in segments:
+                            seg_list.append(SleepSegment(sleep=s,
+                                                         time=seg['datetime'],
+                                                         sleep_type=seg['value']))
+                    except IntegrityError:
+                        # Could happen if Misfit gives us overlapping
+                        # data due to the date range bug in their API.
+                        pass
         SleepSegment.objects.bulk_create(seg_list)