reduced querying time for forever recurring events (#80)

* reduced querying time for forever recurring events

- added indexing
- removed end_recurrence fields for forever recurring events
- added forever field for forever recurring events
- fixed querying to take into account forever events
- creation of forever recurring dummy events optimized

* fixed sample data

got rid of log statement

* fixed yearly recurrences

* fixed output ics multiple all day events

* fixed forever query mixup

* cleaned logging

* cleaned logging

* fixed default date range

* got rid of logging statement

* reorganized importing

abe/document_models/event_documents.py

@@ -13,9 +13,11 @@ class RecurringEventDefinition(EmbeddedDocument):
     until = DateTimeField()
     by_day = ListField(StringField())
     by_month_day = StringField()
-    by_month = ListField(StringField())
+    by_month = StringField()
     by_year_day = ListField(StringField())
 
+    forever = BooleanField(default=False)
+
 
 class RecurringEventExc(EmbeddedDocument):  # TODO: get a better name
     """Model for Exceptions to recurring events"""
@@ -33,6 +35,13 @@ class RecurringEventExc(EmbeddedDocument):  # TODO: get a better name
     _id = ObjectIdField(default=ObjectId)
     UID = StringField()
     allDay = BooleanField(default=False)
+    meta = {
+        'indexes': [
+            'sid',
+            'rec_id',
+            '_id'
+        ]
+    }
 
 
 class Event(Document):
@@ -57,6 +66,11 @@ class Event(Document):
 
     UID = StringField()
     ics_id = ObjectIdField()
-    meta = {'allow_inheritance': True}  # TODO: set indexes
+    meta = {'allow_inheritance': True,
+        'indexes': [
+            'start',
+            'end',
+            'recurrence_end']
+        }
 
     # TODO: look into clean() function for more advanced data validation

abe/helper_functions/ics_helpers.py

@@ -42,8 +42,7 @@ def create_ics_event(event,recurrence=False):
         end_string = 'dtend;VALUE=DATE'
         event_start = date_to_ics(event['start'].isoformat())
         event_end = date_to_ics(event['end'].isoformat())
-        if (event['end'] - event['start']) < timedelta(days=1):
-            event_end = str(int(event_end) + 1)
+        event_end = str(int(event_end) + 1)
     else:
         start_string = 'dtstart'
         end_string = 'dtend'

abe/helper_functions/query_helpers.py

@@ -11,6 +11,7 @@
 from icalendar import Calendar, Event, vCalAddress, vText, vDatetime
 from dateutil.rrule import rrule, MONTHLY, WEEKLY, DAILY, YEARLY
 from datetime import datetime, timedelta, timezone
+from dateutil.relativedelta import relativedelta
 from bson import objectid
 from mongoengine import *
 from icalendar import Calendar
@@ -46,6 +47,12 @@ def get_to_event_search(request):
     for key, process in preprocessing.items():
         if key in search_dict.keys():
             search_dict[key] = process(search_dict[key])
+
+    now = datetime.now()
+    if 'start' not in search_dict:
+        search_dict['start'] = now + relativedelta(months=-1)
+    if 'end' not in search_dict:
+        search_dict['end'] = now + relativedelta(months=+2)
     return search_dict
 
 
@@ -84,7 +91,10 @@ def event_query(search_dict):
         if key in search_dict.keys():
             query_rec_event.update(get_pattern(search_dict[key]))
 
-    query = {'$or': [query_reg_event,query_rec_event]}
+    query_forever = {'forever' : True}
+
+    query = {'$or': [query_rec_event, query_reg_event, query_forever]}
+    #logging.debug("this query: {}".format(query))
     return query
 
 

abe/helper_functions/recurring_helpers.py

@@ -33,7 +33,7 @@ def recurring_to_full(event, events_list, start, end):
                     and sub_event['deleted']==False:
                     events_list.append(sub_event_to_full(mongo_to_dict(sub_event), event))
 
-    rule_list = instance_creation(event)
+    rule_list = instance_creation(event, end)
 
     for instance in rule_list:
         if instance >= start and instance < end:

abe/helper_functions/sub_event_helpers.py

@@ -57,7 +57,8 @@ def update_sub_event(received_data, parent_event, sub_event_id, ics=False):
                 updated_sub_event = db.RecurringEventExc(**updated_sub_event_dict)
                 parent_event.update(pull__sub_events___id=sub_event_id)
                 parent_event.update(add_to_set__sub_events=updated_sub_event_dict)
-                parent_event.recurrence_end = find_recurrence_end(parent_event)
+                if updated_sub_event_dict['forever'] == False:
+                    parent_event.recurrence_end = find_recurrence_end(parent_event)
                 parent_event.save()
                 parent_event.reload()
                 return(updated_sub_event)
@@ -68,7 +69,8 @@ def update_sub_event(received_data, parent_event, sub_event_id, ics=False):
                 updated_sub_event = db.RecurringEventExc(**updated_sub_event_dict)
                 parent_event.update(pull__sub_events__rec_id=sub_event_id)
                 parent_event.update(add_to_set__sub_events=updated_sub_event_dict)
-                parent_event.recurrence_end = find_recurrence_end(parent_event)
+                if updated_sub_event_dict['forever'] == False:
+                    parent_event.recurrence_end = find_recurrence_end(parent_event)
                 parent_event.save()
                 parent_event.reload()
                 return(updated_sub_event)
@@ -111,7 +113,8 @@ def create_new_sub_event_defintion(sub_event, updates, parent_event):
     return(sub_event)
 
 
-def instance_creation(event):
+def instance_creation(event, end=None):
+
     rec_type_list = ['YEARLY', 'MONTHLY', 'WEEKLY', 'DAILY']
 
     day_list = ['MO', 'TU', 'WE', 'TH', 'FR', 'SA', 'SU']
@@ -120,18 +123,30 @@ def instance_creation(event):
     ensure_date_time = lambda a: dateutil.parser.parse(a) if not isinstance(a, datetime) else a
 
     rFrequency = rec_type_list.index(recurrence['frequency'])
+    if recurrence['frequency'] == 'YEARLY':
+        start = ensure_date_time(event['start'])
+        rByMonth = int(start.month)
+        rByMonthDay = int(start.day)
+        rByDay = None
+    else:
+        rByMonthDay = int(recurrence['by_month_day']) if 'by_month_day' in recurrence else None
+        rByMonth = int(recurrence['by_month']) if 'by_month' in recurrence else None
+        if 'by_day' in recurrence:
+            rByDay = []
+            for i in recurrence['by_day']:
+                rByDay.append(day_list.index(i))
+            else:
+                rByDay = None
+
     rInterval = int(recurrence['interval'])
-    rCount = int(recurrence['count']) if 'count' in recurrence else None
-    rUntil = ensure_date_time(recurrence['until']) if 'until' in recurrence else None
-    rByMonth = recurrence['by_month'] if 'by_month' in recurrence else None
-    rByMonthDay = recurrence['by_month_day'] if 'by_month_day' in recurrence else None
-
-    if 'by_day' in recurrence:
-        rByDay = []
-        for i in recurrence['by_day']:
-            rByDay.append(day_list.index(i))
+    if recurrence.forever == True:
+        rUntil = ensure_date_time(end) if end is not None else None
     else:
-        rByDay = None
+        rUntil = ensure_date_time(recurrence['until']) if 'until' in recurrence else None
+    rCount = int(recurrence['count']) if 'count' in recurrence else None
+
+
+
 
     rule_list = list(rrule(freq=rFrequency, count=rCount, interval=rInterval, until=rUntil, bymonth=rByMonth, \
         bymonthday=rByMonthDay, byweekday=rByDay, dtstart=ensure_date_time(event['start'])))
@@ -140,5 +155,7 @@ def instance_creation(event):
 
 
 def find_recurrence_end(event):
+    ensure_date_time = lambda a: dateutil.parser.parse(a) if not isinstance(a, datetime) else a
     rule_list = instance_creation(event)
-    return(rule_list[-1])
+    event_end = rule_list[-1] + timedelta(hours=24)
+    return(event_end)

abe/resource_models/event_resources.py

@@ -60,26 +60,18 @@ def get(self, event_id=None, rec_id=None):
             if not results:
                 return []
 
-            if 'start' in query_dict:
-                start = query_dict['start']
-            else:
-                start = datetime(2017,6,1)
-            if 'end' in query_dict:
-                end = query_dict['end']
-            else:
-                end = datetime(2017, 9, 20)
+            start = query_dict['start']
+            end = query_dict['end']
 
             events_list = []
             for event in results:
                 # checks for recurrent events
                 if 'recurrence' in event:
-                    logging.debug("recurrence end: {}".format(mongo_to_dict(event)))
+
                     # checks for events from a recurrence that's been edited
                     events_list = recurring_to_full(event, events_list, start, end)
                 else:
-                    #logging.debug("normal: {}".format(mongo_to_dict(event)))
                     events_list.append(mongo_to_dict(event))
-            #logging.debug("events_list: {}".format(events_list))
             return events_list
 
     def post(self):
@@ -91,8 +83,9 @@ def post(self):
             if new_event.labels == []:
                 new_event.labels = ['unlabeled']
             if 'recurrence' in new_event:
-                new_event.recurrence_end = find_recurrence_end(new_event)
-                logging.debug("made an end: {}".format(new_event.recurrence_end))
+                if new_event.recurrence.forever == False:
+                    new_event.recurrence_end = find_recurrence_end(new_event)
+                    logging.debug("made an end: {}".format(new_event.recurrence_end))
             new_event.save()
         except ValidationError as error:
             return {'error_type': 'validation',

abe/sample_data.py

@@ -223,6 +223,7 @@ def load_data(
     ics_data=sample_ics
 ):
     import logging
+    from .helper_functions.sub_event_helpers import find_recurrence_end
     logging.basicConfig(level=logging.DEBUG)
     if event_data:
         logging.info("Inserting sample event data")
@@ -234,7 +235,12 @@ def load_data(
                         value,
                         'US/Eastern'
                     ).to('utc').datetime
-            db.Event(**event).save()
+            new_event = db.Event(**event)
+            if 'recurrence' in new_event:
+                if new_event.recurrence.forever == False:
+                    new_event.recurrence_end = find_recurrence_end(new_event)
+                    logging.info("made some end recurrences: {}".format(new_event.recurrence_end))
+            new_event.save()
     if label_data:
         logging.info("Inserting sample label data")
         for index, label in enumerate(label_data):
@@ -249,4 +255,5 @@ def load_data(
 
 if __name__ == '__main__':  # import data
     from . import database as db
+
     load_data(db)