Expand recurring events client-side

caldav/objects.py

@@ -881,17 +881,11 @@ def date_search(
                     continue
                 components = o.vobject_instance.components()
                 for i in components:
-                    if i.name == "VEVENT":
+                    if i.name in ("VEVENT", "VTODO"):
                         recurrance_properties = ["exdate", "exrule", "rdate", "rrule"]
                         if any(key in recurrance_properties for key in i.contents):
-                            if verify_expand:
-                                raise error.ReportError(
-                                    "CalDAV server did not expand recurring vevents as requested. See https://github.com/python-caldav/caldav/issues/157"
-                                )
-                            else:
-                                logging.error(
-                                    "CalDAV server does not support recurring events properly.  See https://github.com/python-caldav/caldav/issues/157"
-                                )
+                            o.expand_rrule(start, end)
+
         return objects
 
     def _request_report_build_resultlist(
@@ -973,7 +967,7 @@ def search(
         * text attribute search parameters: category, uid, summary, omment,
           description, location, status
         * expand - do server side expanding of recurring events/tasks
-        * start, stop: do a time range search
+        * start, end: do a time range search
         * filters - other kind of filters (in lxml tree format)
         * sort_keys - list of attributes to use when sorting
 
@@ -1030,6 +1024,36 @@ def search(
                 )
             (response, objects) = self._request_report_build_resultlist(xml, comp_class)
 
+        if kwargs.get("expand", False):
+            if "start" in kwargs:
+                start = kwargs["start"]
+            else:
+                # TODO get from xml
+                raise NotImplementedError("Getting start from xml is not supported")
+            if "end" in kwargs:
+                end = kwargs["end"]
+            else:
+                # TODO get from xml
+                raise NotImplementedError("Getting start from xml is not supported")
+
+            for o in objects:
+                if not o.data:
+                    continue
+                components = o.vobject_instance.components()
+                for i in components:
+                    if i.name in ("VEVENT", "VTODO"):
+                        ## Those recurrance properties should not be in the returns from the server,
+                        ## if they are present it indicates that server expand didn't work and we'll
+                        ## have to do it on the client side
+                        recurrance_properties = ["exdate", "exrule", "rdate", "rrule"]
+                        if any(key in recurrance_properties for key in i.contents):
+                            o.expand_rrule(start, end)
+            if split_expanded:
+                objects_ = objects
+                objects = []
+                for o in objects_:
+                    objects.extend(o.split_expanded())
+
         def sort_key_func(x):
             ret = []
             for objtype in ("vtodo", "vevent", "vjournal"):
@@ -1644,6 +1668,50 @@ def split_expanded(self):
             ret.append(obj)
         return ret
 
+    def expand_rrule(self, start, end):
+        """This method will transform the calendar content of the
+        event and expand the calendar data from a "master copy" with
+        RRULE set and into a "recurrence set" with RECURRENCE-ID set
+        and no RRULE set.  The main usage is for client-side expansion
+        in case the calendar server does not support server-side
+        expansion.  It should be safe to save back to the server, the
+        server should recognize it as recurrences and should not edit
+        the "master copy".  If doing a `self.load`, the calendar
+        content will be replaced with the "master copy".  However, as
+        of 2022-10 there is no test code verifying this.
+
+        :param event: Event
+        :param start: datetime.datetime
+        :param end: datetime.datetime
+
+        """
+        import recurring_ical_events
+
+        recurrings = recurring_ical_events.of(self.icalendar_instance).between(
+            start, end
+        )
+        recurrance_properties = ["exdate", "exrule", "rdate", "rrule"]
+        # FIXME too much copying
+        stripped_event = self.copy(keep_uid=True)
+        # remove all recurrance properties
+        for component in stripped_event.vobject_instance.components():
+            if component.name in ("VEVENT", "VTODO"):
+                for key in recurrance_properties:
+                    try:
+                        del component.contents[key]
+                    except KeyError:
+                        pass
+
+        calendar = self.icalendar_instance
+        calendar.subcomponents = []
+        for occurance in recurrings:
+            occurance.add("RECURRENCE-ID", occurance.get("DTSTART"))
+            calendar.add_component(occurance)
+        # add other components (except for the VEVENT itself and VTIMEZONE which is not allowed on occurance events)
+        for component in stripped_event.icalendar_instance.subcomponents:
+            if component.name not in ("VEVENT", "VTODO", "VTIMEZONE"):
+                calendar.add_component(component)
+
     def set_relation(
         self, other, reltype=None, set_reverse=True
     ):  ## TODO: logic to find and set siblings?

tests/compatibility_issues.py

@@ -13,21 +13,21 @@
 ## * Consider how to get this into the documentation
 incompatibility_description = {
     'no_expand':
-        """Server may throw errors when asked to do a expanded date search""",
+        """Server may throw errors when asked to do a expanded date search (this is ignored by the tests now, as we're doing client-side expansion)""",
 
     'no_recurring':
         """Server is having issues with recurring events and/or todos. """
         """date searches covering recurrances may yield no results, """
         """and events/todos may not be expanded with recurrances""",
 
     'no_recurring_expandation':
-        """Server will not expand recurring events""",
+        """Server will not expand recurring events (this is ignored by the tests now, as we're doing client-side expansion)""",
 
     'no_recurring_todo':
         """Recurring events are supported, but not recurring todos""",
 
     'no_recurring_todo_expand':
-        """Recurring todos aren't expanded""",
+        """Recurring todos aren't expanded (this is ignored by the tests now, as we're doing client-side expansion)""",
 
     'no_scheduling':
         """RFC6833 is not supported""",

tests/test_caldav.py

@@ -1457,12 +1457,11 @@ def testTodoDatesearch(self):
         # t5 has dtstart and due set prior to the search window
         # t6 has dtstart and due set prior to the search window, but is yearly recurring.
         # What will a date search yield?
-        noexpand = self.check_compatibility_flag("no_expand")
         todos = c.date_search(
             start=datetime(1997, 4, 14),
             end=datetime(2015, 5, 14),
             compfilter="VTODO",
-            expand=not noexpand,
+            expand=True,
         )
         # The RFCs are pretty clear on this.  rfc5545 states:
 
@@ -1493,12 +1492,7 @@ def testTodoDatesearch(self):
         assert len(todos) == foo
 
         ## verify that "expand" works
-        if (
-            not self.check_compatibility_flag("no_recurring_expandation")
-            and not self.check_compatibility_flag("no_expand")
-            and not self.check_compatibility_flag("no_recurring_todo_expand")
-        ):
-            assert len([x for x in todos if "DTSTART:20020415T1330" in x.data]) == 1
+        assert len([x for x in todos if "DTSTART:20020415T1330" in x.data]) == 1
         ## exercise the default for expand (maybe -> False for open-ended search)
         todos = c.date_search(start=datetime(2025, 4, 14), compfilter="VTODO")
 
@@ -1952,39 +1946,31 @@ def testRecurringDateSearch(self):
         # if not self.check_compatibility_flag('no_mkcalendar'):
         # assert len(r) == 0
 
-        if not self.check_compatibility_flag("no_expand"):
-            ## With expand=True, we should find one occurrence
-            r = c.date_search(
-                datetime(2008, 11, 1, 17, 00, 00),
-                datetime(2008, 11, 3, 17, 00, 00),
-                expand=True,
-            )
-            assert len(r) == 1
-            assert r[0].data.count("END:VEVENT") == 1
-            ## due to expandation, the DTSTART should be in 2008
-            if not self.check_compatibility_flag("no_recurring_expandation"):
-                assert r[0].data.count("DTSTART;VALUE=DATE:2008") == 1
-
-            ## With expand=True and searching over two recurrences ...
-            r = c.date_search(
-                datetime(2008, 11, 1, 17, 00, 00),
-                datetime(2009, 11, 3, 17, 00, 00),
-                expand=True,
-            )
+        ## With expand=True, we should find one occurrence
+        r = c.date_search(
+            datetime(2008, 11, 1, 17, 00, 00),
+            datetime(2008, 11, 3, 17, 00, 00),
+            expand=True,
+        )
+        assert len(r) == 1
+        assert r[0].data.count("END:VEVENT") == 1
+        ## due to expandation, the DTSTART should be in 2008
+        assert r[0].data.count("DTSTART;VALUE=DATE:2008") == 1
 
-            ## According to https://tools.ietf.org/html/rfc4791#section-7.8.3, the
-            ## resultset should be one vcalendar with two events.
-            assert len(r) == 1
+        ## With expand=True and searching over two recurrences ...
+        r = c.date_search(
+            datetime(2008, 11, 1, 17, 00, 00),
+            datetime(2009, 11, 3, 17, 00, 00),
+            expand=True,
+        )
 
-            ## not all servers supports expandation
-            if self.check_compatibility_flag("no_recurring_expandation"):
-                ## without expandation, we'll get the original ics,
-                ## with RRULE set
-                assert "RRULE" in r[0].data
-                assert r[0].data.count("END:VEVENT") == 1
-            else:
-                assert "RRULE" not in r[0].data
-                assert r[0].data.count("END:VEVENT") == 2
+        ## According to https://tools.ietf.org/html/rfc4791#section-7.8.3, the
+        ## resultset should be one vcalendar with two events.
+        assert len(r) == 1
+
+        ## not all servers supports expandation
+        assert "RRULE" not in r[0].data
+        assert r[0].data.count("END:VEVENT") == 2
 
         # The recurring events should not be expanded when using the
         # events() method

tests/test_caldav_unit.py

@@ -122,6 +122,38 @@
 END:VCALENDAR
 """
 
+# example from http://www.rfc-editor.org/rfc/rfc5545.txt
+evr = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Example Corp.//CalDAV Client//EN
+BEGIN:VEVENT
+UID:19970901T130000Z-123403@example.com
+DTSTAMP:19970901T130000Z
+DTSTART;VALUE=DATE:19971102
+SUMMARY:Our Blissful Anniversary
+TRANSP:TRANSPARENT
+CLASS:CONFIDENTIAL
+CATEGORIES:ANNIVERSARY,PERSONAL,SPECIAL OCCASION
+RRULE:FREQ=YEARLY
+END:VEVENT
+END:VCALENDAR"""
+
+todo6 = """
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Example Corp.//CalDAV Client//EN
+BEGIN:VTODO
+UID:19920901T130000Z-123408@host.com
+DTSTAMP:19920901T130000Z
+DTSTART:19920415T133000Z
+DUE:19920516T045959Z
+SUMMARY:Yearly Income Tax Preparation
+RRULE:FREQ=YEARLY
+CLASS:CONFIDENTIAL
+CATEGORIES:FAMILY,FINANCE
+PRIORITY:1
+END:VTODO
+END:VCALENDAR"""
 
 def MockedDAVResponse(text):
     """
@@ -144,7 +176,51 @@ def MockedDAVClient(xml_returned):
     client.request = mock.MagicMock(return_value=MockedDAVResponse(xml_returned))
     return client
 
-
+class TestExpandRRule:
+    """
+    Tests the expand_rrule method
+    """
+    def setup(self):
+        cal_url = "http://me:hunter2@calendar.example:80/"
+        client = DAVClient(url=cal_url)
+        self.yearly = Event(client, data=evr)
+        self.todo = Todo(client, data=todo6)
+
+    def testZero(self):
+        ## evr has rrule yearly and dtstart DTSTART 1997-11-02
+        ## This should cause 0 recurrences:
+        self.yearly.expand_rrule(start=datetime(1998,4,4), end=datetime(1998,10,10))
+        assert len(self.yearly.icalendar_instance.subcomponents) == 0
+
+    def testOne(self):
+        self.yearly.expand_rrule(start=datetime(1998,10,10), end=datetime(1998,12,12))
+        assert len(self.yearly.icalendar_instance.subcomponents) == 1
+        assert not 'RRULE' in self.yearly.icalendar_object()
+        assert 'UID' in self.yearly.icalendar_object()
+        assert 'RECURRENCE-ID' in self.yearly.icalendar_object()
+
+    def testThree(self):
+        self.yearly.expand_rrule(start=datetime(1996,10,10), end=datetime(1999,12,12))
+        assert len(self.yearly.icalendar_instance.subcomponents) == 3
+        data1 = self.yearly.icalendar_instance.subcomponents[0].to_ical()
+        data2 = self.yearly.icalendar_instance.subcomponents[1].to_ical()
+        assert data1.replace(b'199711', b'199811') == data2
+
+    def testThreeTodo(self):
+        self.todo.expand_rrule(start=datetime(1996,10,10), end=datetime(1999,12,12))
+        assert len(self.todo.icalendar_instance.subcomponents) == 3
+        data1 = self.todo.icalendar_instance.subcomponents[0].to_ical()
+        data2 = self.todo.icalendar_instance.subcomponents[1].to_ical()
+        assert data1.replace(b'199711', b'199811') == data2
+
+    def testSplit(self):
+        self.yearly.expand_rrule(start=datetime(1996,10,10), end=datetime(1999,12,12))
+        events = self.yearly.split_expanded()
+        assert len(events) == 3
+        assert len(events[0].icalendar_instance.subcomponents) == 1
+        assert events[1].icalendar_object()['UID'] == '19970901T130000Z-123403@example.com'
+
+
 class TestCalDAV:
     """
     Test class for "pure" unit tests (small internal tests, testing that