Merge upstream PR Kozea#1337 (#3)

* Add basic free-busy report
* Fix bug in free busy report serializing a datetime tzinfo
* Improve free-busy report
* Package housekeeping
* Fix for free-busy `fbtype` statuses

---------

Co-authored-by: Ray <ray@react0r.com>

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## master
 
+## 3.2.0.a2
+
+* Fix for free-busy `fbtype` statuses
+
+## 3.2.0.a1
+
+* Added free-busy report
+* Using icalendar's tzinfo on created datetime fix issue with icalendar
+* Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports
+* Refactored some date parsing code
+
 ## 3.1.8
 
 * Fix setuptools requirement if installing wheel

radicale/app/report.py

@@ -21,22 +21,108 @@
 import posixpath
 import socket
 import xml.etree.ElementTree as ET
+import vobject
 from http import client
 from typing import Callable, Iterable, Iterator, Optional, Sequence, Tuple
 from urllib.parse import unquote, urlparse
 
 import radicale.item as radicale_item
-from radicale import httputils, pathutils, storage, types, xmlutils
+from radicale import httputils, pathutils, storage, types, xmlutils, config
 from radicale.app.base import Access, ApplicationBase
 from radicale.item import filter as radicale_filter
 from radicale.log import logger
 
+def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element],
+                     collection: storage.BaseCollection, encoding: str,
+                     unlock_storage_fn: Callable[[], None],
+                     max_occurrence: int
+                     ) -> Tuple[int, str]:
+    multistatus = ET.Element(xmlutils.make_clark("D:multistatus"))
+    if xml_request is None:
+        return client.MULTI_STATUS, multistatus
+    root = xml_request
+    if (root.tag == xmlutils.make_clark("C:free-busy-query") and
+            collection.tag != "VCALENDAR"):
+        logger.warning("Invalid REPORT method %r on %r requested",
+                       xmlutils.make_human_tag(root.tag), path)
+        return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report")
+
+    time_range_element = root.find(xmlutils.make_clark("C:time-range"))
+
+    # Build a single filter from the free busy query for retrieval
+    # TODO: filter for VFREEBUSY in additional to VEVENT but
+    # test_filter doesn't support that yet.
+    vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                   attrib={'name':'VEVENT'})
+    vevent_cf_element.append(time_range_element)
+    vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                   attrib={'name':'VCALENDAR'})
+    vcalendar_cf_element.append(vevent_cf_element)
+    filter_element = ET.Element(xmlutils.make_clark("C:filter"))
+    filter_element.append(vcalendar_cf_element)
+    filters = (filter_element,)
+
+    # First pull from storage
+    retrieved_items = list(collection.get_filtered(filters))
+    # !!! Don't access storage after this !!!
+    unlock_storage_fn()
+
+    cal = vobject.iCalendar()
+    collection_tag = collection.tag
+    while retrieved_items:
+        # Second filtering before evaluating occurrences.
+        # ``item.vobject_item`` might be accessed during filtering.
+        # Don't keep reference to ``item``, because VObject requires a lot of
+        # memory.
+        item, filter_matched = retrieved_items.pop(0)
+        if not filter_matched:
+            try:
+                if not test_filter(collection_tag, item, filter_element):
+                    continue
+            except ValueError as e:
+                raise ValueError("Failed to free-busy filter item %r from %r: %s" %
+                                 (item.href, collection.path, e)) from e
+            except Exception as e:
+                raise RuntimeError("Failed to free-busy filter item %r from %r: %s" %
+                                   (item.href, collection.path, e)) from e
+
+        fbtype = None
+        if item.component_name == 'VEVENT':
+            transp = getattr(item.vobject_item.vevent, 'transp', None)
+            if transp and transp.value != 'OPAQUE':
+                continue
+
+            status = getattr(item.vobject_item.vevent, 'status', None)
+            if not status or status.value == 'CONFIRMED':
+                fbtype = 'BUSY'
+            elif status.value == 'CANCELLED':
+                fbtype = 'FREE'
+            elif status.value == 'TENTATIVE':
+                fbtype = 'BUSY-TENTATIVE'
+            else:
+                # Could do fbtype = status.value for x-name, I prefer this
+                fbtype = 'BUSY'
+
+        # TODO: coalesce overlapping periods
+
+        occurrences = radicale_filter.time_range_fill(item.vobject_item,
+                                                      time_range_element,
+                                                      "VEVENT",
+                                                      n=max_occurrence)
+        for occurrence in occurrences:
+            vfb = cal.add('vfreebusy')
+            vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value
+            vfb.add('dtstart').value, vfb.add('dtend').value = occurrence
+            if fbtype:
+                vfb.add('fbtype').value = fbtype
+    return (client.OK, cal.serialize())
+
 
 def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element],
                collection: storage.BaseCollection, encoding: str,
                unlock_storage_fn: Callable[[], None]
                ) -> Tuple[int, ET.Element]:
-    """Read and answer REPORT requests.
+    """Read and answer REPORT requests that return XML.
 
     Read rfc3253-3.6 for info.
 
@@ -295,13 +381,28 @@ def do_REPORT(self, environ: types.WSGIEnviron, base_prefix: str,
             else:
                 assert item.collection is not None
                 collection = item.collection
-            try:
-                status, xml_answer = xml_report(
-                    base_prefix, path, xml_content, collection, self._encoding,
-                    lock_stack.close)
-            except ValueError as e:
-                logger.warning(
-                    "Bad REPORT request on %r: %s", path, e, exc_info=True)
-                return httputils.BAD_REQUEST
-        headers = {"Content-Type": "text/xml; charset=%s" % self._encoding}
-        return status, headers, self._xml_response(xml_answer)
+
+            if xml_content is not None and \
+               xml_content.tag == xmlutils.make_clark("C:free-busy-query"):
+                max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence")
+                try:
+                    status, body = free_busy_report(
+                        base_prefix, path, xml_content, collection, self._encoding,
+                        lock_stack.close, max_occurrence)
+                except ValueError as e:
+                    logger.warning(
+                        "Bad REPORT request on %r: %s", path, e, exc_info=True)
+                    return httputils.BAD_REQUEST
+                headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding}
+                return status, headers, body
+            else:
+                try:
+                    status, xml_answer = xml_report(
+                        base_prefix, path, xml_content, collection, self._encoding,
+                        lock_stack.close)
+                except ValueError as e:
+                    logger.warning(
+                        "Bad REPORT request on %r: %s", path, e, exc_info=True)
+                    return httputils.BAD_REQUEST
+                headers = {"Content-Type": "text/xml; charset=%s" % self._encoding}
+                return status, headers, self._xml_response(xml_answer)

radicale/config.py

@@ -226,8 +226,13 @@ def _convert_to_bool(value: Any) -> bool:
             "help": "mask passwords in logs",
             "type": bool})])),
     ("headers", OrderedDict([
-        ("_allow_extra", str)]))])
-
+        ("_allow_extra", str)])),
+    ("reporting", OrderedDict([
+        ("max_freebusy_occurrence", {
+            "value": "10000",
+            "help": "number of occurrences per event when reporting",
+            "type": positive_int})]))
+    ])
 
 def parse_compound_paths(*compound_paths: Optional[str]
                          ) -> List[Tuple[str, bool]]:

radicale/item/filter.py

@@ -48,10 +48,32 @@ def date_to_datetime(d: date) -> datetime:
     if not isinstance(d, datetime):
         d = datetime.combine(d, datetime.min.time())
     if not d.tzinfo:
-        d = d.replace(tzinfo=timezone.utc)
+        # NOTE: using vobject's UTC as it wasn't playing well with datetime's.
+        d = d.replace(tzinfo=vobject.icalendar.utc)
     return d
 
 
+def parse_time_range(time_filter: ET.Element) -> Tuple[datetime, datetime]:
+    start_text = time_filter.get("start")
+    end_text = time_filter.get("end")
+    if start_text:
+        start = datetime.strptime(
+            start_text, "%Y%m%dT%H%M%SZ").replace(
+                tzinfo=timezone.utc)
+    else:
+        start = DATETIME_MIN
+    if end_text:
+        end = datetime.strptime(
+            end_text, "%Y%m%dT%H%M%SZ").replace(
+                tzinfo=timezone.utc)
+    else:
+        end = DATETIME_MAX
+    return start, end
+
+def time_range_timestamps(time_filter: ET.Element) -> Tuple[int, int]:
+    start, end = parse_time_range(time_filter)
+    return (math.floor(start.timestamp()), math.ceil(end.timestamp()))
+
 def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool:
     """Check whether the ``item`` matches the comp ``filter_``.
 
@@ -147,21 +169,10 @@ def time_range_match(vobject_item: vobject.base.Component,
     """Check whether the component/property ``child_name`` of
        ``vobject_item`` matches the time-range ``filter_``."""
 
-    start_text = filter_.get("start")
-    end_text = filter_.get("end")
-    if not start_text and not end_text:
+    if not filter_.get("start") and not filter_.get("end"):
         return False
-    if start_text:
-        start = datetime.strptime(start_text, "%Y%m%dT%H%M%SZ")
-    else:
-        start = datetime.min
-    if end_text:
-        end = datetime.strptime(end_text, "%Y%m%dT%H%M%SZ")
-    else:
-        end = datetime.max
-    start = start.replace(tzinfo=timezone.utc)
-    end = end.replace(tzinfo=timezone.utc)
 
+    start, end = parse_time_range(filter_)
     matched = False
 
     def range_fn(range_start: datetime, range_end: datetime,
@@ -181,6 +192,34 @@ def infinity_fn(start: datetime) -> bool:
     return matched
 
 
+def time_range_fill(vobject_item: vobject.base.Component,
+                    filter_: ET.Element, child_name: str, n: int = 1
+                    ) -> List[Tuple[datetime, datetime]]:
+    """Create a list of ``n`` occurances from the component/property ``child_name``
+       of ``vobject_item``."""
+    if not filter_.get("start") and not filter_.get("end"):
+        return []
+
+    start, end = parse_time_range(filter_)
+    ranges: List[Tuple[datetime, datetime]] = []
+    def range_fn(range_start: datetime, range_end: datetime,
+                 is_recurrence: bool) -> bool:
+        nonlocal ranges
+        if start < range_end and range_start < end:
+            ranges.append((range_start, range_end))
+            if n > 0 and len(ranges) >= n:
+                return True
+        if end < range_start and not is_recurrence:
+            return True
+        return False
+
+    def infinity_fn(range_start: datetime) -> bool:
+        return False
+
+    visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn)
+    return ranges
+
+
 def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str,
                       range_fn: Callable[[datetime, datetime, bool], bool],
                       infinity_fn: Callable[[datetime], bool]) -> None:
@@ -543,20 +582,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str
                 if time_filter.tag != xmlutils.make_clark("C:time-range"):
                     simple = False
                     continue
-                start_text = time_filter.get("start")
-                end_text = time_filter.get("end")
-                if start_text:
-                    start = math.floor(datetime.strptime(
-                        start_text, "%Y%m%dT%H%M%SZ").replace(
-                            tzinfo=timezone.utc).timestamp())
-                else:
-                    start = TIMESTAMP_MIN
-                if end_text:
-                    end = math.ceil(datetime.strptime(
-                        end_text, "%Y%m%dT%H%M%SZ").replace(
-                            tzinfo=timezone.utc).timestamp())
-                else:
-                    end = TIMESTAMP_MAX
+                start, end = time_range_timestamps(time_filter)
                 return tag, start, end, simple
             return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple
     return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple

radicale/tests/__init__.py

@@ -27,6 +27,7 @@
 import tempfile
 import wsgiref.util
 import xml.etree.ElementTree as ET
+import vobject
 from io import BytesIO
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -35,7 +36,7 @@
 import radicale
 from radicale import app, config, types, xmlutils
 
-RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]]]]
+RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]], vobject.base.Component]]
 
 # Enable debug output
 radicale.log.logger.setLevel(logging.DEBUG)
@@ -107,8 +108,7 @@ def start_response(status_: str, headers_: List[Tuple[str, str]]
     def parse_responses(text: str) -> RESPONSES:
         xml = DefusedET.fromstring(text)
         assert xml.tag == xmlutils.make_clark("D:multistatus")
-        path_responses: Dict[str, Union[
-            int, Dict[str, Tuple[int, ET.Element]]]] = {}
+        path_responses: RESPONSES = {}
         for response in xml.findall(xmlutils.make_clark("D:response")):
             href = response.find(xmlutils.make_clark("D:href"))
             assert href.text not in path_responses
@@ -133,6 +133,12 @@ def parse_responses(text: str) -> RESPONSES:
                 path_responses[href.text] = prop_respones
         return path_responses
 
+    @staticmethod
+    def parse_free_busy(text: str) -> RESPONSES:
+        path_responses: RESPONSES = {}
+        path_responses[""] = vobject.readOne(text)
+        return path_responses
+
     def get(self, path: str, check: Optional[int] = 200, **kwargs
             ) -> Tuple[int, str]:
         assert "data" not in kwargs
@@ -177,13 +183,18 @@ def proppatch(self, path: str, data: Optional[str] = None,
         return status, responses
 
     def report(self, path: str, data: str, check: Optional[int] = 207,
+               is_xml: Optional[bool] = True,
                **kwargs) -> Tuple[int, RESPONSES]:
         status, _, answer = self.request("REPORT", path, data, check=check,
                                          **kwargs)
         if status < 200 or 300 <= status:
             return status, {}
         assert answer is not None
-        return status, self.parse_responses(answer)
+        if is_xml:
+            parsed = self.parse_responses(answer)
+        else:
+            parsed = self.parse_free_busy(answer)
+        return status, parsed
 
     def delete(self, path: str, check: Optional[int] = 200, **kwargs
                ) -> Tuple[int, RESPONSES]:

radicale/tests/static/event10.ics

@@ -0,0 +1,36 @@
+BEGIN:VCALENDAR
+PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+TZID:Europe/Paris
+X-LIC-LOCATION:Europe/Paris
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20130902T150157Z
+LAST-MODIFIED:20130902T150158Z
+DTSTAMP:20130902T150158Z
+UID:event10
+SUMMARY:Event
+CATEGORIES:some_category1,another_category2
+ORGANIZER:mailto:unclesam@example.com
+ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com
+ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com
+DTSTART;TZID=Europe/Paris:20130901T180000
+DTEND;TZID=Europe/Paris:20130901T190000
+STATUS:CANCELLED
+END:VEVENT
+END:VCALENDAR