No longer use arrow for datetime/timezone things - make them explicit…

… instead

docs/api.rst

@@ -40,4 +40,10 @@ Models
 Settings
 --------
 
-.. automodule:: libres.context.settings
+.. automodule:: libres.context.settings
+
+Other
+-----
+
+.. automodule:: libres.modules.calendar
+    :members:

examples/flask/requirements.txt

@@ -1,3 +1,4 @@
+arrow
 flask
 isodate
 testing.postgresql

libres/db/models/allocation.py

@@ -181,9 +181,9 @@ def display_end(self, timezone=None):
 
     def prepare_daterange(self, start, end):
         if start:
-            start = calendar.normalize_date(start, self.timezone)
+            start = calendar.standardize_date(start, self.timezone)
         if end:
-            end = calendar.normalize_date(end, self.timezone)
+            end = calendar.standardize_date(end, self.timezone)
 
         return start, end
 

libres/db/models/timestamp.py

@@ -1,9 +1,8 @@
-from datetime import datetime
 from libres.db.models.types import UTCDateTime
-from pytz import timezone
+from libres.modules import calendar
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import deferred
 from sqlalchemy.schema import Column
-from sqlalchemy.ext.declarative import declared_attr
 
 
 class TimestampMixin(object):
@@ -18,7 +17,7 @@ class TimestampMixin(object):
 
     @staticmethod
     def timestamp():
-        return datetime.utcnow().replace(tzinfo=timezone('UTC'))
+        return calendar.utcnow()
 
     @declared_attr
     def created(cls):

libres/db/models/types/utcdatetime.py

@@ -1,22 +1,24 @@
+from libres.modules import calendar
 from sqlalchemy import types
-from dateutil.tz import tzutc
-from datetime import datetime
 
 
 class UTCDateTime(types.TypeDecorator):
+    """ Stores dates as UTC.
+
+    Internally, they are stored as timezone naive, because Postgres takes
+    the local timezone into account when working with timezones. We really
+    want to have those dates in UTC at all times, though for convenience we
+    make the dates timezone aware when retrieving the values and we make sure
+    that timezone aware dates are converted to UTC before storing.
+
+    """
 
     impl = types.DateTime
 
     def process_bind_param(self, value, engine):
         if value is not None:
-            assert value.tzinfo, 'datetimes must be timezone-aware'
-
-            # ..though they are stored internally without timezone in utc
-            # the timezone is attached again on date retrieval, see below.
-            return value.astimezone(tzutc()).replace(tzinfo=None)
+            return calendar.to_timezone(value, 'UTC').replace(tzinfo=None)
 
     def process_result_value(self, value, engine):
         if value is not None:
-            return datetime(value.year, value.month, value.day,
-                            value.hour, value.minute, value.second,
-                            value.microsecond, tzinfo=tzutc())
+            return calendar.replace_timezone(value, 'UTC')

libres/db/scheduler.py

@@ -78,15 +78,20 @@ def resource(self):
         return self.generate_uuid(self.name)
 
     def prepare_date(self, date):
-        return calendar.normalize_date(date, self.timezone)
+        return calendar.standardize_date(date, self.timezone)
 
     def prepare_dates(self, dates):
-        return calendar.normalize_dates(dates, self.timezone)
+        return [
+            (
+                calendar.standardize_date(s, self.timezone),
+                calendar.standardize_date(e, self.timezone)
+            ) for s, e in utils.pairs(dates)
+        ]
 
     def prepare_range(self, start, end):
         return (
-            calendar.normalize_date(start, self.timezone),
-            calendar.normalize_date(end, self.timezone)
+            calendar.standardize_date(start, self.timezone),
+            calendar.standardize_date(end, self.timezone)
         )
 
     @serialized
@@ -1124,8 +1129,8 @@ def search_allocations(
             s = datetime.combine(allocation.start.date(), start.time())
             e = datetime.combine(allocation.end.date(), end.time())
 
-            s = s.replace(tzinfo=allocation.start.tzinfo)
-            e = e.replace(tzinfo=allocation.end.tzinfo)
+            s = calendar.replace_timezone(s, allocation.start.tzname())
+            e = calendar.replace_timezone(e, allocation.start.tzname())
 
             if not allocation.overlaps(s, e):
                 continue

libres/modules/calendar.py

@@ -1,41 +1,92 @@
-import arrow
+""" The calendar module provides methods that deal with dates and timezones.
+
+There are projects like `Arrow <https://github.com/crsmithdev/arrow>`_ or
+`Delorean <https://github.com/crsmithdev/arrow>`_ which provide ways to work
+with timezones without having to think about it too much.
+
+Libres doesn't use them because its author *wants* to think about these things,
+to ensure they are correct, and partly because of self-loathing.
+
+Adding another layer makes these things harder.
+
+That being said, further up the stacks - in the web application for example -
+it might very well make sense to use a datetime wrapper library.
+
+"""
+
+import pytz
 
 from datetime import datetime, timedelta
-from libres.modules import errors, utils
+from libres.modules import compat, errors
 
+mindatetime = pytz.utc.localize(datetime.min)
+maxdatetime = pytz.utc.localize(datetime.max)
 
-mindatetime = arrow.get(datetime.min).datetime
-maxdatetime = arrow.get(datetime.max).datetime
 
+def ensure_timezone(timezone):
+    """ Make sure the given timezone is a pytz timezone, not just a string. """
 
-def normalize_dates(dates, timezone):
-    dates = list(utils.pairs(dates))
+    if isinstance(timezone, compat.string_types):
+        return pytz.timezone(timezone)
 
-    # the dates are expected to be given local to the timezone, but
-    # they are converted to utc for storage
-    for ix, (start, end) in enumerate(dates):
-        dates[ix] = [normalize_date(d, timezone) for d in ((start, end))]
+    return timezone
 
-    return dates
 
+def standardize_date(date, timezone):
+    """ Takes the given date and converts it to UTC.
+
+    The given timezone is set on timezone-naive dates and converted to
+    on timezone-aware dates. That essentially means that you should pass
+    the timezone that you know the date to be, even if the date is in another
+    timezone (like UTC) or if the date does not have a timezone set.
+
+    """
+
+    assert timezone, "The timezone may *not* be empty!"
 
-def normalize_date(date, timezone):
     if date.tzinfo is None:
-        date = arrow.get(date).replace(tzinfo=timezone)
+        date = replace_timezone(date, timezone)
 
     return to_timezone(date, 'UTC')
 
 
+def replace_timezone(date, timezone):
+    """ Takes the given date and replaces the timezone with the given timezone.
+
+    No conversion is done in this method, it's simply a safe way to do the
+    following (which is problematic with timzones that have daylight saving
+    times)::
+
+        # don't do this:
+        date.replace(tzinfo=timezone('Europe/Zurich'))
+
+        # do this:
+        calendar.replace_timezone(date, 'Europe/Zurich')
+
+    """
+
+    timezone = ensure_timezone(timezone)
+
+    return timezone.normalize(timezone.localize(date.replace(tzinfo=None)))
+
+
 def to_timezone(date, timezone):
+    """ Takes the given date and converts it to the given timezone.
+
+    The given date must already be timezone aware for this to work.
+
+    """
 
     if not date.tzinfo:
         raise errors.NotTimezoneAware()
 
-    return arrow.get(date).to(timezone).datetime
+    timezone = ensure_timezone(timezone)
+    return timezone.normalize(date.astimezone(timezone))
 
 
 def utcnow():
-    return normalize_date(datetime.utcnow(), 'UTC')
+    """ Returns a timezone-aware datetime.utcnow(). """
+    return replace_timezone(datetime.utcnow(), 'UTC')
 
 
 def is_whole_day(start, end, timezone):
@@ -49,8 +100,11 @@ def is_whole_day(start, end, timezone):
 
     """
 
-    # remove the timezone information after converting, to still detect
-    # days during which sumemrtime is enabled as 24 hour days.
+    # without replacing the tzinfo, the total seconds count later will return
+    # the wrong number - it is correct, because the total seconds do not
+    # constitute a whole day, but we are not interested in the actual time
+    # but we need to know that the day starts at 0:00 and ends at 24:00,
+    # between which we need 24 hours (just looking at the time)
     start = to_timezone(start, timezone).replace(tzinfo=None)
     end = to_timezone(end, timezone).replace(tzinfo=None)
 
@@ -69,6 +123,7 @@ def is_whole_day(start, end, timezone):
 
 
 def overlaps(start, end, otherstart, otherend):
+    """ Returns True if the given dates overlap in any way. """
 
     if otherstart <= start and start <= otherend:
         return True
@@ -80,6 +135,10 @@ def overlaps(start, end, otherstart, otherend):
 
 
 def count_overlaps(dates, start, end):
+    """ Goes through the list of start/end tuples in 'dates' and returns the
+    number of times start/end overlaps with any of the dates.
+
+    """
     count = 0
 
     for otherstart, otherend in dates:
@@ -115,7 +174,7 @@ def align_date_to_day(date, timezone, direction):
     if direction == 'up':
         local = local + timedelta(days=1, microseconds=-1)
 
-    return to_timezone(local, date.tzinfo)
+    return to_timezone(local, date.tzname())
 
 
 def align_range_to_day(start, end, timezone):

libres/tests/test_allocation.py

@@ -30,6 +30,7 @@ def test_add_invalid_allocation(scheduler):
 
 def test_date_functions(scheduler):
     allocation = Allocation(raster=60, resource=scheduler.resource)
+    allocation.timezone = 'UTC'
     allocation.start = datetime(2011, 1, 1, 12, 30, tzinfo=utc)
     allocation.end = datetime(2011, 1, 1, 14, 00, tzinfo=utc)
 

libres/tests/test_calendar.py

@@ -1,4 +1,3 @@
-import arrow
 import pytest
 
 from datetime import datetime
@@ -7,39 +6,46 @@
 from libres.modules import errors
 
 
-def test_normalize_naive_date():
+def test_standardize_naive_date():
     naive_date = datetime(2014, 10, 1, 13, 30)
-    normalized = calendar.normalize_date(naive_date, 'Europe/Zurich')
+    normalized = calendar.standardize_date(naive_date, 'Europe/Zurich')
 
     assert normalized.tzname() == 'UTC'
+    assert normalized.replace(tzinfo=None) == datetime(2014, 10, 1, 11, 30)
+
 
+def test_standardize_aware_date():
+    aware_date = calendar.replace_timezone(
+        datetime(2014, 10, 1, 13, 30), 'Europe/Zurich')
 
-def test_normalize_aware_date():
-    aware_date = datetime(2014, 10, 1, 13, 30)
-    normalized = calendar.normalize_date(aware_date, 'Europe/Zurich')
+    normalized = calendar.standardize_date(aware_date, 'Europe/Zurich')
 
     assert normalized.tzname() == 'UTC'
     assert normalized.replace(tzinfo=None) == datetime(2014, 10, 1, 11, 30)
 
 
 def test_is_whole_day_summertime():
 
-    start = datetime(2014, 10, 26, 0, 0, 0)
-    end = datetime(2014, 10, 26, 23, 59, 59)
+    start = calendar.standardize_date(
+        datetime(2014, 10, 26, 0, 0, 0), 'Europe/Zurich')
 
-    start, end = calendar.normalize_dates((start, end), 'Europe/Zurich')[0]
+    end = calendar.standardize_date(
+        datetime(2014, 10, 26, 23, 59, 59), 'Europe/Zurich')
 
     assert calendar.is_whole_day(start, end, 'Europe/Zurich')
+    assert not calendar.is_whole_day(start, end, 'Europe/Istanbul')
 
 
 def test_is_whole_day_wintertime():
 
-    start = datetime(2015, 3, 29, 0, 0, 0)
-    end = datetime(2015, 3, 29, 23, 59, 59)
+    start = calendar.standardize_date(
+        datetime(2015, 3, 29, 0, 0, 0), 'Europe/Zurich')
 
-    start, end = calendar.normalize_dates((start, end), 'Europe/Zurich')[0]
+    end = calendar.standardize_date(
+        datetime(2015, 3, 29, 23, 59, 59), 'Europe/Zurich')
 
     assert calendar.is_whole_day(start, end, 'Europe/Zurich')
+    assert not calendar.is_whole_day(start, end, 'Europe/Istanbul')
 
 
 def test_require_timezone_awareness():
@@ -81,28 +87,30 @@ def test_overlaps():
     for dates in overlaps:
         assert calendar.overlaps(*dates)
 
-        timezone_aware = [calendar.normalize_date(d, tz) for d in dates]
+        timezone_aware = [calendar.standardize_date(d, tz) for d in dates]
         assert calendar.overlaps(*timezone_aware)
 
     for dates in doesnt:
         assert not calendar.overlaps(*dates)
 
-        timezone_aware = [calendar.normalize_date(d, tz) for d in dates]
+        timezone_aware = [calendar.standardize_date(d, tz) for d in dates]
         assert not calendar.overlaps(*timezone_aware)
 
 
-def test_align_date_to_day():
+def test_align_date_to_day_down():
+
+    unaligned = calendar.standardize_date(datetime(2012, 1, 24, 10), 'UTC')
+    aligned = calendar.align_date_to_day(unaligned, 'Europe/Zurich', 'down')
 
-    aligned = calendar.align_date_to_day(
-        arrow.get(2012, 1, 24, 10), 'Europe/Zurich', 'down'
-    )
-    assert aligned == arrow.get(2012, 1, 24, 0, tzinfo='Europe/Zurich')
     assert aligned.tzname() == 'UTC'
+    assert aligned == calendar.standardize_date(
+        datetime(2012, 1, 24, 0), 'Europe/Zurich')
+
+
+def test_align_date_to_day_up():
+    unaligned = calendar.standardize_date(datetime(2012, 1, 24, 10), 'UTC')
+    aligned = calendar.align_date_to_day(unaligned, 'Europe/Zurich', 'up')
 
-    aligned = calendar.align_date_to_day(
-        arrow.get(2012, 1, 24, 10), 'Europe/Zurich', 'up'
-    )
-    assert aligned == arrow.get(
-        2012, 1, 24, 23, 59, 59, 999999, tzinfo='Europe/Zurich'
-    )
     assert aligned.tzname() == 'UTC'
+    assert aligned == calendar.standardize_date(
+        datetime(2012, 1, 24, 23, 59, 59, 999999), 'Europe/Zurich')