adding UTCDateTimeField, which always returns datetimes with timezone UTC

server/pulp/server/async/scheduler.py

@@ -11,6 +11,7 @@
 
 from pulp.common.constants import (CELERYBEAT_WAIT_SECONDS, RESOURCE_MANAGER_WORKER_NAME,
                                    SCHEDULER_WORKER_NAME, TICK_SECONDS)
+from pulp.common.dateutils import ensure_tz
 from pulp.server.async import worker_watcher
 from pulp.server.async.celery_instance import celery as app
 from pulp.server.async.tasks import _delete_worker
@@ -123,7 +124,8 @@ def check_celery_processes(self):
         msg = _('Checking if pulp_workers, pulp_celerybeat, or pulp_resource_manager '
                 'processes are missing for more than %d seconds') % self.CELERY_TIMEOUT_SECONDS
         _logger.debug(msg)
-        oldest_heartbeat_time = datetime.utcnow() - timedelta(seconds=self.CELERY_TIMEOUT_SECONDS)
+        now = ensure_tz(datetime.utcnow())
+        oldest_heartbeat_time = now - timedelta(seconds=self.CELERY_TIMEOUT_SECONDS)
         worker_list = Worker.objects.all()
         worker_count = 0
         resource_manager_count = 0

server/pulp/server/db/fields.py

@@ -6,7 +6,7 @@
 """
 
 from isodate import ISO8601Error
-from mongoengine import StringField
+from mongoengine import StringField, DateTimeField
 
 from pulp.common import dateutils
 
@@ -25,3 +25,26 @@ def validate(self, value):
             dateutils.parse_iso8601_datetime(value)
         except ISO8601Error, e:
             self.error(str(e))
+
+
+class UTCDateTimeField(DateTimeField):
+    """
+    Any datetime value retrived from this field will have its timezone set to UTC. It is otherwise
+    identical to the mongoengine DateTimeField. It can replace the mongoengine DateTimeField without
+    need for a database migration.
+    """
+
+    def to_python(self, value):
+        """
+        Ensures that the datetime object returned has timezone UTC set. This assumes that if the
+        value lacks a timezone, the data is already UTC, and the corresponding timezone object
+        will be added.
+
+        :param value: a datetime object
+        :type  value: datetime.datetime
+
+        :return: an equivalent datetime object with the timezone set to UTC
+        :rtype:  datetime.datetime
+        """
+        ret = super(UTCDateTimeField, self).to_python(value)
+        return dateutils.ensure_tz(ret)

server/pulp/server/db/model/__init__.py

@@ -7,7 +7,7 @@
 from collections import namedtuple
 from hmac import HMAC
 
-from mongoengine import (DateTimeField, DictField, Document, DynamicField, IntField,
+from mongoengine import (DictField, Document, DynamicField, IntField,
                          ListField, StringField, UUIDField, ValidationError)
 from mongoengine import signals
 
@@ -20,7 +20,7 @@
 from pulp.server.async.emit import send as send_taskstatus_message
 from pulp.server.db.connection import UnsafeRetry
 from pulp.server.compat import digestmod
-from pulp.server.db.fields import ISO8601StringField
+from pulp.server.db.fields import ISO8601StringField, UTCDateTimeField
 from pulp.server.db.model.reaper_base import ReaperMixin
 from pulp.server.db.querysets import CriteriaQuerySet, RepoQuerySet
 from pulp.server.util import Singleton
@@ -95,9 +95,9 @@ class Repository(AutoRetryDocument):
                       across multiple operations.
     :type scratchpad: mongoengine.DictField
     :ivar last_unit_added: Datetime of the most recent occurence of adding a unit to the repo
-    :type last_unit_added: mongoengine.DateTimeField
+    :type last_unit_added: UTCDateTimeField
     :ivar last_unit_removed: Datetime of the most recent occurence of removing a unit from the repo
-    :type last_unit_removed: mongoengine.DateTimeField
+    :type last_unit_removed: UTCDateTimeField
     :ivar _ns: (Deprecated) Namespace of repo, included for backwards compatibility.
     :type _is: mongoengine.StringField
     """
@@ -111,8 +111,8 @@ class Repository(AutoRetryDocument):
     notes = DictField()
     scratchpad = DictField(default={})
     content_unit_counts = DictField(default={})
-    last_unit_added = DateTimeField()
-    last_unit_removed = DateTimeField()
+    last_unit_added = UTCDateTimeField()
+    last_unit_removed = UTCDateTimeField()
 
     # For backward compatibility
     _ns = StringField(default='repos')
@@ -269,10 +269,10 @@ class Worker(AutoRetryDocument):
     :ivar name:    worker name, in the form of "worker_type@hostname"
     :type name:    mongoengine.StringField
     :ivar last_heartbeat:  A timestamp of the last heartbeat from the Worker
-    :type last_heartbeat:  mongoengine.DateTimeField
+    :type last_heartbeat:  UTCDateTimeField
     """
     name = StringField(primary_key=True)
-    last_heartbeat = DateTimeField()
+    last_heartbeat = UTCDateTimeField()
 
     # For backward compatibility
     _ns = StringField(default='workers')
@@ -739,7 +739,7 @@ class CeleryBeatLock(AutoRetryDocument):
     :type _ns: mongoengine.StringField
     """
     celerybeat_name = StringField(required=True)
-    timestamp = DateTimeField(required=True)
+    timestamp = UTCDateTimeField(required=True)
     lock = StringField(required=True, default="locked", unique=True)
 
     # For backward compatibility

server/test/unit/server/db/test_fields.py

@@ -1,7 +1,7 @@
 import unittest
-from datetime import datetime
+from datetime import datetime, timedelta, tzinfo
 
-from mongoengine import ValidationError
+import mongoengine
 
 from pulp.common import dateutils
 from pulp.server.db import fields
@@ -16,4 +16,48 @@ def test_iso8601_string_field(self):
 
         invalid_values = ['date', {}, [], 1, datetime.now()]
         for invalid in invalid_values:
-            self.assertRaises(ValidationError, iso8601_field.validate, invalid)
+            self.assertRaises(mongoengine.ValidationError, iso8601_field.validate, invalid)
+
+
+class TestUTCDateTimeField(unittest.TestCase):
+    class TestModel(mongoengine.Document):
+        timestamp = fields.UTCDateTimeField()
+
+    class EST(tzinfo):
+        def utcoffset(self, dt):
+            return timedelta(hours=-5)
+
+        def tzname(self, dt):
+            return 'EST'
+
+        def dst(self, dt):
+            return timedelta(0)
+
+    def test_timezone_aware(self):
+        """
+        quoting docs [0]: "A datetime object d is aware if d.tzinfo is not None and
+        d.tzinfo.utcoffset(d) does not return None."
+
+        [0] https://docs.python.org/2/library/datetime.html#datetime.tzinfo
+        """
+        now = datetime.utcnow()
+        self.assertTrue(now.tzinfo is None)
+        model = self.TestModel(timestamp=now)
+
+        self.assertTrue(model.timestamp.tzinfo is not None)
+        self.assertTrue(model.timestamp.tzinfo.utcoffset(model.timestamp) is not None)
+
+    def test_timezone_is_utc(self):
+        now = datetime.utcnow()
+        self.assertTrue(now.tzinfo is None)
+        model = self.TestModel(timestamp=now)
+
+        self.assertEqual(model.timestamp.tzinfo.utcoffset(model.timestamp), timedelta(0))
+
+    def test_convert_other_timezone(self):
+        timestamp = datetime(year=2015, month=11, day=23, hour=13, minute=52, tzinfo=self.EST())
+        model = self.TestModel(timestamp=timestamp)
+
+        self.assertEqual(model.timestamp.hour, 18)
+        self.assertEqual(model.timestamp.tzinfo.utcoffset(model.timestamp), timedelta(0))
+        self.assertEqual(timestamp, model.timestamp)