Permalink
Browse files

Use millis since epoch for DateType, compatible custom Types

  • Loading branch information...
1 parent e7d24f8 commit 307b2da007b6fa2f1597f7b4178b0000a8550bfd @thobbs thobbs committed Mar 2, 2012
Showing with 123 additions and 5 deletions.
  1. +7 −5 pycassa/marshal.py
  2. +91 −0 pycassa/types.py
  3. +25 −0 tests/test_autopacking.py
View
@@ -65,14 +65,14 @@ def _to_timestamp(v):
# Expects Value to be either date or datetime
try:
converted = time.mktime(v.timetuple())
- converted = converted * 1e6 + getattr(v, 'microsecond', 0)
+ converted = converted * 1e3 + getattr(v, 'microsecond', 0)/1e3
except AttributeError:
# Ints and floats are valid timestamps too
if type(v) not in _number_types:
raise TypeError('DateType arguments must be a datetime or timestamp')
- converted = v * 1e6
- return converted
+ converted = v * 1e3
+ return long(converted)
def get_composite_packer(typestr):
packers = map(packer_for, _get_inner_types(typestr))
@@ -242,9 +242,11 @@ def unpacker_for(typestr):
elif data_type == 'DateType':
if _have_struct:
- return lambda v: datetime.fromtimestamp(_long_packer.unpack(v)[0] / 1e6)
+ return lambda v: datetime.fromtimestamp(
+ _long_packer.unpack(v)[0] / 1e3)
else:
- return lambda v: datetime.fromtimestamp(struct.unpack('>q', v)[0] / 1e6)
+ return lambda v: datetime.fromtimestamp(
+ struct.unpack('>q', v)[0] / 1e3)
elif data_type == 'BooleanType':
if _have_struct:
View
@@ -21,6 +21,10 @@
"""
+import time
+import struct
+from datetime import datetime
+
import pycassa.marshal as marshal
class CassandraType(object):
@@ -111,6 +115,93 @@ class DateType(CassandraType):
"""
pass
+def _to_timestamp(v, use_micros=False):
+ # Expects Value to be either date or datetime
+ scale = 1e6 if use_micros else 1e3
+ micro_scale = 1.0 if use_micros else 1e3
+ try:
+ converted = time.mktime(v.timetuple())
+ converted = (converted * scale) + \
+ (getattr(v, 'microsecond', 0) / micro_scale)
+ except AttributeError:
+ # Ints and floats are valid timestamps too
+ if type(v) not in marshal._number_types:
+ raise TypeError('DateType arguments must be a datetime or timestamp')
+
+ converted = v * scale
+ return long(converted)
+
+class OldPycassaDateType(CassandraType):
+ """
+ This class can only read and write the DateType format
+ used by pycassa versions 1.2.0 to 1.5.0.
+
+ This formats store the number of microseconds since the
+ unix epoch, rather than the number of milliseconds, which
+ is what cassandra-cli and other clients supporting DateType
+ use.
+ """
+
+ @staticmethod
+ def pack(v, *args, **kwargs):
+ ts = _to_timestamp(v, use_micros=True)
+ print "packing (old): %10.10f" % ts
+ if marshal._have_struct:
+ return marshal._long_packer.pack(ts)
+ else:
+ return struct.pack('>q', ts)
+
+ @staticmethod
+ def unpack(v):
+ if marshal._have_struct:
+ ts = marshal._long_packer.unpack(v)[0] / 1e6
+ else:
+ ts = struct.unpack('>q', v)[0] / 1e6
+ print "unpacking (old): %10.10f" % ts
+ return datetime.fromtimestamp(ts)
+
+class IntermediateDateType(CassandraType):
+ """
+ This class is capable of reading either the DateType
+ format by pycassa versions 1.2.0 to 1.5.0 or the correct
+ format used in pycassa 1.5.1+. It will only write the
+ new, correct format.
+
+ This type is a good choice when you are using DateType
+ as the validator for non-indexed column values and you are
+ in the process of converting from thee old format to
+ the new format.
+
+ It almost certainly *should not be used* for row keys,
+ column names (if you care about the sorting), or column
+ values that have a secondary index on them.
+ """
+
+ @staticmethod
+ def pack(v, *args, **kwargs):
+ ts = _to_timestamp(v, use_micros=False)
+ print "packing (intermediate): %10.10f" % ts
+ if marshal._have_struct:
+ return marshal._long_packer.pack(ts)
+ else:
+ return struct.pack('>q', ts)
+
+ @staticmethod
+ def unpack(v):
+ if marshal._have_struct:
+ raw_ts = marshal._long_packer.unpack(v)[0] / 1e3
+ else:
+ raw_ts = struct.unpack('>q', v)[0] / 1e3
+
+ print "raw: %10.10f" % raw_ts
+ try:
+ return datetime.fromtimestamp(raw_ts)
+ except ValueError:
+ # convert from bad microsecond format to millis
+ corrected_ts = raw_ts / 1e3
+ print "corrected: %10.10f" % corrected_ts
+ return datetime.fromtimestamp(corrected_ts)
+
class CompositeType(CassandraType):
"""
A type composed of one or more components, each of
View
@@ -982,6 +982,31 @@ def test_packing_disabled(self):
assert_raises(TypeError, self.cf.insert, args=('key', {123: 123}))
self.cf.remove('key')
+class TestDateTypes(unittest.TestCase):
+
+ def _compare_dates(self, d1, d2):
+ self.assertEquals(d1.timetuple(), d2.timetuple())
+ self.assertEquals(int(d1.microsecond/1e3), int(d2.microsecond/1e3))
+
+ def test_compatibility(self):
+ self.cf = ColumnFamily(pool, 'Standard1')
+ self.cf.column_validators['date'] = OldPycassaDateType()
+
+ d = datetime.now()
+ self.cf.insert('key1', {'date': d})
+ self._compare_dates(self.cf.get('key1')['date'], d)
+
+ self.cf.column_validators['date'] = IntermediateDateType()
+ self._compare_dates(self.cf.get('key1')['date'], d)
+ self.cf.insert('key1', {'date': d})
+ self._compare_dates(self.cf.get('key1')['date'], d)
+
+ self.cf.column_validators['date'] = DateType()
+ self._compare_dates(self.cf.get('key1')['date'], d)
+ self.cf.insert('key1', {'date': d})
+ self._compare_dates(self.cf.get('key1')['date'], d)
+ self.cf.remove('key1')
+
class TestCustomTypes(unittest.TestCase):
class IntString(types.CassandraType):

0 comments on commit 307b2da

Please sign in to comment.