Add UuidSerializer and improve serialization service and its test suite

hazelcast/core.py

@@ -293,6 +293,12 @@ def loads(self):
         """
         return json.loads(self._json_string)
 
+    def __eq__(self, other):
+        return isinstance(other, HazelcastJsonValue) and self._json_string == other._json_string
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
 
 class MemberVersion(object):
     __slots__ = ("major", "minor", "patch")

hazelcast/protocol/builtin.py

@@ -4,7 +4,7 @@
 from hazelcast.protocol.client_message import NULL_FRAME_BUF, BEGIN_FRAME_BUF, END_FRAME_BUF, \
     SIZE_OF_FRAME_LENGTH_AND_FLAGS, _IS_FINAL_FLAG, NULL_FINAL_FRAME_BUF, END_FINAL_FRAME_BUF
 from hazelcast.serialization import LONG_SIZE_IN_BYTES, UUID_SIZE_IN_BYTES, LE_INT, LE_LONG, BOOLEAN_SIZE_IN_BYTES, \
-    INT_SIZE_IN_BYTES, LE_ULONG, LE_UINT16, LE_INT8
+    INT_SIZE_IN_BYTES, LE_ULONG, LE_UINT16, LE_INT8, UUID_MSB_SHIFT, UUID_LSB_MASK
 from hazelcast.serialization.data import Data
 
 
@@ -199,10 +199,6 @@ def decode(msg):
         return result
 
 
-_UUID_MSB_SHIFT = 64
-_UUID_LSB_MASK = 0xFFFFFFFFFFFFFFFF
-
-
 class FixSizedTypesCodec(object):
     @staticmethod
     def encode_int(buf, offset, value):
@@ -247,8 +243,8 @@ def encode_uuid(buf, offset, value):
             return
 
         o = offset + BOOLEAN_SIZE_IN_BYTES
-        LE_ULONG.pack_into(buf, o, value.int >> _UUID_MSB_SHIFT)
-        LE_ULONG.pack_into(buf, o + LONG_SIZE_IN_BYTES, value.int & _UUID_LSB_MASK)
+        LE_ULONG.pack_into(buf, o, value.int >> UUID_MSB_SHIFT)
+        LE_ULONG.pack_into(buf, o + LONG_SIZE_IN_BYTES, value.int & UUID_LSB_MASK)
 
     @staticmethod
     def decode_uuid(buf, offset):

hazelcast/serialization/base.py

@@ -34,14 +34,6 @@ def index_for_default_type(type_id):
     return -type_id
 
 
-def is_dataserializable(obj):
-    return isinstance(obj, IdentifiedDataSerializable)
-
-
-def is_portable(obj):
-    return isinstance(obj, Portable)
-
-
 class BaseSerializationService(object):
     def __init__(self, version, global_partition_strategy, output_buffer_size, is_big_endian, int_type):
         self._registry = SerializerRegistry(int_type)
@@ -199,7 +191,8 @@ def serializer_for(self, obj):
 
         obj_type = type(obj)
 
-        # 2-Default serializers, Dataserializable, Portable, primitives, arrays, String and some helper types(BigInteger etc)
+        # 2-Default serializers, DataSerializable, Portable, primitives, arrays, String, UUID
+        # and some helper types(BigInteger etc)
         serializer = self.lookup_default_serializer(obj_type, obj)
 
         # 3-Custom registered types by user
@@ -219,15 +212,17 @@ def serializer_for(self, obj):
         return serializer
 
     def lookup_default_serializer(self, obj_type, obj):
-        if is_dataserializable(obj):
+        if isinstance(obj, IdentifiedDataSerializable):
             return self._data_serializer
-        if is_portable(obj):
+        if isinstance(obj, Portable):
             return self._portable_serializer
-        type_id = None
+
         if isinstance(obj, six.string_types):
-            type_id = CONSTANT_TYPE_STRING
+            return self.serializer_by_type_id(CONSTANT_TYPE_STRING)
+
+        type_id = None
         # LOCATE NUMERIC TYPES
-        elif obj_type in six.integer_types:
+        if obj_type in six.integer_types:
             if self.int_type == INTEGER_TYPE.BYTE:
                 type_id = CONSTANT_TYPE_BYTE
             elif self.int_type == INTEGER_TYPE.SHORT:
@@ -249,8 +244,10 @@ def lookup_default_serializer(self, obj_type, obj):
                     type_id = CONSTANT_TYPE_LONG
                 else:
                     type_id = JAVA_DEFAULT_TYPE_BIG_INTEGER
-        return self.serializer_by_type_id(type_id) if type_id is not None else self._constant_type_dict.get(obj_type,
-                                                                                                            None)
+            if type_id:
+                return self.serializer_by_type_id(type_id)
+
+        return self._constant_type_dict.get(obj_type, None)
 
     def lookup_custom_serializer(self, obj_type):
         serializer = self._type_dict.get(obj_type, None)

hazelcast/serialization/bits.py

@@ -39,29 +39,17 @@
 NULL_ARRAY_LENGTH = -1
 
 # LIMITS
-MAX_BYTE = 127
-MIN_BYTE = -128
+MAX_BYTE = 2 ** 7 - 1
+MIN_BYTE = -2 ** 7
 
-MAX_SHORT = 2 ** 16 - 1
+MAX_SHORT = 2 ** 15 - 1
 MIN_SHORT = -2 ** 15
 
-MAX_INT = 2 ** 32 - 1
+MAX_INT = 2 ** 31 - 1
 MIN_INT = -2 ** 31
 
-MAX_LONG = 2 ** 64 - 1
+MAX_LONG = 2 ** 63 - 1
 MIN_LONG = -2 ** 63
 
-MAX_FLOAT32 = 3.4028235e+38
-MIN_FLOAT32 = 1.4e-45
-
-
-def calculate_size_str(val):
-    return len(val) + INT_SIZE_IN_BYTES
-
-
-def calculate_size_data(val):
-    return len(val) + INT_SIZE_IN_BYTES
-
-
-def calculate_size_address(val):
-    return calculate_size_str(val.host) + INT_SIZE_IN_BYTES
+UUID_MSB_SHIFT = 64
+UUID_LSB_MASK = 0xFFFFFFFFFFFFFFFF

hazelcast/serialization/input.py

@@ -27,7 +27,7 @@ def read_into(self, buff, offset=None, length=None):
         _len = length if length is not None else len(buff)
         if _off < 0 or _len < 0 or (_off + _len) > len(self._buffer):
             raise IndexError()
-        elif length == 0:
+        elif _len == 0:
             return
         if self._pos > self._size:
             raise IndexError()

hazelcast/serialization/output.py

@@ -31,7 +31,7 @@ def write_from(self, buff, offset=None, length=None):
         elif length == 0:
             return
         self._ensure_available(_len)
-        self._buffer[self._pos: self._pos + _len] = buff[:]
+        self._buffer[self._pos: self._pos + _len] = buff
         self._pos += _len
 
     def write_boolean(self, boolean):

hazelcast/serialization/serialization_const.py

@@ -23,14 +23,14 @@
 CONSTANT_TYPE_FLOAT_ARRAY = -18
 CONSTANT_TYPE_DOUBLE_ARRAY = -19
 CONSTANT_TYPE_STRING_ARRAY = -20
+CONSTANT_TYPE_UUID = -21
 
 # ------------------------------------------------------------
 # DEFAULT SERIALIZERS
 
 JAVA_DEFAULT_TYPE_CLASS = -24
 JAVA_DEFAULT_TYPE_DATE = -25
 JAVA_DEFAULT_TYPE_BIG_INTEGER = -26
-JAVA_DEFAULT_TYPE_BIG_DECIMAL = -27
 JAVA_DEFAULT_TYPE_ARRAY_LIST = -29
 JAVA_DEFAULT_TYPE_LINKED_LIST = -30
 JAVASCRIPT_JSON_SERIALIZATION_TYPE = -130

hazelcast/serialization/serializer.py

@@ -1,6 +1,7 @@
 import binascii
+import time
+import uuid
 from datetime import datetime
-from time import time
 
 from hazelcast import six
 from hazelcast.core import HazelcastJsonValue
@@ -10,6 +11,8 @@
 from hazelcast.serialization.serialization_const import *
 from hazelcast.six.moves import range, cPickle
 
+from hazelcast.util import to_signed
+
 if not six.PY2:
     long = int
 
@@ -24,7 +27,8 @@ class NoneSerializer(BaseSerializer):
     def read(self, inp):
         return None
 
-    # "write(self, out, obj)" is never called so not implemented here
+    def write(self, out, obj):
+        pass
 
     def get_type_id(self):
         return CONSTANT_TYPE_NULL
@@ -78,10 +82,7 @@ def read(self, inp):
         return inp.read_int()
 
     def write(self, out, obj):
-        if obj.bit_length() < 32:
-            out.write_int(obj)
-        else:
-            raise ValueError("Serialization only supports 32 bit ints")
+        out.write_int(obj)
 
     def get_type_id(self):
         return CONSTANT_TYPE_INTEGER
@@ -92,10 +93,7 @@ def read(self, inp):
         return inp.read_long()
 
     def write(self, out, obj):
-        if obj.bit_length() < 64:
-            out.write_long(obj)
-        else:
-            raise ValueError("Serialization only supports 64 bit longs")
+        out.write_long(obj)
 
     def get_type_id(self):
         return CONSTANT_TYPE_LONG
@@ -105,8 +103,7 @@ class FloatSerializer(BaseSerializer):
     def read(self, inp):
         return inp.read_float()
 
-    def write(self, out, obj):
-        out.write_float(obj)
+    # "write(self, out, obj)" is never called so not implemented here
 
     def get_type_id(self):
         return CONSTANT_TYPE_FLOAT
@@ -134,6 +131,23 @@ def get_type_id(self):
         return CONSTANT_TYPE_STRING
 
 
+class UuidSerializer(BaseSerializer):
+    def read(self, inp):
+        buf = bytearray(16)
+        inp.read_into(buf)
+        return uuid.UUID(bytes=bytes(buf))
+
+    def write(self, out, obj):
+        i = obj.int
+        msb = to_signed(i >> UUID_MSB_SHIFT, 64)
+        lsb = to_signed(i & UUID_LSB_MASK, 64)
+        out.write_long(msb)
+        out.write_long(lsb)
+
+    def get_type_id(self):
+        return CONSTANT_TYPE_UUID
+
+
 class HazelcastJsonValueSerializer(BaseSerializer):
     def read(self, inp):
         return HazelcastJsonValue(inp.read_utf())
@@ -160,7 +174,8 @@ class ByteArraySerializer(BaseSerializer):
     def read(self, inp):
         return inp.read_byte_array()
 
-    # "write(self, out, obj)" is never called so not implemented here
+    def write(self, out, obj):
+        out.write_byte_array(obj)
 
     def get_type_id(self):
         return CONSTANT_TYPE_BYTE_ARRAY
@@ -243,7 +258,7 @@ def read(self, inp):
         return datetime.fromtimestamp(long_time / 1000.0)
 
     def write(self, out, obj):
-        long_time = long(time.mktime(obj.timetuple()))
+        long_time = long(time.mktime(obj.timetuple())) * 1000
         out.write_long(long_time)
 
     def get_type_id(self):
@@ -266,11 +281,11 @@ def read(self, inp):
         return int(binascii.hexlify(result), 16)
 
     def write(self, out, obj):
-        the_big_int = -obj-1 if obj < 0 else obj
+        the_big_int = -obj - 1 if obj < 0 else obj
         end_index = -1 if (type(obj) == long and six.PY2) else None
         hex_str = hex(the_big_int)[2:end_index]
         if len(hex_str) % 2 == 1:
-            prefix = '0' # "f" if obj < 0 else "0"
+            prefix = '0'  # "f" if obj < 0 else "0"
             hex_str = prefix + hex_str
         num_array = bytearray(binascii.unhexlify(bytearray(hex_str, encoding="utf-8")))
         if obj < 0:
@@ -284,23 +299,11 @@ def get_type_id(self):
         return JAVA_DEFAULT_TYPE_BIG_INTEGER
 
 
-class BigDecimalSerializer(BaseSerializer):
-    def read(self, inp):
-        raise NotImplementedError("Big decimal numbers not supported")
-
-    def write(self, out, obj):
-        raise NotImplementedError("Big decimal numbers not supported")
-
-    def get_type_id(self):
-        return JAVA_DEFAULT_TYPE_BIG_DECIMAL
-
-
 class JavaClassSerializer(BaseSerializer):
     def read(self, inp):
         return inp.read_utf()
 
-    def write(self, out, obj):
-        out.write_utf(obj)
+    # "write(self, out, obj)" is never called so not implemented here
 
     def get_type_id(self):
         return JAVA_DEFAULT_TYPE_CLASS
@@ -330,8 +333,7 @@ def read(self, inp):
             return [inp.read_object() for _ in range(0, size)]
         return None
 
-    def write(self, out, obj):
-        raise NotImplementedError("writing Link lists not supported")
+    # "write(self, out, obj)" is never called so not implemented here
 
     def get_type_id(self):
         return JAVA_DEFAULT_TYPE_LINKED_LIST
@@ -368,11 +370,13 @@ def read(self, inp):
 
         factory = self._factories.get(factory_id, None)
         if factory is None:
-            raise HazelcastSerializationError("No DataSerializerFactory registered for namespace: {}".format(factory_id))
+            raise HazelcastSerializationError(
+                "No DataSerializerFactory registered for namespace: {}".format(factory_id))
         identified = factory.get(class_id, None)
         if identified is None:
             raise HazelcastSerializationError(
-                    "{} is not be able to create an instance for id: {} on factoryId: {}".format(factory, class_id, factory_id))
+                "{} is not be able to create an instance for id: {} on factoryId: {}".format(factory, class_id,
+                                                                                             factory_id))
         instance = identified()
         instance.read_data(inp)
         return instance

hazelcast/serialization/service.py

@@ -1,3 +1,5 @@
+import uuid
+
 from hazelcast.serialization.base import BaseSerializationService
 from hazelcast.serialization.portable.classdef import FieldType
 from hazelcast.serialization.portable.context import PortableContext
@@ -52,9 +54,10 @@ def _register_constant_serializers(self):
         self._registry.register_constant_serializer(LongSerializer())
         self._registry.register_constant_serializer(FloatSerializer())
         self._registry.register_constant_serializer(DoubleSerializer(), float)
+        self._registry.register_constant_serializer(UuidSerializer(), uuid.UUID)
         self._registry.register_constant_serializer(StringSerializer())
         # Arrays of primitives and String
-        self._registry.register_constant_serializer(ByteArraySerializer())
+        self._registry.register_constant_serializer(ByteArraySerializer(), bytearray)
         self._registry.register_constant_serializer(BooleanArraySerializer())
         self._registry.register_constant_serializer(CharArraySerializer())
         self._registry.register_constant_serializer(ShortArraySerializer())
@@ -66,7 +69,6 @@ def _register_constant_serializers(self):
         # EXTENSIONS
         self._registry.register_constant_serializer(DateTimeSerializer(), datetime)
         self._registry.register_constant_serializer(BigIntegerSerializer())
-        self._registry.register_constant_serializer(BigDecimalSerializer())
         self._registry.register_constant_serializer(JavaClassSerializer())
         self._registry.register_constant_serializer(ArrayListSerializer(), list)
         self._registry.register_constant_serializer(LinkedListSerializer())

hazelcast/util.py

@@ -320,3 +320,10 @@ def create_git_info():
 
 def to_list(*args, **kwargs):
     return list(*args, **kwargs)
+
+
+def to_signed(unsigned, bit_len):
+    mask = (1 << bit_len) - 1
+    if unsigned & (1 << (bit_len - 1)):
+        return unsigned | ~mask
+    return unsigned & mask