From f037d55cf45238db20eda367095845f300ebe018 Mon Sep 17 00:00:00 2001 From: mdumandag Date: Tue, 8 Sep 2020 17:55:03 +0300 Subject: [PATCH 1/2] add uuid serializer --- hazelcast/protocol/builtin.py | 10 ++---- hazelcast/serialization/base.py | 29 +++++++-------- hazelcast/serialization/bits.py | 13 ++----- hazelcast/serialization/input.py | 2 +- hazelcast/serialization/output.py | 2 +- .../serialization/serialization_const.py | 1 + hazelcast/serialization/serializer.py | 36 +++++++++++++------ hazelcast/serialization/service.py | 5 ++- hazelcast/util.py | 7 ++++ 9 files changed, 58 insertions(+), 47 deletions(-) diff --git a/hazelcast/protocol/builtin.py b/hazelcast/protocol/builtin.py index a70d7bd1a6..5feaa384a7 100644 --- a/hazelcast/protocol/builtin.py +++ b/hazelcast/protocol/builtin.py @@ -4,7 +4,7 @@ from hazelcast.protocol.client_message import NULL_FRAME_BUF, BEGIN_FRAME_BUF, END_FRAME_BUF, \ SIZE_OF_FRAME_LENGTH_AND_FLAGS, _IS_FINAL_FLAG, NULL_FINAL_FRAME_BUF, END_FINAL_FRAME_BUF from hazelcast.serialization import LONG_SIZE_IN_BYTES, UUID_SIZE_IN_BYTES, LE_INT, LE_LONG, BOOLEAN_SIZE_IN_BYTES, \ - INT_SIZE_IN_BYTES, LE_ULONG, LE_UINT16, LE_INT8 + INT_SIZE_IN_BYTES, LE_ULONG, LE_UINT16, LE_INT8, UUID_MSB_SHIFT, UUID_LSB_MASK from hazelcast.serialization.data import Data @@ -199,10 +199,6 @@ def decode(msg): return result -_UUID_MSB_SHIFT = 64 -_UUID_LSB_MASK = 0xFFFFFFFFFFFFFFFF - - class FixSizedTypesCodec(object): @staticmethod def encode_int(buf, offset, value): @@ -247,8 +243,8 @@ def encode_uuid(buf, offset, value): return o = offset + BOOLEAN_SIZE_IN_BYTES - LE_ULONG.pack_into(buf, o, value.int >> _UUID_MSB_SHIFT) - LE_ULONG.pack_into(buf, o + LONG_SIZE_IN_BYTES, value.int & _UUID_LSB_MASK) + LE_ULONG.pack_into(buf, o, value.int >> UUID_MSB_SHIFT) + LE_ULONG.pack_into(buf, o + LONG_SIZE_IN_BYTES, value.int & UUID_LSB_MASK) @staticmethod def decode_uuid(buf, offset): diff --git a/hazelcast/serialization/base.py b/hazelcast/serialization/base.py index 3f5ff23218..92c62170a7 100644 --- a/hazelcast/serialization/base.py +++ b/hazelcast/serialization/base.py @@ -34,14 +34,6 @@ def index_for_default_type(type_id): return -type_id -def is_dataserializable(obj): - return isinstance(obj, IdentifiedDataSerializable) - - -def is_portable(obj): - return isinstance(obj, Portable) - - class BaseSerializationService(object): def __init__(self, version, global_partition_strategy, output_buffer_size, is_big_endian, int_type): self._registry = SerializerRegistry(int_type) @@ -199,7 +191,8 @@ def serializer_for(self, obj): obj_type = type(obj) - # 2-Default serializers, Dataserializable, Portable, primitives, arrays, String and some helper types(BigInteger etc) + # 2-Default serializers, DataSerializable, Portable, primitives, arrays, String, UUID + # and some helper types(BigInteger etc) serializer = self.lookup_default_serializer(obj_type, obj) # 3-Custom registered types by user @@ -219,15 +212,17 @@ def serializer_for(self, obj): return serializer def lookup_default_serializer(self, obj_type, obj): - if is_dataserializable(obj): + if isinstance(obj, IdentifiedDataSerializable): return self._data_serializer - if is_portable(obj): + if isinstance(obj, Portable): return self._portable_serializer - type_id = None + if isinstance(obj, six.string_types): - type_id = CONSTANT_TYPE_STRING + return self.serializer_by_type_id(CONSTANT_TYPE_STRING) + + type_id = None # LOCATE NUMERIC TYPES - elif obj_type in six.integer_types: + if obj_type in six.integer_types: if self.int_type == INTEGER_TYPE.BYTE: type_id = CONSTANT_TYPE_BYTE elif self.int_type == INTEGER_TYPE.SHORT: @@ -249,8 +244,10 @@ def lookup_default_serializer(self, obj_type, obj): type_id = CONSTANT_TYPE_LONG else: type_id = JAVA_DEFAULT_TYPE_BIG_INTEGER - return self.serializer_by_type_id(type_id) if type_id is not None else self._constant_type_dict.get(obj_type, - None) + if type_id: + return self.serializer_by_type_id(type_id) + + return self._constant_type_dict.get(obj_type, None) def lookup_custom_serializer(self, obj_type): serializer = self._type_dict.get(obj_type, None) diff --git a/hazelcast/serialization/bits.py b/hazelcast/serialization/bits.py index e0cae0c017..a5d910c5ed 100644 --- a/hazelcast/serialization/bits.py +++ b/hazelcast/serialization/bits.py @@ -54,14 +54,5 @@ MAX_FLOAT32 = 3.4028235e+38 MIN_FLOAT32 = 1.4e-45 - -def calculate_size_str(val): - return len(val) + INT_SIZE_IN_BYTES - - -def calculate_size_data(val): - return len(val) + INT_SIZE_IN_BYTES - - -def calculate_size_address(val): - return calculate_size_str(val.host) + INT_SIZE_IN_BYTES \ No newline at end of file +UUID_MSB_SHIFT = 64 +UUID_LSB_MASK = 0xFFFFFFFFFFFFFFFF diff --git a/hazelcast/serialization/input.py b/hazelcast/serialization/input.py index 785390e3a3..133139f1f1 100644 --- a/hazelcast/serialization/input.py +++ b/hazelcast/serialization/input.py @@ -27,7 +27,7 @@ def read_into(self, buff, offset=None, length=None): _len = length if length is not None else len(buff) if _off < 0 or _len < 0 or (_off + _len) > len(self._buffer): raise IndexError() - elif length == 0: + elif _len == 0: return if self._pos > self._size: raise IndexError() diff --git a/hazelcast/serialization/output.py b/hazelcast/serialization/output.py index b7e5d71c06..a8f4c79c63 100644 --- a/hazelcast/serialization/output.py +++ b/hazelcast/serialization/output.py @@ -31,7 +31,7 @@ def write_from(self, buff, offset=None, length=None): elif length == 0: return self._ensure_available(_len) - self._buffer[self._pos: self._pos + _len] = buff[:] + self._buffer[self._pos: self._pos + _len] = buff self._pos += _len def write_boolean(self, boolean): diff --git a/hazelcast/serialization/serialization_const.py b/hazelcast/serialization/serialization_const.py index 5e342e2f3c..73cdfae182 100644 --- a/hazelcast/serialization/serialization_const.py +++ b/hazelcast/serialization/serialization_const.py @@ -23,6 +23,7 @@ CONSTANT_TYPE_FLOAT_ARRAY = -18 CONSTANT_TYPE_DOUBLE_ARRAY = -19 CONSTANT_TYPE_STRING_ARRAY = -20 +CONSTANT_TYPE_UUID = -21 # ------------------------------------------------------------ # DEFAULT SERIALIZERS diff --git a/hazelcast/serialization/serializer.py b/hazelcast/serialization/serializer.py index 5f4ea938cc..ede51bf9bc 100644 --- a/hazelcast/serialization/serializer.py +++ b/hazelcast/serialization/serializer.py @@ -1,4 +1,5 @@ import binascii +import uuid from datetime import datetime from time import time @@ -10,6 +11,8 @@ from hazelcast.serialization.serialization_const import * from hazelcast.six.moves import range, cPickle +from hazelcast.util import to_signed + if not six.PY2: long = int @@ -24,7 +27,8 @@ class NoneSerializer(BaseSerializer): def read(self, inp): return None - # "write(self, out, obj)" is never called so not implemented here + def write(self, out, obj): + pass def get_type_id(self): return CONSTANT_TYPE_NULL @@ -78,10 +82,7 @@ def read(self, inp): return inp.read_int() def write(self, out, obj): - if obj.bit_length() < 32: - out.write_int(obj) - else: - raise ValueError("Serialization only supports 32 bit ints") + out.write_int(obj) def get_type_id(self): return CONSTANT_TYPE_INTEGER @@ -92,10 +93,7 @@ def read(self, inp): return inp.read_long() def write(self, out, obj): - if obj.bit_length() < 64: - out.write_long(obj) - else: - raise ValueError("Serialization only supports 64 bit longs") + out.write_long(obj) def get_type_id(self): return CONSTANT_TYPE_LONG @@ -134,6 +132,23 @@ def get_type_id(self): return CONSTANT_TYPE_STRING +class UuidSerializer(BaseSerializer): + def read(self, inp): + buf = bytearray(16) + inp.read_into(buf) + return uuid.UUID(bytes=bytes(buf)) + + def write(self, out, obj): + i = obj.int + msb = to_signed(i >> UUID_MSB_SHIFT, 64) + lsb = to_signed(i & UUID_LSB_MASK, 64) + out.write_long(msb) + out.write_long(lsb) + + def get_type_id(self): + return CONSTANT_TYPE_UUID + + class HazelcastJsonValueSerializer(BaseSerializer): def read(self, inp): return HazelcastJsonValue(inp.read_utf()) @@ -160,7 +175,8 @@ class ByteArraySerializer(BaseSerializer): def read(self, inp): return inp.read_byte_array() - # "write(self, out, obj)" is never called so not implemented here + def write(self, out, obj): + out.write_byte_array(obj) def get_type_id(self): return CONSTANT_TYPE_BYTE_ARRAY diff --git a/hazelcast/serialization/service.py b/hazelcast/serialization/service.py index 11cd7ed74f..d80cca615c 100644 --- a/hazelcast/serialization/service.py +++ b/hazelcast/serialization/service.py @@ -1,3 +1,5 @@ +import uuid + from hazelcast.serialization.base import BaseSerializationService from hazelcast.serialization.portable.classdef import FieldType from hazelcast.serialization.portable.context import PortableContext @@ -52,9 +54,10 @@ def _register_constant_serializers(self): self._registry.register_constant_serializer(LongSerializer()) self._registry.register_constant_serializer(FloatSerializer()) self._registry.register_constant_serializer(DoubleSerializer(), float) + self._registry.register_constant_serializer(UuidSerializer(), uuid.UUID) self._registry.register_constant_serializer(StringSerializer()) # Arrays of primitives and String - self._registry.register_constant_serializer(ByteArraySerializer()) + self._registry.register_constant_serializer(ByteArraySerializer(), bytearray) self._registry.register_constant_serializer(BooleanArraySerializer()) self._registry.register_constant_serializer(CharArraySerializer()) self._registry.register_constant_serializer(ShortArraySerializer()) diff --git a/hazelcast/util.py b/hazelcast/util.py index 0572c53f50..ce108bf9d8 100644 --- a/hazelcast/util.py +++ b/hazelcast/util.py @@ -320,3 +320,10 @@ def create_git_info(): def to_list(*args, **kwargs): return list(*args, **kwargs) + + +def to_signed(unsigned, bit_len): + mask = (1 << bit_len) - 1 + if unsigned & (1 << (bit_len - 1)): + return unsigned | ~mask + return unsigned & mask From af259beebd9771bbc57e7abe32a7cddffb8f7da0 Mon Sep 17 00:00:00 2001 From: mdumandag Date: Wed, 9 Sep 2020 18:05:06 +0300 Subject: [PATCH 2/2] improve serialization tests --- hazelcast/core.py | 6 + hazelcast/serialization/bits.py | 13 +- .../serialization/serialization_const.py | 1 - hazelcast/serialization/serializer.py | 34 +- hazelcast/serialization/service.py | 1 - tests/serialization/serializers_test.py | 500 ++++++++++++++---- 6 files changed, 432 insertions(+), 123 deletions(-) diff --git a/hazelcast/core.py b/hazelcast/core.py index f01c23723c..7bc6eb9dcd 100644 --- a/hazelcast/core.py +++ b/hazelcast/core.py @@ -293,6 +293,12 @@ def loads(self): """ return json.loads(self._json_string) + def __eq__(self, other): + return isinstance(other, HazelcastJsonValue) and self._json_string == other._json_string + + def __ne__(self, other): + return not self.__eq__(other) + class MemberVersion(object): __slots__ = ("major", "minor", "patch") diff --git a/hazelcast/serialization/bits.py b/hazelcast/serialization/bits.py index a5d910c5ed..23cfd5a506 100644 --- a/hazelcast/serialization/bits.py +++ b/hazelcast/serialization/bits.py @@ -39,20 +39,17 @@ NULL_ARRAY_LENGTH = -1 # LIMITS -MAX_BYTE = 127 -MIN_BYTE = -128 +MAX_BYTE = 2 ** 7 - 1 +MIN_BYTE = -2 ** 7 -MAX_SHORT = 2 ** 16 - 1 +MAX_SHORT = 2 ** 15 - 1 MIN_SHORT = -2 ** 15 -MAX_INT = 2 ** 32 - 1 +MAX_INT = 2 ** 31 - 1 MIN_INT = -2 ** 31 -MAX_LONG = 2 ** 64 - 1 +MAX_LONG = 2 ** 63 - 1 MIN_LONG = -2 ** 63 -MAX_FLOAT32 = 3.4028235e+38 -MIN_FLOAT32 = 1.4e-45 - UUID_MSB_SHIFT = 64 UUID_LSB_MASK = 0xFFFFFFFFFFFFFFFF diff --git a/hazelcast/serialization/serialization_const.py b/hazelcast/serialization/serialization_const.py index 73cdfae182..6fbca9f14b 100644 --- a/hazelcast/serialization/serialization_const.py +++ b/hazelcast/serialization/serialization_const.py @@ -31,7 +31,6 @@ JAVA_DEFAULT_TYPE_CLASS = -24 JAVA_DEFAULT_TYPE_DATE = -25 JAVA_DEFAULT_TYPE_BIG_INTEGER = -26 -JAVA_DEFAULT_TYPE_BIG_DECIMAL = -27 JAVA_DEFAULT_TYPE_ARRAY_LIST = -29 JAVA_DEFAULT_TYPE_LINKED_LIST = -30 JAVASCRIPT_JSON_SERIALIZATION_TYPE = -130 diff --git a/hazelcast/serialization/serializer.py b/hazelcast/serialization/serializer.py index ede51bf9bc..d8cac6e579 100644 --- a/hazelcast/serialization/serializer.py +++ b/hazelcast/serialization/serializer.py @@ -1,7 +1,7 @@ import binascii +import time import uuid from datetime import datetime -from time import time from hazelcast import six from hazelcast.core import HazelcastJsonValue @@ -103,8 +103,7 @@ class FloatSerializer(BaseSerializer): def read(self, inp): return inp.read_float() - def write(self, out, obj): - out.write_float(obj) + # "write(self, out, obj)" is never called so not implemented here def get_type_id(self): return CONSTANT_TYPE_FLOAT @@ -259,7 +258,7 @@ def read(self, inp): return datetime.fromtimestamp(long_time / 1000.0) def write(self, out, obj): - long_time = long(time.mktime(obj.timetuple())) + long_time = long(time.mktime(obj.timetuple())) * 1000 out.write_long(long_time) def get_type_id(self): @@ -282,11 +281,11 @@ def read(self, inp): return int(binascii.hexlify(result), 16) def write(self, out, obj): - the_big_int = -obj-1 if obj < 0 else obj + the_big_int = -obj - 1 if obj < 0 else obj end_index = -1 if (type(obj) == long and six.PY2) else None hex_str = hex(the_big_int)[2:end_index] if len(hex_str) % 2 == 1: - prefix = '0' # "f" if obj < 0 else "0" + prefix = '0' # "f" if obj < 0 else "0" hex_str = prefix + hex_str num_array = bytearray(binascii.unhexlify(bytearray(hex_str, encoding="utf-8"))) if obj < 0: @@ -300,23 +299,11 @@ def get_type_id(self): return JAVA_DEFAULT_TYPE_BIG_INTEGER -class BigDecimalSerializer(BaseSerializer): - def read(self, inp): - raise NotImplementedError("Big decimal numbers not supported") - - def write(self, out, obj): - raise NotImplementedError("Big decimal numbers not supported") - - def get_type_id(self): - return JAVA_DEFAULT_TYPE_BIG_DECIMAL - - class JavaClassSerializer(BaseSerializer): def read(self, inp): return inp.read_utf() - def write(self, out, obj): - out.write_utf(obj) + # "write(self, out, obj)" is never called so not implemented here def get_type_id(self): return JAVA_DEFAULT_TYPE_CLASS @@ -346,8 +333,7 @@ def read(self, inp): return [inp.read_object() for _ in range(0, size)] return None - def write(self, out, obj): - raise NotImplementedError("writing Link lists not supported") + # "write(self, out, obj)" is never called so not implemented here def get_type_id(self): return JAVA_DEFAULT_TYPE_LINKED_LIST @@ -384,11 +370,13 @@ def read(self, inp): factory = self._factories.get(factory_id, None) if factory is None: - raise HazelcastSerializationError("No DataSerializerFactory registered for namespace: {}".format(factory_id)) + raise HazelcastSerializationError( + "No DataSerializerFactory registered for namespace: {}".format(factory_id)) identified = factory.get(class_id, None) if identified is None: raise HazelcastSerializationError( - "{} is not be able to create an instance for id: {} on factoryId: {}".format(factory, class_id, factory_id)) + "{} is not be able to create an instance for id: {} on factoryId: {}".format(factory, class_id, + factory_id)) instance = identified() instance.read_data(inp) return instance diff --git a/hazelcast/serialization/service.py b/hazelcast/serialization/service.py index d80cca615c..c3f55a031d 100644 --- a/hazelcast/serialization/service.py +++ b/hazelcast/serialization/service.py @@ -69,7 +69,6 @@ def _register_constant_serializers(self): # EXTENSIONS self._registry.register_constant_serializer(DateTimeSerializer(), datetime) self._registry.register_constant_serializer(BigIntegerSerializer()) - self._registry.register_constant_serializer(BigDecimalSerializer()) self._registry.register_constant_serializer(JavaClassSerializer()) self._registry.register_constant_serializer(ArrayListSerializer(), list) self._registry.register_constant_serializer(LinkedListSerializer()) diff --git a/tests/serialization/serializers_test.py b/tests/serialization/serializers_test.py index 78e447fc12..7d78701a7e 100644 --- a/tests/serialization/serializers_test.py +++ b/tests/serialization/serializers_test.py @@ -1,113 +1,433 @@ -import binascii +# coding=utf-8 +import datetime +import unittest +import uuid from hazelcast import six from hazelcast.core import HazelcastJsonValue from hazelcast.config import SerializationConfig, INTEGER_TYPE -from hazelcast.serialization.data import Data -from hazelcast.serialization.serialization_const import CONSTANT_TYPE_DOUBLE +from hazelcast.errors import HazelcastSerializationError +from hazelcast.serialization import BE_INT, MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG +from hazelcast.serialization.predicate import * from hazelcast.serialization.service import SerializationServiceV1 from tests.base import SingleMemberTestCase from tests.hzrc.ttypes import Lang +from tests.util import random_string +if not six.PY2: + long = int -class SerializersTestCase(SingleMemberTestCase): + +class A(object): + def __init__(self, x): + self.x = x + + def __eq__(self, other): + return isinstance(other, A) and self.x == other.x + + def __ne__(self, other): + return not self.__eq__(other) + + +class SerializersTest(unittest.TestCase): + def setUp(self): + self.service = SerializationServiceV1(SerializationConfig()) + + def tearDown(self): + self.service.destroy() + + def test_integer(self): + self.validate(14) + self.validate(-14) + + def test_float(self): + self.validate(545.3) + self.validate(-545.3) + + def test_large_int_raises(self): + with self.assertRaises(HazelcastSerializationError): + self.validate(1 << 63) + + def test_bool(self): + self.validate(True) + self.validate(False) + + def test_string(self): + self.validate("") + self.validate("client") + self.validate(six.u("1⚐中💦2😭‍🙆😔5")) + self.validate(six.u("Iñtërnâtiônàlizætiøn")) + self.validate(six.u("\u0040\u0041\u01DF\u06A0\u12E0\u1D30")) + + def test_bytearray(self): + self.validate(bytearray("abc".encode())) + + def test_none(self): + self.validate(None) + + def test_hazelcast_json_value(self): + self.validate(HazelcastJsonValue("{\"abc\": \"abc\", \"five\": 5}")) + + def test_uuid(self): + self.validate(uuid.uuid4()) + + def test_datetime(self): + d = datetime.datetime.now() + serialized = self.service.to_data(d) + deserialized = self.service.to_object(serialized) + self.assertEqual(d.timetuple(), deserialized.timetuple()) + + def test_list(self): + self.validate([1, 2.0, "a", None, bytearray("abc".encode()), [], [1, 2, 3]]) + + def test_python_object(self): + self.validate(A(123)) + + def test_predicates(self): + self.validate_predicate(sql("test")) + self.validate_predicate(and_(true(), true())) + self.validate_predicate(is_between("this", 0, 1)) + self.validate_predicate(is_equal_to("this", 10)) + self.validate_predicate(is_greater_than("this", 10)) + self.validate_predicate(is_greater_than_or_equal_to("this", 10)) + self.validate_predicate(is_less_than("this", 10)) + self.validate_predicate(is_less_than_or_equal_to("this", 10)) + self.validate_predicate(is_like("this", "*")) + self.validate_predicate(is_ilike("this", "*")) + self.validate_predicate(is_in("this", 10, 11, 12)) + self.validate_predicate(is_instance_of("java.lang.Serializable")) + self.validate_predicate(is_not_equal_to("this", 10)) + self.validate_predicate(not_(true())) + self.validate_predicate(or_(true(), true())) + self.validate_predicate(matches_regex("this", "/abc/")) + self.validate_predicate(true()) + self.validate_predicate(false()) + + def validate(self, value): + serialized = self.service.to_data(value) + deserialized = self.service.to_object(serialized) + self.assertEqual(value, deserialized) + + def validate_predicate(self, predicate): + serialized = self.service.to_data(predicate) + self.assertEqual(-2, serialized.get_type()) # Identified + b = serialized.to_bytes() + # 4(partition hash) + 4(serializer type) + 1(is_identified) + 4(factory id) + 4(class id) + payload(if any) + self.assertTrue(len(b) >= 17) + self.assertEqual(predicate.get_factory_id(), BE_INT.unpack_from(b, 9)[0]) + self.assertEqual(predicate.get_class_id(), BE_INT.unpack_from(b, 13)[0]) + + +class SerializersLiveTest(SingleMemberTestCase): @classmethod def configure_client(cls, config): config.cluster_name = cls.cluster.id return config def setUp(self): - config = SerializationConfig() - config.default_integer_type = INTEGER_TYPE.BIG_INT - self.service = SerializationServiceV1(serialization_config=config) + self.map = self.client.get_map(random_string()).blocking() def tearDown(self): - self.service.destroy() + self.map.clear() - def test_none_serializer(self): - none = None - data_n = self.service.to_data(none) - self.assertIsNone(data_n) - self.assertIsNone(self.service.to_object(Data())) - - def test_boolean_serializer(self): - true = True - false = False - data_t = self.service.to_data(true) - data_f = self.service.to_data(false) - - obj_t = self.service.to_object(data_t) - obj_f = self.service.to_object(data_f) - self.assertEqual(true, obj_t) - self.assertEqual(false, obj_f) - - def test_char_type_serializer(self): - buff = bytearray(binascii.unhexlify("00000000fffffffb00e7")) - data = Data(buff) - obj = self.service.to_object(data) - self.assertEqual(six.unichr(0x00e7), obj) + def get_from_server(self): + script = """var StringArray = Java.type("java.lang.String[]"); + function foo() { + var map = instance_0.getMap(\"""" + self.map.name + """\"); + var res = map.get("key"); + if (res.getClass().isArray()) { + return Java.from(res); + } else { + return res; + } + } + result = ""+foo();""" + response = self.rc.executeOnController(self.cluster.id, script, Lang.JAVASCRIPT) + return response.result.decode("utf-8") - def test_float(self): - buff = bytearray(binascii.unhexlify("00000000fffffff700000000")) - data = Data(buff) - obj = self.service.to_object(data) - self.assertEqual(0.0, obj) + def set_on_server(self, obj): + script = """var map = instance_0.getMap(\"""" + self.map.name + """\"); + map.set("key", """ + obj + """);""" + response = self.rc.executeOnController(self.cluster.id, script, Lang.JAVASCRIPT) + return response.success + + def replace_serialization_service(self, integer_type): + config = SerializationConfig() + config.default_integer_type = integer_type + new_service = SerializationServiceV1(config) + self.map._wrapped._to_data = new_service.to_data + self.map._wrapped._to_object = new_service.to_object + + def test_bool(self): + value = True + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() == "true" + self.assertEqual(value, response) + + def test_byte(self): + self.replace_serialization_service(INTEGER_TYPE.BYTE) + value = (1 << 7) - 1 + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = int(self.get_from_server()) + self.assertEqual(value, response) + + def test_short(self): + self.replace_serialization_service(INTEGER_TYPE.SHORT) + value = -1 * (1 << 15) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = int(self.get_from_server()) + self.assertEqual(value, response) + + def test_int(self): + value = (1 << 31) - 1 + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = int(self.get_from_server()) + self.assertEqual(value, response) + + def test_long(self): + self.replace_serialization_service(INTEGER_TYPE.LONG) + value = -1 * (1 << 63) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) def test_double(self): - double = 1.0 - data = self.service.to_data(double) - obj = self.service.to_object(data) - self.assertEqual(data.get_type(), CONSTANT_TYPE_DOUBLE) - self.assertEqual(double, obj) + value = 123.0 + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = float(self.get_from_server()) + self.assertEqual(value, response) + + def test_string(self): + value = "value" + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(value, response) + + def test_utf_string(self): + value = six.u("Iñtërnâtiônàlizætiøn") + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(value, response) + + def test_emoji(self): + value = six.u("1⚐中💦2😭‍🙆😔5") + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(value, response) + + def test_utf_chars(self): + value = six.u("\u0040\u0041\u01DF\u06A0\u12E0\u1D306") + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(value, response) + + def test_uuid(self): + value = uuid.uuid4() + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = uuid.UUID(hex=self.get_from_server()) + self.assertEqual(value, response) + + def test_hjv(self): + value = HazelcastJsonValue({"a": 3}) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = HazelcastJsonValue(self.get_from_server()) + self.assertEqual(value, response) + + def test_bytearray(self): + value = bytearray("value".encode()) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = bytearray(map(int, self.get_from_server().split(","))) + self.assertEqual(value, response) def test_datetime(self): - year = 2000 - month = 11 - day = 15 - hour = 23 - minute = 59 - second = 49 - script = """ -from java.util import Date, Calendar -cal = Calendar.getInstance() -cal.set({}, ({}-1), {}, {}, {}, {}) -result=instance_0.getSerializationService().toBytes(cal.getTime()) -""".format(year, month, day, hour, minute, second) - response = self.rc.executeOnController(self.cluster.id, script, Lang.PYTHON) - data = Data(response.result) - val = self.service.to_object(data) - self.assertEqual(year, val.year) - self.assertEqual(month, val.month) - self.assertEqual(day, val.day) - self.assertEqual(hour, val.hour) - self.assertEqual(minute, val.minute) - self.assertEqual(second, val.second) - - def test_hazelcast_json_vale(self): - json_value = HazelcastJsonValue('{"key": "value"}') - json_data = self.service.to_data(json_value) - json_deserialized = self.service.to_object(json_data) - self.assertEqual(json_value.to_string(), json_deserialized.to_string()) - - def test_big_int_small(self): - self._big_int_test(12) - - def test_big_int_small_neg(self): - self._big_int_test(-13) - - def test_big_int(self): - self._big_int_test(1234567890123456789012345678901234567890) - - def test_big_int_neg(self): - self._big_int_test(-1234567890123456789012345678901234567890) - - def _big_int_test(self, big_int): - script = """from java.math import BigInteger -result=instance_0.getSerializationService().toBytes(BigInteger("{}",10))""".format(big_int) - response = self.rc.executeOnController(self.cluster.id, script, Lang.PYTHON) - data = Data(response.result) - val = self.service.to_object(data) - data_local = self.service.to_data(big_int) - - self.assertEqual(binascii.hexlify(data._buffer), binascii.hexlify(data_local._buffer)) - self.assertEqual(big_int, val) + value = datetime.datetime.now() + self.map.set("key", value) + self.assertEqual(value.timetuple(), self.map.get("key").timetuple()) + response = self.get_from_server() + self.assertTrue(response.startswith(value.strftime("%a %b %d %H:%M:%S"))) + + def test_big_integer(self): + self.replace_serialization_service(INTEGER_TYPE.BIG_INT) + value = 1 << 128 + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + def test_variable_integer(self): + self.replace_serialization_service(INTEGER_TYPE.VAR) + value = MAX_BYTE + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + value = MAX_SHORT + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + value = MAX_INT + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + value = MAX_LONG + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + value = 1234567890123456789012345678901234567890 + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = long(self.get_from_server()) + self.assertEqual(value, response) + + def test_list(self): + value = [1, 2, 3] + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(value, list(map(int, response[1:-1].split(", ")))) + + def test_bool_from_server(self): + self.assertTrue(self.set_on_server("true")) + self.assertEqual(True, self.map.get("key")) + + def test_byte_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Byte(-23)")) + self.assertEqual(-23, self.map.get("key")) + + def test_char_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Character('x')")) + self.assertEqual("x", self.map.get("key")) + + def test_short_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Short(23)")) + self.assertEqual(23, self.map.get("key")) + + def test_integer_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Integer(" + str(1 << 30) + ")")) + self.assertEqual(1 << 30, self.map.get("key")) + + def test_long_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Long(-1 * " + str(1 << 63) + ")")) + self.assertEqual(-1 * (1 << 63), self.map.get("key")) + + def test_float_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Float(32.0)")) + self.assertEqual(32.0, self.map.get("key")) + + def test_double_from_server(self): + self.assertTrue(self.set_on_server("new java.lang.Double(-12332.0)")) + self.assertEqual(-12332.0, self.map.get("key")) + + def test_string_from_server(self): + self.assertTrue(self.set_on_server(six.u("\"1⚐中💦2😭‍🙆😔5\""))) + self.assertEqual(six.u("1⚐中💦2😭‍🙆😔5"), self.map.get("key")) + + def test_uuid_from_server(self): + self.assertTrue(self.set_on_server("new java.util.UUID(0, 1)")) + self.assertEqual(uuid.UUID(int=1), self.map.get("key")) + + def test_hjv_from_server(self): + self.assertTrue(self.set_on_server("new com.hazelcast.core.HazelcastJsonValue(\"{\\\"a\\\": 3}\")")) + self.assertEqual(HazelcastJsonValue({"a": 3}), self.map.get("key")) + + def test_bool_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([true, false], \"boolean[]\")")) + self.assertEqual([True, False], self.map.get("key")) + + def test_byte_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([3, 123], \"byte[]\")")) + self.assertEqual(bytearray([3, 123]), self.map.get("key")) + + def test_char_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to(['x', 'y'], \"char[]\")")) + self.assertEqual(["x", "y"], self.map.get("key")) + + def test_short_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([1323, -1232], \"short[]\")")) + self.assertEqual([1323, -1232], self.map.get("key")) + + def test_int_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([2147483647, -2147483648], \"int[]\")")) + self.assertEqual([2147483647, -2147483648], self.map.get("key")) + + def test_long_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([1152921504606846976, -1152921504606846976], \"long[]\")")) + self.assertEqual([1152921504606846976, -1152921504606846976], self.map.get("key")) + + def test_float_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([3123.0, -123.0], \"float[]\")")) + self.assertEqual([3123.0, -123.0], self.map.get("key")) + + def test_double_array_from_server(self): + self.assertTrue(self.set_on_server("Java.to([3123.0, -123.0], \"double[]\")")) + self.assertEqual([3123.0, -123.0], self.map.get("key")) + + def test_string_array_from_server(self): + self.assertTrue(self.set_on_server(six.u("Java.to([\"hey\", \"1⚐中💦2😭‍🙆😔5\"], \"java.lang.String[]\")"))) + self.assertEqual(["hey", six.u("1⚐中💦2😭‍🙆😔5")], self.map.get("key")) + + def test_date_from_server(self): + self.assertTrue(self.set_on_server("new java.util.Date(100, 11, 15, 23, 59, 49)")) + # server adds 1900 to year. Also, month is 0-based for server and 1-based for the client + self.assertEqual(datetime.datetime(2000, 12, 15, 23, 59, 49), self.map.get("key")) + + def test_big_integer_from_server(self): + self.assertTrue(self.set_on_server("new java.math.BigInteger(\"12\", 10)")) + self.assertEqual(12, self.map.get("key")) + + self.assertTrue(self.set_on_server("new java.math.BigInteger(\"-13\", 10)")) + self.assertEqual(-13, self.map.get("key")) + + self.assertTrue( + self.set_on_server("new java.math.BigInteger(\"1234567890123456789012345678901234567890\", 10)")) + self.assertEqual(1234567890123456789012345678901234567890, self.map.get("key")) + + self.assertTrue( + self.set_on_server("new java.math.BigInteger(\"-1234567890123456789012345678901234567890\", 10)")) + self.assertEqual(-1234567890123456789012345678901234567890, self.map.get("key")) + + def test_java_class_from_server(self): + self.assertTrue(self.set_on_server("java.lang.String.class")) + self.assertEqual("java.lang.String", self.map.get("key")) + + def test_array_list_from_server(self): + script = """var list = new java.util.ArrayList(); + list.add(1); + list.add(2); + list.add(3); + var map = instance_0.getMap(\"""" + self.map.name + """\"); + map.set("key", list);""" + response = self.rc.executeOnController(self.cluster.id, script, Lang.JAVASCRIPT) + self.assertTrue(response.success) + self.assertEqual([1, 2, 3], self.map.get("key")) + + def test_linked_list_from_server(self): + script = """var list = new java.util.LinkedList(); + list.add("a"); + list.add("b"); + list.add("c"); + var map = instance_0.getMap(\"""" + self.map.name + """\"); + map.set("key", list);""" + response = self.rc.executeOnController(self.cluster.id, script, Lang.JAVASCRIPT) + self.assertTrue(response.success) + self.assertEqual(["a", "b", "c"], self.map.get("key"))