[API-700] v5 SQL: Initial PR (#456)

* v5 SQL: Initial PR

This is an initial PR for the v5 SQL. It consists minimal changes
required to test the v5 SQL. Follow up PRs will come after that.

Here are the list of changes made:

- Set client version to 5.0
- Update remote controller scripts so that it uses hazelcast+hazelcast-sql
JARs instead of hazelcast-all, since hazelcast-all is not produced anymore.
Also, update some documentation about it.
- Use ints to represent years in DATE column type
- Use built-in types for DATE, TIME, TIMESTAMP, TIMESTAMP WITH TIMEZONE, and DECIMAL
which are datetime.date, datetime.time, datetime.datetime,
datetime.datetime(with non-None tzinfo), and decimal.Decimal.
- Update SQL tests and add some Jet tests
- Fix distributed objects tests by filtering internal objects out.

* address review comments

docs/conf.py

@@ -72,9 +72,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = "4.2.1"
+version = "5.0"
 # The full version, including alpha/beta/rc tags.
-release = "4.2.1"
+release = "5.0"
 
 autodoc_member_order = "bysource"
 autoclass_content = "both"

docs/using_python_client_with_hazelcast_imdg.rst

@@ -1507,49 +1507,24 @@ Data Types
 The SQL service supports a set of SQL data types. Every data type is mapped to
 a Python type that represents the type’s value.
 
-======================== ===============
+======================== ========================================
 Type Name                Python Type
-======================== ===============
+======================== ========================================
 BOOLEAN                  bool
 VARCHAR                  str
 TINYINT                  int
 SMALLINT                 int
 INTEGER                  int
 BIGINT                   int
-DECIMAL                  str
+DECIMAL                  decimal.Decimal
 REAL                     float
 DOUBLE                   float
-DATE                     str
-TIME                     str
-TIMESTAMP                str
-TIMESTAMP_WITH_TIME_ZONE str
+DATE                     datetime.date
+TIME                     datetime.time
+TIMESTAMP                datetime.datetime
+TIMESTAMP_WITH_TIME_ZONE datetime.datetime (with non-None tzinfo)
 OBJECT                   Any Python type
-======================== ===============
-
-Note that, the following types are returned as strings, with the following
-formats.
-
-- ``DATE`` with the ``YYYY-MM-DD`` format.
-- ``TIME`` with the ``HH:MM:SS[.ffffff]`` format.
-- ``TIMESTAMP`` with the ``YYYY-MM-DDTHH:MM:SS[.ffffff]`` format.
-- ``TIMESTAMP_WITH_TIME_ZONE`` with the
-  ``YYYY-MM-DDTHH:MM:SS[.ffffff](+|-)HH:MM[:SS]`` format.
-- ``DECIMAL`` with the floating point number format.
-
-If you want to use these types in queries, you have to send them as strings
-and add explicit ``CAST`` to queries.
-
-``CAST`` operator has the following syntax:
-
-.. code:: sql
-
-    CAST(? AS TYPE)
-
-An example usage is shown below:
-
-.. code:: python
-
-    client.sql.execute("SELECT * FROM map WHERE date < CAST(? AS DATE)", "2021-06-02")
+======================== ========================================
 
 SELECT
 ~~~~~~

hazelcast/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.2.1"
+__version__ = "5.0"
 
 # Set the default handler to "hazelcast" loggers
 # to avoid "No handlers could be found" warnings.

hazelcast/protocol/builtin.py

@@ -36,7 +36,7 @@
 from hazelcast.serialization.data import Data
 from hazelcast.util import int_from_bytes, timezone
 
-_LOCAL_DATE_SIZE_IN_BYTES = SHORT_SIZE_IN_BYTES + BYTE_SIZE_IN_BYTES * 2
+_LOCAL_DATE_SIZE_IN_BYTES = INT_SIZE_IN_BYTES + BYTE_SIZE_IN_BYTES * 2
 _LOCAL_TIME_SIZE_IN_BYTES = BYTE_SIZE_IN_BYTES * 3 + INT_SIZE_IN_BYTES
 _LOCAL_DATE_TIME_SIZE_IN_BYTES = _LOCAL_DATE_SIZE_IN_BYTES + _LOCAL_TIME_SIZE_IN_BYTES
 _OFFSET_DATE_TIME_SIZE_IN_BYTES = _LOCAL_DATE_TIME_SIZE_IN_BYTES + INT_SIZE_IN_BYTES
@@ -303,9 +303,9 @@ def decode_double(buf, offset):
 
     @staticmethod
     def decode_local_date(buf, offset):
-        year = FixSizedTypesCodec.decode_short(buf, offset)
-        month = FixSizedTypesCodec.decode_byte(buf, offset + SHORT_SIZE_IN_BYTES)
-        day = FixSizedTypesCodec.decode_byte(buf, offset + SHORT_SIZE_IN_BYTES + BYTE_SIZE_IN_BYTES)
+        year = FixSizedTypesCodec.decode_int(buf, offset)
+        month = FixSizedTypesCodec.decode_byte(buf, offset + INT_SIZE_IN_BYTES)
+        day = FixSizedTypesCodec.decode_byte(buf, offset + INT_SIZE_IN_BYTES + BYTE_SIZE_IN_BYTES)
 
         return date(year, month, day)
 
@@ -652,49 +652,33 @@ class ListCNLocalDateCodec(object):
     @staticmethod
     def decode(msg):
         return ListCNFixedSizeCodec.decode(
-            msg, _LOCAL_DATE_SIZE_IN_BYTES, ListCNLocalDateCodec._decode_item
+            msg, _LOCAL_DATE_SIZE_IN_BYTES, FixSizedTypesCodec.decode_local_date
         )
 
-    @staticmethod
-    def _decode_item(buf, offset):
-        return FixSizedTypesCodec.decode_local_date(buf, offset).isoformat()
-
 
 class ListCNLocalTimeCodec(object):
     @staticmethod
     def decode(msg):
         return ListCNFixedSizeCodec.decode(
-            msg, _LOCAL_TIME_SIZE_IN_BYTES, ListCNLocalTimeCodec._decode_item
+            msg, _LOCAL_TIME_SIZE_IN_BYTES, FixSizedTypesCodec.decode_local_time
         )
 
-    @staticmethod
-    def _decode_item(buf, offset):
-        return FixSizedTypesCodec.decode_local_time(buf, offset).isoformat()
-
 
 class ListCNLocalDateTimeCodec(object):
     @staticmethod
     def decode(msg):
         return ListCNFixedSizeCodec.decode(
-            msg, _LOCAL_DATE_TIME_SIZE_IN_BYTES, ListCNLocalDateTimeCodec._decode_item
+            msg, _LOCAL_DATE_TIME_SIZE_IN_BYTES, FixSizedTypesCodec.decode_local_date_time
         )
 
-    @staticmethod
-    def _decode_item(buf, offset):
-        return FixSizedTypesCodec.decode_local_date_time(buf, offset).isoformat()
-
 
 class ListCNOffsetDateTimeCodec(object):
     @staticmethod
     def decode(msg):
         return ListCNFixedSizeCodec.decode(
-            msg, _OFFSET_DATE_TIME_SIZE_IN_BYTES, ListCNOffsetDateTimeCodec._decode_item
+            msg, _OFFSET_DATE_TIME_SIZE_IN_BYTES, FixSizedTypesCodec.decode_offset_date_time
         )
 
-    @staticmethod
-    def _decode_item(buf, offset):
-        return FixSizedTypesCodec.decode_offset_date_time(buf, offset).isoformat()
-
 
 class BigDecimalCodec(object):
     @staticmethod
@@ -704,9 +688,7 @@ def decode(msg):
         unscaled_value = int_from_bytes(buf[INT_SIZE_IN_BYTES : INT_SIZE_IN_BYTES + size])
         scale = FixSizedTypesCodec.decode_int(buf, INT_SIZE_IN_BYTES + size)
         sign = 0 if unscaled_value >= 0 else 1
-        return str(
-            Decimal((sign, tuple(int(digit) for digit in str(abs(unscaled_value))), -1 * scale))
-        )
+        return Decimal((sign, tuple(int(digit) for digit in str(abs(unscaled_value))), -1 * scale))
 
 
 class SqlPageCodec(object):

hazelcast/sql.py

@@ -129,9 +129,7 @@ def write_portable(self, writer):
 
         When an SQL statement is submitted to a member, it is parsed and
         optimized by the ``hazelcast-sql`` module. The ``hazelcast-sql`` must
-        be in the classpath, otherwise an exception will be thrown. If you're
-        using the ``hazelcast-all`` or ``hazelcast-enterprise-all`` packages, the
-        ``hazelcast-sql`` module is included in them by default. If not, i.e., you
+        be in the classpath, otherwise an exception will be thrown. If you
         are using ``hazelcast`` or ``hazelcast-enterprise``, then you need to have
         ``hazelcast-sql`` in the classpath. If you are using the Docker image,
         the SQL module is included by default.
@@ -338,7 +336,7 @@ class SqlColumnType(object):
 
     DECIMAL = 6
     """
-    Represented by ``str``.
+    Represented by ``decimal.Decimal``.
     """
 
     REAL = 7
@@ -353,23 +351,22 @@ class SqlColumnType(object):
 
     DATE = 9
     """
-    Represented by ``str`` with the ``YYYY-MM-DD`` format.
+    Represented by ``datetime.date``.
     """
 
     TIME = 10
     """
-    Represented by ``str`` with the ``HH:MM:SS[.ffffff]`` format.
+    Represented by ``datetime.time``.
     """
 
     TIMESTAMP = 11
     """
-    Represented by ``str`` with the ``YYYY-MM-DDTHH:MM:SS[.ffffff]`` format.
+    Represented by ``datetime.datetime`` with ``None`` ``tzinfo``.
     """
 
     TIMESTAMP_WITH_TIME_ZONE = 12
     """
-    Represented by ``str`` with the 
-    ``YYYY-MM-DDTHH:MM:SS[.ffffff](+|-)HH:MM[:SS]`` format.
+    Represented by ``datetime.datetime`` with ``non-None`` ``tzinfo``.
     """
 
     OBJECT = 13

start_rc.py

@@ -3,7 +3,7 @@
 import sys
 from os.path import isfile
 
-SERVER_VERSION = "4.2.1"
+SERVER_VERSION = "5.0-SNAPSHOT"
 RC_VERSION = "0.8-SNAPSHOT"
 
 RELEASE_REPO = "http://repo1.maven.apache.org/maven2"
@@ -65,22 +65,22 @@ def start_rc(stdout=None, stderr=None):
 
     rc = download_if_necessary(RC_REPO, "hazelcast-remote-controller", RC_VERSION)
     tests = download_if_necessary(REPO, "hazelcast", SERVER_VERSION, True)
+    sql = download_if_necessary(REPO, "hazelcast-sql", SERVER_VERSION)
 
-    artifacts.append(rc)
-    artifacts.append(tests)
+    artifacts.extend([rc, tests, sql])
 
     enterprise_key = os.environ.get("HAZELCAST_ENTERPRISE_KEY", None)
 
     if enterprise_key:
-        server = download_if_necessary(ENTERPRISE_REPO, "hazelcast-enterprise-all", SERVER_VERSION)
+        server = download_if_necessary(ENTERPRISE_REPO, "hazelcast-enterprise", SERVER_VERSION)
         ep_tests = download_if_necessary(
             ENTERPRISE_REPO, "hazelcast-enterprise", SERVER_VERSION, True
         )
 
         artifacts.append(server)
         artifacts.append(ep_tests)
     else:
-        server = download_if_necessary(REPO, "hazelcast-all", SERVER_VERSION)
+        server = download_if_necessary(REPO, "hazelcast", SERVER_VERSION)
         artifacts.append(server)
 
     class_path = CLASS_PATH_SEPARATOR.join(artifacts)

tests/base.py

@@ -118,14 +118,15 @@ class SingleMemberTestCase(HazelcastTestCase):
     """
 
     rc = None
+    cluster = None
+    member = None
     client = None
 
     @classmethod
     def setUpClass(cls):
         cls.rc = cls.create_rc()
         cls.cluster = cls.create_cluster(cls.rc, cls.configure_cluster())
         cls.member = cls.cluster.start_member()
-
         cls.client = hazelcast.HazelcastClient(**cls.configure_client(dict()))
 
     @classmethod

tests/integration/backward_compatible/proxy/distributed_objects_test.py

@@ -30,29 +30,39 @@ def tearDown(self):
         self.member.shutdown()
 
     def test_get_distributed_objects(self):
-        six.assertCountEqual(self, [], self.client.get_distributed_objects())
+        six.assertCountEqual(
+            self, [], self._filter_internal_objects(self.client.get_distributed_objects())
+        )
 
         m = self.client.get_map("map")
         s = self.client.get_set("set")
         q = self.client.get_queue("queue")
 
         self.assertTrueEventually(
-            lambda: six.assertCountEqual(self, [m, s, q], self.client.get_distributed_objects())
+            lambda: six.assertCountEqual(
+                self,
+                [m, s, q],
+                self._filter_internal_objects(self.client.get_distributed_objects()),
+            )
         )
 
     def test_get_distributed_objects_clears_destroyed_proxies(self):
         m = self.client.get_map("map")
 
         self.assertTrueEventually(
-            lambda: six.assertCountEqual(self, [m], self.client.get_distributed_objects())
+            lambda: six.assertCountEqual(
+                self, [m], self._filter_internal_objects(self.client.get_distributed_objects())
+            )
         )
 
         other_client = hazelcast.HazelcastClient(**self.config)
         other_clients_map = other_client.get_map("map")
         other_clients_map.destroy()
 
         self.assertTrueEventually(
-            lambda: six.assertCountEqual(self, [], self.client.get_distributed_objects())
+            lambda: six.assertCountEqual(
+                self, [], self._filter_internal_objects(self.client.get_distributed_objects())
+            )
         )
         other_client.shutdown()
 
@@ -133,3 +143,7 @@ def assert_event():
 
     def test_remove_invalid_distributed_object_listener(self):
         self.assertFalse(self.client.remove_distributed_object_listener("invalid-reg-id").result())
+
+    @staticmethod
+    def _filter_internal_objects(distributed_objects):
+        return [obj for obj in distributed_objects if not obj.name.startswith("__")]