Version 0.9.99 changes

1. Test Coverage -> Fixed Test report coverage. The Python Server process's coverage was not being reported.
2. Unitest -> Use pytest fixtures for unittesting. Organized test cases.
3. SQLiteClient -> Fix in retry logic. Retry was not sending the request(again), it was simply polling the REQ Socket.
4. SQLiteServer -> Removed server.stop method. Keyboard interupt handling is handled in server.start() method
5. Serialization -> Changed msgpack-python to faster and efficient msgpack
6. Exceptions - The SQLiteClient will not raise an SQLiteRxConnectionError exception if the server is offline.
7. Exceptions - Renamed the error classes. SQLiteRxCompressionError, SQLiteRxConnectionError, SQLiteRxTransportError SQLiteRxSerializationError

.coveragerc

@@ -2,10 +2,11 @@
 parallel = True
 branch = False
 source = sqlite_rx
+concurrency = multiprocessing
 omit =
     /*/tests/*
     .env/*
-    /sqlite_rx/cli
+    ./sqlite_rx/cli/__init__.py
     /home/travis/virtualenv/*
 
 [report]

.dockerignore

@@ -8,3 +8,6 @@ dist
 sqlite_rx.egg-info
 .coveragerc
 docker_examples
+.coverage
+.pytest_cache
+.coverage

.gitignore

@@ -5,4 +5,6 @@ sqlite_rx.egg-info/
 cover/
 build/
 dist/
+.coverage
+.pytest_cache
 .coverage

.travis.yml

@@ -3,34 +3,18 @@ matrix:
   include:
     - os: linux
       python: 3.6
-      env:
-        - SITECUSTOMIZELIBPATH=/home/travis/virtualenv/python3.6/lib/python3.6/site-packages/sitecustomize.py
-        - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc
     - os: linux
       python: 3.7
-      env:
-        - SITECUSTOMIZELIBPATH=/home/travis/virtualenv/python3.7/lib/python3.7/site-packages/sitecustomize.py
-        - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc
     - os: linux
       python: 3.8
-      env:
-        - SITECUSTOMIZELIBPATH=/home/travis/virtualenv/python3.8/lib/python3.8/site-packages/sitecustomize.py
-        - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc
     - os: linux
       python: pypy3
-      env:
-        - SITECUSTOMIZELIBPATH=/home/travis/virtualenv/pypy3/site-packages/sitecustomize.py
-        - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc
+install:
+  - pip install --upgrade pip
+  - pip install pytest pytest-cov coverage coveralls
+  - pip install -e .
 script:
-  - cp ${TRAVIS_BUILD_DIR}/.travis/sitecustomize.py ${SITECUSTOMIZELIBPATH}
-  - cat ${SITECUSTOMIZELIBPATH}
-  - pip install nose
-  - pip install coverage
-  - pip install coveralls
-  - python -m site
-  - python --version
-  - python setup.py nosetests --with-coverage
-  - coverage run --concurrency=multiprocessing -m unittest discover -s sqlite_rx/tests
+  - coverage run -m pytest --verbose sqlite_rx/tests
   - coverage combine
   - coverage report -i -m
   - coveralls

requirements.txt

@@ -1,4 +1,4 @@
 click
-msgpack-python
+msgpack
 pyzmq
 tornado

setup.py

@@ -20,7 +20,7 @@
 
 PACKAGES = ['sqlite_rx']
 
-DEPENDENCIES = ['msgpack-python', 'pyzmq', 'tornado', 'click']
+DEPENDENCIES = ['msgpack', 'pyzmq', 'tornado', 'click']
 
 classifiers = [
     'Development Status :: 4 - Beta',

sqlite_rx/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.9.98"
+__version__ = "0.9.99"
 __author__ = "Abhishek Singh"
 __authoremail__ = "aosingh@asu.edu"
 
@@ -39,4 +39,8 @@ def get_default_logger_settings(level: str = "DEBUG"):
                 'propagate': False
             },
         }
-    }
+    }
+
+
+def get_version():
+    return __version__

sqlite_rx/cli/__init__.py

@@ -29,7 +29,7 @@ def main(log_level,
          curve_dir,
          key_id):
     logging.config.dictConfig(get_default_logger_settings(level=log_level))
-    LOG.info("Python Platform %s" % platform.python_implementation())
+    LOG.info("Python Platform %s", platform.python_implementation())
     kwargs = {
         'bind_address': f'tcp://{advertise_host}:{port}',
         'database': database,
@@ -38,18 +38,9 @@ def main(log_level,
         'use_encryption': curvezmq,
         'server_curve_id': key_id
     }
-    LOG.info('Args %s' % pformat(kwargs))
+    LOG.info('Args %s', pformat(kwargs))
     server = SQLiteServer(**kwargs)
-    try:
-        server.start()
-    except KeyboardInterrupt:
-        LOG.warning("Keyboard Interrupt")
-        server.stop()
-    except Exception:
-        LOG.exception("Exception in Server Thread")
-        server.stop()
-        raise
-
+    server.start()
 
 
 

sqlite_rx/client.py

@@ -178,6 +178,22 @@ def execute(self,
         raise SQLiteRxConnectionError("No response after retrying. Abandoning Request")
 
     def shutdown(self):
-        self._client.setsockopt(zmq.LINGER, 0)
-        self._client.close()
-        self._poller.unregister(self._client)
+        try:
+            self._client.setsockopt(zmq.LINGER, 0)
+            self._client.close()
+            self._poller.unregister(self._client)
+        except zmq.ZMQError as e:
+            if e.errno in (zmq.EINVAL,
+                          zmq.EPROTONOSUPPORT,
+                          zmq.ENOCOMPATPROTO,
+                          zmq.EADDRINUSE,
+                          zmq.EADDRNOTAVAIL,):
+                LOG.error("ZeroMQ Transportation endpoint was not setup")
+
+            elif e.errno in (zmq.ENODEV, zmq.ENOTSOCK,):
+                LOG.error("ZeroMQ request was made against a non-existent device or invalid socket")
+
+            elif e.errno in (zmq.ETERM, zmq.EMTHREAD,):
+                LOG.error("ZeroMQ context is not a state to handle this request for socket")
+        except Exception:
+            LOG.exception("Exception while shutting down SQLiteClient")

sqlite_rx/server.py

@@ -1,6 +1,7 @@
 import logging.config
 import multiprocessing
 import os
+import platform
 import socket
 import sqlite3
 import sys
@@ -11,9 +12,10 @@
 
 import msgpack
 import zmq
+from sqlite_rx import get_version
 from sqlite_rx.auth import Authorizer, KeyMonkey
 from sqlite_rx.exception import SQLiteRxZAPSetupError
-from tornado import ioloop
+from tornado import ioloop, version
 from zmq.auth.ioloop import IOLoopAuthenticator
 from zmq.eventloop import zmqstream
 
@@ -32,15 +34,17 @@ def __init__(self, *args, **kwargs):
         This class represents some of the abstractions for isolated server process
 
         """
+        super(SQLiteZMQProcess, self).__init__(*args, **kwargs)
         self.context = None
         self.loop = None
         self.socket = None
         self.auth = None
-        super(SQLiteZMQProcess, self).__init__(*args, **kwargs)
 
     def setup(self):
-        """Creates a ZMQ `context` and a Tornado `eventloop` for the SQLiteServer process
-        """
+        LOG.info("Python Platform %s", platform.python_implementation())
+        LOG.info("libzmq version %s", zmq.zmq_version())
+        LOG.info("pyzmq version %s", zmq.__version__)
+        LOG.info("tornado version %s", version)
         self.context = zmq.Context()
         self.loop = ioloop.IOLoop()
 
@@ -126,6 +130,7 @@ def __init__(self,
             use_zap_auth : True means use `ZAP` authentication. False means don't
 
         """
+        super(SQLiteServer, self).__init__(*args, *kwargs)
         self._bind_address = bind_address
         self._database = database
         self._auth_config = auth_config
@@ -134,15 +139,13 @@ def __init__(self,
         self.server_curve_id = server_curve_id
         self.curve_dir = curve_dir
         self.rep_stream = None
-        super(SQLiteServer, self).__init__(*args, *kwargs)
 
     def setup(self):
         """
         Start a zmq.REP socket stream and register a callback :class: `sqlite_rx.server.QueryStreamHandler`
 
         """
         super().setup()
-
         # Depending on the initialization parameters either get a plain stream or secure stream.
         self.rep_stream = self.stream(zmq.REP,
                                       self._bind_address,
@@ -157,12 +160,16 @@ def setup(self):
 
     def run(self):
         self.setup()
-        LOG.info("Server Event Loop started")
-        self.loop.start()
-
-    def stop(self):
-        self.loop.stop()
-        self.socket.close()
+        LOG.info("SQLiteServer version %s", get_version())
+        LOG.info("SQLiteServer (Tornado) i/o loop started..")
+        try:
+            LOG.info("Ready to accept client connections on %s", self._bind_address)
+            self.loop.start()
+        except KeyboardInterrupt:
+            LOG.info("SQLiteServer Shutting down")
+            self.rep_stream.close()
+            self.socket.close()
+            self.loop.stop()
 
 
 class QueryStreamHandler:

sqlite_rx/tests/__init__.py

@@ -0,0 +1,29 @@
+import os
+
+import socket
+
+import tempfile
+from contextlib import contextmanager
+import logging.config
+
+from sqlite_rx import get_default_logger_settings
+from sqlite_rx.auth import KeyGenerator
+
+logging.config.dictConfig(get_default_logger_settings(level="DEBUG"))
+
+LOG = logging.getLogger(__file__)
+
+
+@contextmanager
+def get_server_auth_files():
+    """Generate Temporary Private and Public keys for ZAP and CurveZMQ SQLiteServer
+
+    """
+    with tempfile.TemporaryDirectory() as curve_dir:
+        LOG.info("Curve dir is %s", curve_dir)
+        server_key_id = "id_server_{}_curve".format(socket.gethostname())
+        key_generator = KeyGenerator(destination_dir=curve_dir, key_id=server_key_id)
+        key_generator.generate()
+        server_public_key = os.path.join(curve_dir, "{}.key".format(server_key_id))
+        server_private_key = os.path.join(curve_dir, "{}.key_secret".format(server_key_id))
+        yield curve_dir, server_key_id, server_public_key, server_private_key

sqlite_rx/tests/curezmq/conftest.py

@@ -0,0 +1,55 @@
+import pytest
+
+import os
+import socket
+import shutil
+import signal
+import sqlite3
+import logging.config
+
+from sqlite_rx import get_default_logger_settings
+from sqlite_rx.auth import KeyGenerator
+from sqlite_rx.client import SQLiteClient
+from sqlite_rx.server import SQLiteServer
+from sqlite_rx.tests import get_server_auth_files
+
+logging.config.dictConfig(get_default_logger_settings(level="DEBUG"))
+
+LOG = logging.getLogger(__file__)
+
+@pytest.fixture(scope="module")
+def curvezmq_client():
+    with get_server_auth_files() as auth_files:
+        curve_dir, server_key_id, server_public_key, server_private_key = auth_files
+        client_key_id = "id_client_{}_curve".format(socket.gethostname())
+        key_generator = KeyGenerator(destination_dir=curve_dir, key_id=client_key_id)
+        key_generator.generate()
+        client_public_key = os.path.join(curve_dir, "{}.key".format(client_key_id))
+        client_private_key = os.path.join(curve_dir, "{}.key_secret".format(client_key_id))
+        shutil.copyfile(client_public_key, os.path.join(curve_dir,
+                                                        'authorized_clients',
+                                                        "{}.key".format(client_key_id)))
+        auth_config = {
+            sqlite3.SQLITE_OK: {
+                sqlite3.SQLITE_DROP_TABLE
+            }
+        }
+        server = SQLiteServer(bind_address="tcp://127.0.0.1:5002",
+                              use_encryption=True,
+                              curve_dir=curve_dir,
+                              server_curve_id=server_key_id,
+                              auth_config=auth_config,
+                              database=":memory:")
+
+        client = SQLiteClient(connect_address="tcp://127.0.0.1:5002",
+                              server_curve_id=server_key_id,
+                              client_curve_id=client_key_id,
+                              curve_dir=curve_dir,
+                              use_encryption=True)
+        server.start()
+        LOG.info("Started Test SQLiteServer")
+        yield client
+        os.kill(server.pid, signal.SIGINT)
+        server.join()
+        client.shutdown()
+

sqlite_rx/tests/curezmq/test_queries.py

@@ -0,0 +1,59 @@
+import platform
+
+sqlite_error_prefix = "sqlite3.OperationalError"
+
+if platform.python_implementation() == "PyPy":
+    sqlite_error_prefix = "_sqlite3.OperationalError"
+
+
+def test_not_present(curvezmq_client):
+    result = curvezmq_client.execute('SELECT * FROM IDOLS')
+    expected_result = {
+        'items': [],
+        'error': {
+            'message': '{0}: no such table: IDOLS'.format(sqlite_error_prefix),
+            'type': '{0}'.format(sqlite_error_prefix)}}
+    assert type(result) == dict
+    assert result == expected_result
+
+
+def test_table_creation(curvezmq_client):
+    result = curvezmq_client.execute('CREATE TABLE stocks (date text, trans text, symbol text, qty real, price real)')
+    expected_result = {"error": None, 'items': []}
+    assert result == expected_result
+
+
+def test_table_rows_insertion(curvezmq_client):
+    purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ]
+
+    result = curvezmq_client.execute('INSERT INTO stocks VALUES (?,?,?,?,?)', *purchases, execute_many=True)
+    expected_result = {'error': None, 'items': [], 'row_count': 27}
+    assert result == expected_result
+