From 5d34617c987f6174b2dfecc398ce9cdf19a9c96d Mon Sep 17 00:00:00 2001 From: shveenkov Date: Fri, 17 Jul 2015 17:39:48 +0300 Subject: [PATCH 1/4] fix tarantool connector for python 3.4 with six library usage --- requirements.txt | 1 + setup.py | 1 + tarantool/connection.py | 21 +++++++++++---------- tarantool/error.py | 10 ++++++---- tarantool/schema.py | 16 +++++++++------- tarantool/utils.py | 6 +++++- tests/suites/lib/tarantool_server.py | 4 ++-- tests/suites/test_dml.py | 10 ++++++---- tests/suites/test_schema.py | 4 ++-- 9 files changed, 43 insertions(+), 30 deletions(-) diff --git a/requirements.txt b/requirements.txt index 68a8a575..0a393586 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ msgpack-python>=0.4.0 pyyaml>=3.10 +six diff --git a/setup.py b/setup.py index adde97e5..7902ce37 100755 --- a/setup.py +++ b/setup.py @@ -83,5 +83,6 @@ def find_version(*file_paths): command_options=command_options, install_requires=[ 'msgpack-python>=0.4', + 'six', ] ) diff --git a/tarantool/connection.py b/tarantool/connection.py index ad86c22e..e54ba3a8 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -4,6 +4,7 @@ This module provides low-level API for Tarantool ''' +import six import time import errno import ctypes @@ -154,7 +155,7 @@ def connect(self): raise NetworkError(e) def _recv(self, to_read): - buf = '' + buf = b"" while to_read > 0: try: tmp = self._socket.recv(to_read) @@ -188,7 +189,7 @@ def _send_request_wo_reconnect(self, request): # Repeat request in a loop if the server returns completion_status == 1 # (try again) - for attempt in xrange(RETRY_MAX_ATTEMPTS): # pylint: disable=W0612 + for attempt in range(RETRY_MAX_ATTEMPTS): # pylint: disable=W0612 self._socket.sendall(bytes(request)) response = Response(self, self._read_response()) @@ -328,7 +329,7 @@ def replace(self, space_name, values): :rtype: `Response` instance ''' - if isinstance(space_name, basestring): + if isinstance(space_name, six.string_types): space_name = self.schema.get_space(space_name).sid request = RequestReplace(self, space_name, values) return self._send_request(request) @@ -377,7 +378,7 @@ def insert(self, space_name, values): :rtype: `Response` instance ''' - if isinstance(space_name, basestring): + if isinstance(space_name, six.string_types): space_name = self.schema.get_space(space_name).sid request = RequestInsert(self, space_name, values) return self._send_request(request) @@ -397,9 +398,9 @@ def delete(self, space_name, key, **kwargs): index_name = kwargs.get("index", 0) key = check_key(key) - if isinstance(space_name, basestring): + if isinstance(space_name, six.string_types): space_name = self.schema.get_space(space_name).sid - if isinstance(index_name, basestring): + if isinstance(index_name, six.string_types): index_name = self.schema.get_index(space_name, index_name).iid request = RequestDelete(self, space_name, index_name, key) return self._send_request(request) @@ -427,9 +428,9 @@ def update(self, space_name, key, op_list, **kwargs): index_name = kwargs.get("index", 0) key = check_key(key) - if isinstance(space_name, basestring): + if isinstance(space_name, six.string_types): space_name = self.schema.get_space(space_name).sid - if isinstance(index_name, basestring): + if isinstance(index_name, six.string_types): index_name = self.schema.get_index(space_name, index_name).iid request = RequestUpdate(self, space_name, index_name, key, op_list) return self._send_request(request) @@ -503,9 +504,9 @@ def select(self, space_name, key=None, **kwargs): # tuples) key = check_key(key, select=True) - if isinstance(space_name, basestring): + if isinstance(space_name, six.string_types): space_name = self.schema.get_space(space_name).sid - if isinstance(index_name, basestring): + if isinstance(index_name, six.string_types): index_name = self.schema.get_index(space_name, index_name).iid request = RequestSelect(self, space_name, index_name, key, offset, limit, iterator_type) diff --git a/tarantool/error.py b/tarantool/error.py index 649397d3..dbc4d54b 100644 --- a/tarantool/error.py +++ b/tarantool/error.py @@ -24,10 +24,12 @@ import sys import warnings - -class Error(StandardError): - - '''Base class for error exceptions''' +try: + class Error(StandardError): + '''Base class for error exceptions''' +except NameError: + class Error(Exception): + '''Base class for error exceptions''' class DatabaseError(Error): diff --git a/tarantool/schema.py b/tarantool/schema.py index 6477e97d..b0b147a1 100644 --- a/tarantool/schema.py +++ b/tarantool/schema.py @@ -5,6 +5,8 @@ It is a Tarantool schema description. ''' +import six + from tarantool.error import SchemaError import tarantool.const as const @@ -12,11 +14,11 @@ class SchemaIndex(object): def __init__(self, array, space): self.iid = array[1] - self.name = array[2] + self.name = array[2].decode() self.index = array[3] self.unique = array[4] self.parts = [] - for i in xrange(array[5]): + for i in range(array[5]): self.parts.append((array[5 + 1 + i * 2], array[5 + 2 + i * 2])) self.space = space self.space.indexes[self.iid] = self @@ -33,7 +35,7 @@ class SchemaSpace(object): def __init__(self, array, schema): self.sid = array[0] self.arity = array[1] - self.name = array[2] + self.name = array[2].decode() self.indexes = {} self.schema = schema self.schema[self.sid] = self @@ -57,14 +59,14 @@ def get_space(self, space): except KeyError: pass _index = (const.INDEX_SPACE_NAME - if isinstance(space, basestring) + if isinstance(space, six.string_types) else const.INDEX_SPACE_PRIMARY) array = self.con.select(const.SPACE_SPACE, space, index=_index) if len(array) > 1: raise SchemaError('Some strange output from server: \n' + array) elif len(array) == 0 or not len(array[0]): - temp_name = ('name' if isinstance(space, basestring) else 'id') + temp_name = ('name' if isinstance(space, six.string_types) else 'id') raise SchemaError( "There's no space with {1} '{0}'".format(space, temp_name)) array = array[0] @@ -77,7 +79,7 @@ def get_index(self, space, index): except KeyError: pass _index = (const.INDEX_INDEX_NAME - if isinstance(index, basestring) + if isinstance(index, six.string_types) else const.INDEX_INDEX_PRIMARY) array = self.con.select(const.SPACE_INDEX, [_space.sid, index], @@ -85,7 +87,7 @@ def get_index(self, space, index): if len(array) > 1: raise SchemaError('Some strange output from server: \n' + array) elif len(array) == 0 or not len(array[0]): - temp_name = ('name' if isinstance(index, basestring) else 'id') + temp_name = ('name' if isinstance(index, six.string_types) else 'id') raise SchemaError( "There's no index with {2} '{0}' in space '{1}'".format( index, _space.name, temp_name)) diff --git a/tarantool/utils.py b/tarantool/utils.py index 2deba0a7..bbc353be 100644 --- a/tarantool/utils.py +++ b/tarantool/utils.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- + +import six + def check_key(*args, **kwargs): if 'first' not in kwargs: kwargs['first'] = True @@ -12,5 +16,5 @@ def check_key(*args, **kwargs): elif args[0] is None and kwargs['select']: return [] for key in args: - assert isinstance(key, (int, long, basestring)) + assert isinstance(key, six.integer_types + six.string_types) return list(args) diff --git a/tests/suites/lib/tarantool_server.py b/tests/suites/lib/tarantool_server.py index 9cdf9cd8..1f80db0f 100644 --- a/tests/suites/lib/tarantool_server.py +++ b/tests/suites/lib/tarantool_server.py @@ -193,7 +193,7 @@ def find_exe(self): exe = os.path.join(_dir, self.default_tarantool["bin"]) if os.access(exe, os.X_OK): return os.path.abspath(exe) - raise RuntimeError("Can't find server executable in " + path) + raise RuntimeError("Can't find server executable in " + os.environ["PATH"]) def generate_configuration(self): os.putenv("PRIMARY_PORT", str(self.args['primary'])) @@ -240,7 +240,7 @@ def start(self): self.generate_configuration() if self.script: shutil.copy(self.script, self.script_dst) - os.chmod(self.script_dst, 0777) + os.chmod(self.script_dst, 0o777) args = self.prepare_args() self.process = subprocess.Popen(args, cwd = self.vardir, diff --git a/tests/suites/test_dml.py b/tests/suites/test_dml.py index 4018554b..2311eaae 100644 --- a/tests/suites/test_dml.py +++ b/tests/suites/test_dml.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- + +import six import yaml import unittest import tarantool @@ -7,8 +9,8 @@ class Request(unittest.TestCase): @classmethod def setUpClass(self): - print ' DML '.center(70, '=') - print '-' * 70 + print(' DML '.center(70, '=')) + print('-' * 70) self.srv = TarantoolServer() self.srv.script = 'tests/suites/box.lua' self.srv.start() @@ -32,7 +34,7 @@ def test_00_01_space_created(self): def test_00_02_fill_space(self): # Fill space with values - for i in xrange(1, 500): + for i in range(1, 500): self.assertEqual( self.con.insert('space_1', [i, i%5, 'tuple_'+str(i)])[0], [i, i%5, 'tuple_'+str(i)] @@ -145,7 +147,7 @@ def test_07_call(self): ans = self.con.call('fiber.time64') self.assertEqual(len(ans), 1) self.assertEqual(len(ans[0]), 1) - self.assertIsInstance(ans[0][0], (int, long)) + self.assertIsInstance(ans[0][0], six.integer_types) ans = self.con.call('uuid.str') self.assertEqual(len(ans), 1) self.assertEqual(len(ans[0]), 1) diff --git a/tests/suites/test_schema.py b/tests/suites/test_schema.py index 9f1cbfbb..871687a9 100755 --- a/tests/suites/test_schema.py +++ b/tests/suites/test_schema.py @@ -7,8 +7,8 @@ class TestSuite_Schema(unittest.TestCase): @classmethod def setUpClass(self): - print ' SCHEMA '.center(70, '=') - print '-' * 70 + print(' SCHEMA '.center(70, '=')) + print('-' * 70) self.srv = TarantoolServer() self.srv.script = 'tests/suites/box.lua' self.srv.start() From f8d91575a424c5e1e383f70bfac3b867fb0f1f9f Mon Sep 17 00:00:00 2001 From: shveenkov Date: Fri, 17 Jul 2015 18:43:34 +0300 Subject: [PATCH 2/4] fix tarantool connector for python 3.4 with six library usage, add yaml to install_requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7902ce37..6e2be7d2 100755 --- a/setup.py +++ b/setup.py @@ -84,5 +84,6 @@ def find_version(*file_paths): install_requires=[ 'msgpack-python>=0.4', 'six', + 'PyYAML>=3.10', ] ) From dc773ca59f6cad9bdb6769077efdf2f418c47d07 Mon Sep 17 00:00:00 2001 From: shveenkov Date: Sat, 18 Jul 2015 12:59:12 +0300 Subject: [PATCH 3/4] fix for tests ok --- tarantool/request.py | 10 +++++++--- tarantool/response.py | 7 ++++++- tarantool/schema.py | 4 ++-- tests/suites/lib/tarantool_server.py | 19 ++++++++++++++----- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/tarantool/request.py b/tarantool/request.py index 94af9bc7..eda04f8d 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -4,6 +4,7 @@ Request types definitions ''' +import six import msgpack import hashlib @@ -105,14 +106,17 @@ def __init__(self, conn, salt, user, password): super(RequestAuthenticate, self).__init__(conn) def sha1(values): - sha1 = hashlib.sha1() + sha = hashlib.sha1() for i in values: - sha1.update(i) - return sha1.digest() + sha.update(i if isinstance(i, six.binary_type) else i.encode()) + return sha.digest() def strxor(rhs, lhs): + if six.PY2: return "".join(chr(ord(x) ^ ord(y)) for x, y in zip(rhs, lhs)) + return bytes([x ^ y for x, y in zip(rhs, lhs)]) + hash1 = sha1((password,)) hash2 = sha1((hash1,)) scramble = sha1((salt, hash2)) diff --git a/tarantool/response.py b/tarantool/response.py index 29d9277c..ca9548ec 100644 --- a/tarantool/response.py +++ b/tarantool/response.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # pylint: disable=C0301,W0105,W0401,W0614 +import six import sys import msgpack import yaml @@ -41,7 +42,11 @@ def __init__(self, conn, response): # created in the __new__(). But let it be. super(Response, self).__init__() - unpacker = msgpack.Unpacker(use_list=True) + if six.PY2: + unpacker = msgpack.Unpacker(use_list=True) + else: + unpacker = msgpack.Unpacker(use_list=True, encoding="utf-8") + unpacker.feed(response) header = unpacker.unpack() diff --git a/tarantool/schema.py b/tarantool/schema.py index b0b147a1..5046ed31 100644 --- a/tarantool/schema.py +++ b/tarantool/schema.py @@ -14,7 +14,7 @@ class SchemaIndex(object): def __init__(self, array, space): self.iid = array[1] - self.name = array[2].decode() + self.name = array[2] self.index = array[3] self.unique = array[4] self.parts = [] @@ -35,7 +35,7 @@ class SchemaSpace(object): def __init__(self, array, schema): self.sid = array[0] self.arity = array[1] - self.name = array[2].decode() + self.name = array[2] self.indexes = {} self.schema = schema self.schema[self.sid] = self diff --git a/tests/suites/lib/tarantool_server.py b/tests/suites/lib/tarantool_server.py index 1f80db0f..05fe2897 100644 --- a/tests/suites/lib/tarantool_server.py +++ b/tests/suites/lib/tarantool_server.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- + import os +import os.path import errno import shlex import random @@ -39,6 +42,7 @@ def __init__(self, host, port): self.port = port self.is_connected = False self.socket = None + def connect(self): self.socket = socket.create_connection((self.host, self.port)) self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) @@ -87,7 +91,7 @@ def execute_no_reconnect(self, command): if not command: return cmd = command.replace('\n', ' ') + '\n' - self.socket.sendall(cmd) + self.socket.sendall(cmd.encode()) bufsiz = 4096 res = "" @@ -96,7 +100,7 @@ def execute_no_reconnect(self, command): buf = self.socket.recv(bufsiz) if not buf: break - res = res + buf + res = res + buf.decode() if (res.rfind("\n...\n") >= 0 or res.rfind("\r\n...\r\n") >= 0): break @@ -134,6 +138,7 @@ def script_dst(self): def script(self): if not hasattr(self, '_script'): self._script = None return self._script + @script.setter def script(self, val): if val is None: @@ -153,6 +158,7 @@ def _admin(self): if not hasattr(self, 'admin'): self.admin = None return self.admin + @_admin.setter def _admin(self, port): try: @@ -168,6 +174,7 @@ def log_des(self): if not hasattr(self, '_log_des'): self._log_des = open(self.logfile_path, 'a') return self._log_des + @log_des.deleter def log_des(self): if not hasattr(self, '_log_des'): @@ -249,15 +256,17 @@ def start(self): self.wait_until_started() def stop(self): - self.process.terminate() - self.process.wait() + if self.process.poll() is None: + self.process.terminate() + self.process.wait() def restart(self): self.stop() self.start() def clean(self): - shutil.rmtree(self.vardir) + if os.path.isdir(self.vardir): + shutil.rmtree(self.vardir) def __del__(self): self.stop() From 22b87d6cb05561bad2225b31145889252c0e6097 Mon Sep 17 00:00:00 2001 From: shveenkov Date: Wed, 29 Jul 2015 11:00:49 +0300 Subject: [PATCH 4/4] fix for empty password --- tarantool/request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tarantool/request.py b/tarantool/request.py index eda04f8d..23f23c6a 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -108,7 +108,8 @@ def __init__(self, conn, salt, user, password): def sha1(values): sha = hashlib.sha1() for i in values: - sha.update(i if isinstance(i, six.binary_type) else i.encode()) + if i is not None: + sha.update(i if isinstance(i, six.binary_type) else i.encode()) return sha.digest() def strxor(rhs, lhs):