diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3095941..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -language: python -sudo: required - -addons: - apt: - packages: - - openjdk-8-jdk - -env: - global: - - IGNITE_VERSION=2.9.1 - - IGNITE_HOME=/opt/ignite - -before_install: - - curl -L https://apache-mirror.rbc.ru/pub/apache/ignite/${IGNITE_VERSION}/apache-ignite-slim-${IGNITE_VERSION}-bin.zip > ignite.zip - - unzip ignite.zip -d /opt - - mv /opt/apache-ignite-slim-${IGNITE_VERSION}-bin /opt/ignite - - mv /opt/ignite/libs/optional/ignite-log4j2 /opt/ignite/libs/ - -jobs: - include: - - python: '3.6' - arch: amd64 - env: TOXENV=py36-no-ssl,py36-ssl,py36-ssl-password - - python: '3.7' - arch: amd64 - env: TOXENV=py37-no-ssl,py37-ssl,py37-ssl-password - - python: '3.8' - arch: amd64 - env: TOXENV=py38-no-ssl,py38-ssl,py38-ssl-password - -install: pip install tox -script: tox \ No newline at end of file diff --git a/pyignite/connection/connection.py b/pyignite/connection/connection.py index 6ab6c6a..8db304e 100644 --- a/pyignite/connection/connection.py +++ b/pyignite/connection/connection.py @@ -34,7 +34,7 @@ from pyignite.constants import * from pyignite.exceptions import ( - HandshakeError, ParameterError, SocketError, connection_errors, + HandshakeError, ParameterError, SocketError, connection_errors, AuthenticationError, ) from pyignite.datatypes import Byte, Int, Short, String, UUIDObject from pyignite.datatypes.internal import Struct @@ -43,6 +43,8 @@ from .ssl import wrap from ..stream import BinaryStream, READ_BACKWARD +CLIENT_STATUS_AUTH_FAILURE = 2000 + class Connection: """ @@ -180,7 +182,7 @@ def read_response(self) -> Union[dict, OrderedDict]: ('length', Int), ('op_code', Byte), ]) - with BinaryStream(self, self.recv()) as stream: + with BinaryStream(self, self.recv(reconnect=False)) as stream: start_class = response_start.parse(stream) start = stream.read_ctype(start_class, direction=READ_BACKWARD) data = response_start.to_python(start) @@ -191,6 +193,7 @@ def read_response(self) -> Union[dict, OrderedDict]: ('version_minor', Short), ('version_patch', Short), ('message', String), + ('client_status', Int) ]) elif self.get_protocol_version() >= (1, 4, 0): response_end = Struct([ @@ -267,7 +270,7 @@ def _connect_version( with BinaryStream(self) as stream: hs_request.from_python(stream) - self.send(stream.getbuffer()) + self.send(stream.getbuffer(), reconnect=False) hs_response = self.read_response() if hs_response['op_code'] == 0: @@ -291,6 +294,8 @@ def _connect_version( client_patch=protocol_version[2], **hs_response ) + elif hs_response['client_status'] == CLIENT_STATUS_AUTH_FAILURE: + raise AuthenticationError(error_text) raise HandshakeError(( hs_response['version_major'], hs_response['version_minor'], @@ -313,12 +318,13 @@ def reconnect(self): except connection_errors: pass - def send(self, data: Union[bytes, bytearray, memoryview], flags=None): + def send(self, data: Union[bytes, bytearray, memoryview], flags=None, reconnect=True): """ Send data down the socket. :param data: bytes to send, :param flags: (optional) OS-specific flags. + :param reconnect: (optional) reconnect on failure, default True. """ if self.closed: raise SocketError('Attempt to use closed connection.') @@ -334,7 +340,13 @@ def send(self, data: Union[bytes, bytearray, memoryview], flags=None): self.reconnect() raise - def recv(self, flags=None) -> bytearray: + def recv(self, flags=None, reconnect=True) -> bytearray: + """ + Receive data from the socket. + + :param flags: (optional) OS-specific flags. + :param reconnect: (optional) reconnect on failure, default True. + """ def _recv(buffer, num_bytes): bytes_to_receive = num_bytes while bytes_to_receive > 0: @@ -344,7 +356,8 @@ def _recv(buffer, num_bytes): raise SocketError('Connection broken.') except connection_errors: self.failed = True - self.reconnect() + if reconnect: + self.reconnect() raise buffer = buffer[bytes_rcvd:] diff --git a/pyignite/constants.py b/pyignite/constants.py index fc840d6..02f7124 100644 --- a/pyignite/constants.py +++ b/pyignite/constants.py @@ -49,7 +49,7 @@ PROTOCOL_STRING_ENCODING = 'utf-8' PROTOCOL_CHAR_ENCODING = 'utf-16le' -SSL_DEFAULT_VERSION = ssl.PROTOCOL_TLSv1_1 +SSL_DEFAULT_VERSION = ssl.PROTOCOL_TLSv1_2 SSL_DEFAULT_CIPHERS = ssl._DEFAULT_CIPHERS FNV1_OFFSET_BASIS = 0x811c9dc5 diff --git a/pyignite/exceptions.py b/pyignite/exceptions.py index 1b41d32..5933228 100644 --- a/pyignite/exceptions.py +++ b/pyignite/exceptions.py @@ -25,6 +25,15 @@ class ParseError(Exception): pass +class AuthenticationError(Exception): + """ + This exception is raised on authentication failure. + """ + + def __init__(self, message: str): + self.message = message + + class HandshakeError(SocketError): """ This exception is raised on Ignite binary protocol handshake failure, diff --git a/requirements/install.txt b/requirements/install.txt index cecea8f..1ee12a9 100644 --- a/requirements/install.txt +++ b/requirements/install.txt @@ -1,3 +1,3 @@ # these pip packages are necessary for the pyignite to run -attrs==18.1.0 +attrs==20.3.0 diff --git a/requirements/setup.txt b/requirements/setup.txt index 7c55f83..d202467 100644 --- a/requirements/setup.txt +++ b/requirements/setup.txt @@ -1,3 +1,3 @@ # additional package for integrating pytest in setuptools -pytest-runner==4.2 +pytest-runner==5.3.0 diff --git a/requirements/tests.txt b/requirements/tests.txt index 893928e..5d5ae84 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,7 +1,7 @@ # these packages are used for testing -pytest==3.6.1 -pytest-cov==2.5.1 -teamcity-messages==1.21 -psutil==5.6.5 +pytest==6.2.2 +pytest-cov==2.11.1 +teamcity-messages==1.28 +psutil==5.8.0 jinja2==2.11.3 diff --git a/tests/affinity/conftest.py b/tests/affinity/conftest.py new file mode 100644 index 0000000..b682d01 --- /dev/null +++ b/tests/affinity/conftest.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pyignite import Client +from pyignite.api import cache_create, cache_destroy +from tests.util import start_ignite_gen + + +@pytest.fixture(scope='module', autouse=True) +def server1(): + yield from start_ignite_gen(1) + + +@pytest.fixture(scope='module', autouse=True) +def server2(): + yield from start_ignite_gen(2) + + +@pytest.fixture(scope='module', autouse=True) +def server3(): + yield from start_ignite_gen(3) + + +@pytest.fixture +def client(): + client = Client(partition_aware=True) + + client.connect([('127.0.0.1', 10800 + i) for i in range(1, 4)]) + + yield client + + client.close() + + +@pytest.fixture +def client_not_connected(): + client = Client(partition_aware=True) + yield client + client.close() + + +@pytest.fixture +def cache(connected_client): + cache_name = 'my_bucket' + conn = connected_client.random_node + + cache_create(conn, cache_name) + yield cache_name + cache_destroy(conn, cache_name) + + +@pytest.fixture(scope='module', autouse=True) +def skip_if_no_affinity(request, server1): + client = Client(partition_aware=True) + client.connect('127.0.0.1', 10801) + + if not client.partition_awareness_supported_by_protocol: + pytest.skip(f'skipped {request.node.name}, partition awareness is not supported.') diff --git a/tests/test_affinity.py b/tests/affinity/test_affinity.py similarity index 80% rename from tests/test_affinity.py rename to tests/affinity/test_affinity.py index a55251b..ee8f6c0 100644 --- a/tests/test_affinity.py +++ b/tests/affinity/test_affinity.py @@ -27,12 +27,11 @@ from pyignite.datatypes.prop_codes import * -def test_get_node_partitions(client_partition_aware): +def test_get_node_partitions(client): + conn = client.random_node - conn = client_partition_aware.random_node - - cache_1 = client_partition_aware.get_or_create_cache('test_cache_1') - cache_2 = client_partition_aware.get_or_create_cache({ + cache_1 = client.get_or_create_cache('test_cache_1') + cache_2 = client.get_or_create_cache({ PROP_NAME: 'test_cache_2', PROP_CACHE_KEY_CONFIGURATION: [ { @@ -41,9 +40,9 @@ def test_get_node_partitions(client_partition_aware): } ], }) - cache_3 = client_partition_aware.get_or_create_cache('test_cache_3') - cache_4 = client_partition_aware.get_or_create_cache('test_cache_4') - cache_5 = client_partition_aware.get_or_create_cache('test_cache_5') + client.get_or_create_cache('test_cache_3') + client.get_or_create_cache('test_cache_4') + client.get_or_create_cache('test_cache_5') result = cache_get_node_partitions( conn, @@ -115,9 +114,8 @@ def test_get_node_partitions(client_partition_aware): ], ) -def test_affinity(client_partition_aware, key, key_hint): - - cache_1 = client_partition_aware.get_or_create_cache({ +def test_affinity(client, key, key_hint): + cache_1 = client.get_or_create_cache({ PROP_NAME: 'test_cache_1', PROP_CACHE_MODE: CacheMode.PARTITIONED, }) @@ -126,7 +124,7 @@ def test_affinity(client_partition_aware, key, key_hint): best_node = cache_1.get_best_node(key, key_hint=key_hint) - for node in filter(lambda n: n.alive, client_partition_aware._nodes): + for node in filter(lambda n: n.alive, client._nodes): result = cache_local_peek( node, cache_1.cache_id, key, key_hint=key_hint, ) @@ -142,9 +140,8 @@ def test_affinity(client_partition_aware, key, key_hint): cache_1.destroy() -def test_affinity_for_generic_object(client_partition_aware): - - cache_1 = client_partition_aware.get_or_create_cache({ +def test_affinity_for_generic_object(client): + cache_1 = client.get_or_create_cache({ PROP_NAME: 'test_cache_1', PROP_CACHE_MODE: CacheMode.PARTITIONED, }) @@ -166,7 +163,7 @@ class KeyClass( best_node = cache_1.get_best_node(key, key_hint=BinaryObject) - for node in filter(lambda n: n.alive, client_partition_aware._nodes): + for node in filter(lambda n: n.alive, client._nodes): result = cache_local_peek( node, cache_1.cache_id, key, key_hint=BinaryObject, ) @@ -182,16 +179,8 @@ class KeyClass( cache_1.destroy() -def test_affinity_for_generic_object_without_type_hints(client_partition_aware): - - if not client_partition_aware.partition_awareness_supported_by_protocol: - pytest.skip( - 'Best effort affinity is not supported by the protocol {}.'.format( - client_partition_aware.protocol_version - ) - ) - - cache_1 = client_partition_aware.get_or_create_cache({ +def test_affinity_for_generic_object_without_type_hints(client): + cache_1 = client.get_or_create_cache({ PROP_NAME: 'test_cache_1', PROP_CACHE_MODE: CacheMode.PARTITIONED, }) @@ -213,7 +202,7 @@ class KeyClass( best_node = cache_1.get_best_node(key) - for node in filter(lambda n: n.alive, client_partition_aware._nodes): + for node in filter(lambda n: n.alive, client._nodes): result = cache_local_peek( node, cache_1.cache_id, key ) diff --git a/tests/test_affinity_bad_servers.py b/tests/affinity/test_affinity_bad_servers.py similarity index 66% rename from tests/test_affinity_bad_servers.py rename to tests/affinity/test_affinity_bad_servers.py index dce09de..8abf4a0 100644 --- a/tests/test_affinity_bad_servers.py +++ b/tests/affinity/test_affinity_bad_servers.py @@ -16,22 +16,20 @@ import pytest from pyignite.exceptions import ReconnectError -from tests.util import * +from tests.util import start_ignite, kill_process_tree -def test_client_with_multiple_bad_servers(start_client): - client = start_client(partition_aware=True) +def test_client_with_multiple_bad_servers(client_not_connected): with pytest.raises(ReconnectError) as e_info: - client.connect([("127.0.0.1", 10900), ("127.0.0.1", 10901)]) + client_not_connected.connect([("127.0.0.1", 10900), ("127.0.0.1", 10901)]) assert str(e_info.value) == "Can not connect." -def test_client_with_failed_server(request, start_ignite_server, start_client): - srv = start_ignite_server(4) +def test_client_with_failed_server(request, client_not_connected): + srv = start_ignite(idx=4) try: - client = start_client() - client.connect([("127.0.0.1", 10804)]) - cache = client.get_or_create_cache(request.node.name) + client_not_connected.connect([("127.0.0.1", 10804)]) + cache = client_not_connected.get_or_create_cache(request.node.name) cache.put(1, 1) kill_process_tree(srv.pid) with pytest.raises(ConnectionResetError): @@ -40,17 +38,16 @@ def test_client_with_failed_server(request, start_ignite_server, start_client): kill_process_tree(srv.pid) -def test_client_with_recovered_server(request, start_ignite_server, start_client): - srv = start_ignite_server(4) +def test_client_with_recovered_server(request, client_not_connected): + srv = start_ignite(idx=4) try: - client = start_client() - client.connect([("127.0.0.1", 10804)]) - cache = client.get_or_create_cache(request.node.name) + client_not_connected.connect([("127.0.0.1", 10804)]) + cache = client_not_connected.get_or_create_cache(request.node.name) cache.put(1, 1) # Kill and restart server kill_process_tree(srv.pid) - srv = start_ignite_server(4) + srv = start_ignite(idx=4) # First request fails with pytest.raises(Exception): diff --git a/tests/test_affinity_request_routing.py b/tests/affinity/test_affinity_request_routing.py similarity index 89% rename from tests/test_affinity_request_routing.py rename to tests/affinity/test_affinity_request_routing.py index 3489dea..101db39 100644 --- a/tests/test_affinity_request_routing.py +++ b/tests/affinity/test_affinity_request_routing.py @@ -70,10 +70,8 @@ def check_grid_idx(): @pytest.mark.parametrize("key,grid_idx", [(1, 1), (2, 2), (3, 3), (4, 1), (5, 1), (6, 2), (11, 1), (13, 1), (19, 1)]) @pytest.mark.parametrize("backups", [0, 1, 2, 3]) -def test_cache_operation_on_primitive_key_routes_request_to_primary_node( - request, key, grid_idx, backups, client_partition_aware): - - cache = client_partition_aware.get_or_create_cache({ +def test_cache_operation_on_primitive_key_routes_request_to_primary_node(request, key, grid_idx, backups, client): + cache = client.get_or_create_cache({ PROP_NAME: request.node.name + str(backups), PROP_BACKUPS_NUMBER: backups, }) @@ -132,8 +130,7 @@ def test_cache_operation_on_complex_key_routes_request_to_primary_node(): @pytest.mark.parametrize("key,grid_idx", [(1, 2), (2, 1), (3, 1), (4, 2), (5, 2), (6, 3)]) @pytest.mark.skip(reason="Custom key objects are not supported yet") -def test_cache_operation_on_custom_affinity_key_routes_request_to_primary_node( - request, client_partition_aware, key, grid_idx): +def test_cache_operation_on_custom_affinity_key_routes_request_to_primary_node(request, client, key, grid_idx): class AffinityTestType1( metaclass=GenericObjectMeta, type_name='AffinityTestType1', @@ -153,7 +150,7 @@ class AffinityTestType1( }, ], } - cache = client_partition_aware.create_cache(cache_config) + cache = client.create_cache(cache_config) # noinspection PyArgumentList key_obj = AffinityTestType1( @@ -167,17 +164,18 @@ class AffinityTestType1( assert requests.pop() == grid_idx -def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server, start_client): - client = start_client(partition_aware=True) - client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802), ("127.0.0.1", 10803), ("127.0.0.1", 10804)]) - cache = client.get_or_create_cache(request.node.name) +def test_cache_operation_routed_to_new_cluster_node(request, client_not_connected): + client_not_connected.connect( + [("127.0.0.1", 10801), ("127.0.0.1", 10802), ("127.0.0.1", 10803), ("127.0.0.1", 10804)] + ) + cache = client_not_connected.get_or_create_cache(request.node.name) key = 12 wait_for_affinity_distribution(cache, key, 3) cache.put(key, key) cache.put(key, key) assert requests.pop() == 3 - srv = start_ignite_server(4) + srv = start_ignite(idx=4) try: # Wait for rebalance and partition map exchange wait_for_affinity_distribution(cache, key, 4) @@ -190,8 +188,8 @@ def test_cache_operation_routed_to_new_cluster_node(request, start_ignite_server kill_process_tree(srv.pid) -def test_replicated_cache_operation_routed_to_random_node(request, client_partition_aware): - cache = client_partition_aware.get_or_create_cache({ +def test_replicated_cache_operation_routed_to_random_node(request, client): + cache = client.get_or_create_cache({ PROP_NAME: request.node.name, PROP_CACHE_MODE: CacheMode.REPLICATED, }) diff --git a/tests/test_affinity_single_connection.py b/tests/affinity/test_affinity_single_connection.py similarity index 90% rename from tests/test_affinity_single_connection.py rename to tests/affinity/test_affinity_single_connection.py index 1943384..0768011 100644 --- a/tests/test_affinity_single_connection.py +++ b/tests/affinity/test_affinity_single_connection.py @@ -13,9 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest -def test_all_cache_operations_with_partition_aware_client_on_single_server(request, client_partition_aware_single_server): - cache = client_partition_aware_single_server.get_or_create_cache(request.node.name) +from pyignite import Client + + +@pytest.fixture(scope='module') +def client(): + client = Client(partition_aware=True) + client.connect('127.0.0.1', 10801) + yield client + client.close() + + +def test_all_cache_operations_with_partition_aware_client_on_single_server(request, client): + cache = client.get_or_create_cache(request.node.name) key = 1 key2 = 2 diff --git a/tests/common/conftest.py b/tests/common/conftest.py new file mode 100644 index 0000000..402aede --- /dev/null +++ b/tests/common/conftest.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pyignite import Client +from pyignite.api import cache_create, cache_destroy +from tests.util import start_ignite_gen + + +@pytest.fixture(scope='module', autouse=True) +def server1(): + yield from start_ignite_gen(1) + + +@pytest.fixture(scope='module', autouse=True) +def server2(): + yield from start_ignite_gen(2) + + +@pytest.fixture(scope='module', autouse=True) +def server3(): + yield from start_ignite_gen(3) + + +@pytest.fixture(scope='module') +def client(): + client = Client() + + client.connect('127.0.0.1', 10801) + + yield client + + client.close() + + +@pytest.fixture +def cache(client): + cache_name = 'my_bucket' + conn = client.random_node + + cache_create(conn, cache_name) + yield cache_name + cache_destroy(conn, cache_name) diff --git a/tests/test_binary.py b/tests/common/test_binary.py similarity index 100% rename from tests/test_binary.py rename to tests/common/test_binary.py diff --git a/tests/test_cache_class.py b/tests/common/test_cache_class.py similarity index 100% rename from tests/test_cache_class.py rename to tests/common/test_cache_class.py diff --git a/tests/test_cache_class_sql.py b/tests/common/test_cache_class_sql.py similarity index 100% rename from tests/test_cache_class_sql.py rename to tests/common/test_cache_class_sql.py diff --git a/tests/test_cache_composite_key_class_sql.py b/tests/common/test_cache_composite_key_class_sql.py similarity index 100% rename from tests/test_cache_composite_key_class_sql.py rename to tests/common/test_cache_composite_key_class_sql.py diff --git a/tests/test_cache_config.py b/tests/common/test_cache_config.py similarity index 100% rename from tests/test_cache_config.py rename to tests/common/test_cache_config.py diff --git a/tests/test_datatypes.py b/tests/common/test_datatypes.py similarity index 100% rename from tests/test_datatypes.py rename to tests/common/test_datatypes.py diff --git a/tests/test_generic_object.py b/tests/common/test_generic_object.py similarity index 100% rename from tests/test_generic_object.py rename to tests/common/test_generic_object.py diff --git a/tests/test_get_names.py b/tests/common/test_get_names.py similarity index 100% rename from tests/test_get_names.py rename to tests/common/test_get_names.py diff --git a/tests/test_key_value.py b/tests/common/test_key_value.py similarity index 100% rename from tests/test_key_value.py rename to tests/common/test_key_value.py diff --git a/tests/test_scan.py b/tests/common/test_scan.py similarity index 100% rename from tests/test_scan.py rename to tests/common/test_scan.py diff --git a/tests/test_sql.py b/tests/common/test_sql.py similarity index 98% rename from tests/test_sql.py rename to tests/common/test_sql.py index f25fedd..cc68a02 100644 --- a/tests/test_sql.py +++ b/tests/common/test_sql.py @@ -182,7 +182,7 @@ def test_long_multipage_query(client): client.sql('DROP TABLE LongMultipageQuery IF EXISTS') client.sql("CREATE TABLE LongMultiPageQuery (%s, %s)" % - (fields[0] + " INT(11) PRIMARY KEY", ",".join(map(lambda f: f + " INT(11)", fields[1:])))) + (fields[0] + " INT(11) PRIMARY KEY", ",".join(map(lambda f: f + " INT(11)", fields[1:])))) for id in range(1, 21): client.sql( diff --git a/tests/config/ignite-config.xml.jinja2 b/tests/config/ignite-config.xml.jinja2 index 834b5d8..85daf0f 100644 --- a/tests/config/ignite-config.xml.jinja2 +++ b/tests/config/ignite-config.xml.jinja2 @@ -27,6 +27,20 @@ http://www.springframework.org/schema/util/spring-util.xsd"> + {% if use_auth %} + + + + + + + + + + + + {% endif %} + {% if use_ssl %} {% endif %} diff --git a/tests/conftest.py b/tests/conftest.py index bd86f9c..59b7d3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,188 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import argparse -from distutils.util import strtobool -import ssl - import pytest -from pyignite import Client -from pyignite.constants import * -from pyignite.api import cache_create, cache_destroy -from tests.util import _start_ignite, start_ignite_gen - - -class BoolParser(argparse.Action): - - def __call__(self, parser, namespace, values, option_string=None): - values = True if values is None else bool(strtobool(values)) - setattr(namespace, self.dest, values) - - -class CertReqsParser(argparse.Action): - conv_map = { - 'NONE': ssl.CERT_NONE, - 'OPTIONAL': ssl.CERT_OPTIONAL, - 'REQUIRED': ssl.CERT_REQUIRED, - } - - def __call__(self, parser, namespace, values, option_string=None): - value = values.upper() - if value in self.conv_map: - setattr(namespace, self.dest, self.conv_map[value]) - else: - raise ValueError( - 'Undefined argument: --ssl-cert-reqs={}'.format(value) - ) - - -class SSLVersionParser(argparse.Action): - conv_map = { - 'TLSV1_1': ssl.PROTOCOL_TLSv1_1, - 'TLSV1_2': ssl.PROTOCOL_TLSv1_2, - } - - def __call__(self, parser, namespace, values, option_string=None): - value = values.upper() - if value in self.conv_map: - setattr(namespace, self.dest, self.conv_map[value]) - else: - raise ValueError( - 'Undefined argument: --ssl-version={}'.format(value) - ) - - -@pytest.fixture(scope='session', autouse=True) -def server1(request): - yield from start_ignite_server_gen(1, request) - - -@pytest.fixture(scope='session', autouse=True) -def server2(request): - yield from start_ignite_server_gen(2, request) - - -@pytest.fixture(scope='session', autouse=True) -def server3(request): - yield from start_ignite_server_gen(3, request) - - -@pytest.fixture(scope='module') -def start_ignite_server(use_ssl): - def start(idx=1): - return _start_ignite(idx, use_ssl=use_ssl) - - return start - - -def start_ignite_server_gen(idx, request): - use_ssl = request.config.getoption("--use-ssl") - yield from start_ignite_gen(idx, use_ssl) - - -@pytest.fixture(scope='module') -def client( - node, timeout, partition_aware, use_ssl, ssl_keyfile, ssl_keyfile_password, - ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, - username, password, -): - yield from client0(node, timeout, partition_aware, use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, - ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, password) - - -@pytest.fixture(scope='module') -def client_partition_aware( - node, timeout, use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, - ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, - password -): - yield from client0(node, timeout, True, use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, ssl_ca_certfile, - ssl_cert_reqs, ssl_ciphers, ssl_version, username, password) - - -@pytest.fixture(scope='module') -def client_partition_aware_single_server( - node, timeout, use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, - ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, - password -): - node = node[:1] - yield from client0(node, timeout, True, use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, ssl_ca_certfile, - ssl_cert_reqs, ssl_ciphers, ssl_version, username, password) - - -@pytest.fixture -def cache(client): - cache_name = 'my_bucket' - conn = client.random_node - - cache_create(conn, cache_name) - yield cache_name - cache_destroy(conn, cache_name) - - -@pytest.fixture(scope='module') -def start_client(use_ssl, ssl_keyfile, ssl_keyfile_password, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, - ssl_version,username, password): - def start(**kwargs): - cli_kw = kwargs.copy() - cli_kw.update({ - 'use_ssl': use_ssl, - 'ssl_keyfile': ssl_keyfile, - 'ssl_keyfile_password': ssl_keyfile_password, - 'ssl_certfile': ssl_certfile, - 'ssl_ca_certfile': ssl_ca_certfile, - 'ssl_cert_reqs': ssl_cert_reqs, - 'ssl_ciphers': ssl_ciphers, - 'ssl_version': ssl_version, - 'username': username, - 'password': password - }) - return Client(**cli_kw) - - return start - - -def client0( - node, timeout, partition_aware, use_ssl, ssl_keyfile, ssl_keyfile_password, - ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, - username, password, -): - client = Client( - timeout=timeout, - partition_aware=partition_aware, - use_ssl=use_ssl, - ssl_keyfile=ssl_keyfile, - ssl_keyfile_password=ssl_keyfile_password, - ssl_certfile=ssl_certfile, - ssl_ca_certfile=ssl_ca_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ciphers=ssl_ciphers, - ssl_version=ssl_version, - username=username, - password=password, - ) - nodes = [] - for n in node: - host, port = n.split(':') - port = int(port) - nodes.append((host, port)) - client.connect(nodes) - yield client - client.close() - - -@pytest.fixture -def examples(request): - return request.config.getoption("--examples") - @pytest.fixture(autouse=True) -def run_examples(request, examples): +def run_examples(request): + run_examples = request.config.getoption("--examples") if request.node.get_closest_marker('examples'): - if not examples: + if not run_examples: pytest.skip('skipped examples: --examples is not passed') @@ -213,103 +39,6 @@ def skip_if_no_cext(request): def pytest_addoption(parser): - parser.addoption( - '--node', - action='append', - default=None, - help=( - 'Ignite binary protocol test server connection string ' - '(default: "localhost:10801")' - ) - ) - parser.addoption( - '--timeout', - action='store', - type=float, - default=2.0, - help=( - 'Timeout (in seconds) for each socket operation. Can accept ' - 'integer or float value. Default is None' - ) - ) - parser.addoption( - '--partition-aware', - action=BoolParser, - nargs='?', - default=False, - help='Turn on the best effort affinity feature' - ) - parser.addoption( - '--use-ssl', - action=BoolParser, - nargs='?', - default=False, - help='Use SSL encryption' - ) - parser.addoption( - '--ssl-keyfile', - action='store', - default=None, - type=str, - help='a path to SSL key file to identify local party' - ) - parser.addoption( - '--ssl-keyfile-password', - action='store', - default=None, - type=str, - help='password for SSL key file' - ) - parser.addoption( - '--ssl-certfile', - action='store', - default=None, - type=str, - help='a path to ssl certificate file to identify local party' - ) - parser.addoption( - '--ssl-ca-certfile', - action='store', - default=None, - type=str, - help='a path to a trusted certificate or a certificate chain' - ) - parser.addoption( - '--ssl-cert-reqs', - action=CertReqsParser, - default=ssl.CERT_NONE, - help=( - 'determines how the remote side certificate is treated: ' - 'NONE (ignore, default), ' - 'OPTIONAL (validate, if provided) or ' - 'REQUIRED (valid remote certificate is required)' - ) - ) - parser.addoption( - '--ssl-ciphers', - action='store', - default=SSL_DEFAULT_CIPHERS, - type=str, - help='ciphers to use' - ) - parser.addoption( - '--ssl-version', - action=SSLVersionParser, - default=SSL_DEFAULT_VERSION, - help='SSL version: TLSV1_1 or TLSV1_2' - ) - parser.addoption( - '--username', - action='store', - type=str, - help='user name' - ) - parser.addoption( - '--password', - action='store', - type=str, - help='password' - ) parser.addoption( '--examples', action='store_true', @@ -322,38 +51,11 @@ def pytest_addoption(parser): ) -def pytest_generate_tests(metafunc): - session_parameters = { - 'node': ['{host}:{port}'.format(host='127.0.0.1', port=10801), - '{host}:{port}'.format(host='127.0.0.1', port=10802), - '{host}:{port}'.format(host='127.0.0.1', port=10803)], - 'timeout': None, - 'partition_aware': False, - 'use_ssl': False, - 'ssl_keyfile': None, - 'ssl_keyfile_password': None, - 'ssl_certfile': None, - 'ssl_ca_certfile': None, - 'ssl_cert_reqs': ssl.CERT_NONE, - 'ssl_ciphers': SSL_DEFAULT_CIPHERS, - 'ssl_version': SSL_DEFAULT_VERSION, - 'username': None, - 'password': None, - } - - for param_name in session_parameters: - if param_name in metafunc.fixturenames: - param = metafunc.config.getoption(param_name) - # TODO: This does not work for bool - if param is None: - param = session_parameters[param_name] - if param_name == 'node' or type(param) is not list: - param = [param] - metafunc.parametrize(param_name, param, scope='session') - - def pytest_configure(config): - config.addinivalue_line( - "markers", "examples: mark test to run only if --examples are set\n" - "skip_if_no_cext: mark test to run only if c extension is available" - ) + marker_docs = [ + "skip_if_no_cext: mark test to run only if c extension is available", + "examples: mark test to run only if --examples are set" + ] + + for marker_doc in marker_docs: + config.addinivalue_line("markers", marker_doc) diff --git a/tests/security/conftest.py b/tests/security/conftest.py new file mode 100644 index 0000000..d5de5a1 --- /dev/null +++ b/tests/security/conftest.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest + +from tests.util import get_test_dir + + +@pytest.fixture +def ssl_params(): + yield __create_ssl_param(False) + + +@pytest.fixture +def ssl_params_with_password(): + yield __create_ssl_param(True) + + +def __create_ssl_param(with_password=False): + cert_path = os.path.join(get_test_dir(), 'config', 'ssl') + + if with_password: + cert = os.path.join(cert_path, 'client_with_pass_full.pem') + return { + 'ssl_keyfile': cert, + 'ssl_keyfile_password': '654321', + 'ssl_certfile': cert, + 'ssl_ca_certfile': cert, + } + else: + cert = os.path.join(cert_path, 'client_full.pem') + return { + 'ssl_keyfile': cert, + 'ssl_certfile': cert, + 'ssl_ca_certfile': cert + } diff --git a/tests/security/test_auth.py b/tests/security/test_auth.py new file mode 100644 index 0000000..2dd19a0 --- /dev/null +++ b/tests/security/test_auth.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from pyignite.exceptions import AuthenticationError +from tests.util import start_ignite_gen, clear_ignite_work_dir, get_client + +DEFAULT_IGNITE_USERNAME = 'ignite' +DEFAULT_IGNITE_PASSWORD = 'ignite' + + +@pytest.fixture(params=['with-ssl', 'without-ssl']) +def with_ssl(request): + return request.param == 'with-ssl' + + +@pytest.fixture(autouse=True) +def server(with_ssl, cleanup): + yield from start_ignite_gen(use_ssl=with_ssl, use_auth=True) + + +@pytest.fixture(scope='module', autouse=True) +def cleanup(): + clear_ignite_work_dir() + yield None + clear_ignite_work_dir() + + +def test_auth_success(with_ssl, ssl_params): + ssl_params['use_ssl'] = with_ssl + + with get_client(username=DEFAULT_IGNITE_USERNAME, password=DEFAULT_IGNITE_PASSWORD, **ssl_params) as client: + client.connect("127.0.0.1", 10801) + + assert all(node.alive for node in client._nodes) + + +@pytest.mark.parametrize( + 'username, password', + [ + [DEFAULT_IGNITE_USERNAME, None], + ['invalid_user', 'invalid_password'], + [None, None] + ] +) +def test_auth_failed(username, password, with_ssl, ssl_params): + ssl_params['use_ssl'] = with_ssl + + with pytest.raises(AuthenticationError): + with get_client(username=username, password=password, **ssl_params) as client: + client.connect("127.0.0.1", 10801) diff --git a/tests/security/test_ssl.py b/tests/security/test_ssl.py new file mode 100644 index 0000000..6463a03 --- /dev/null +++ b/tests/security/test_ssl.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from pyignite.exceptions import ReconnectError +from tests.util import start_ignite_gen, get_client, get_or_create_cache + + +@pytest.fixture(scope='module', autouse=True) +def server(): + yield from start_ignite_gen(use_ssl=True, use_auth=False) + + +def test_connect_ssl_keystore_with_password(ssl_params_with_password): + __test_connect_ssl(**ssl_params_with_password) + + +def test_connect_ssl(ssl_params): + __test_connect_ssl(**ssl_params) + +def __test_connect_ssl(**kwargs): + kwargs['use_ssl'] = True + + with get_client(**kwargs) as client: + client.connect("127.0.0.1", 10801) + + with get_or_create_cache(client, 'test-cache') as cache: + cache.put(1, 1) + + assert cache.get(1) == 1 + + +@pytest.mark.parametrize( + 'invalid_ssl_params', + [ + {'use_ssl': False}, + {'use_ssl': True}, + {'use_ssl': True, 'ssl_keyfile': 'invalid.pem', 'ssl_certfile': 'invalid.pem'} + ] +) +def test_connection_error_with_incorrect_config(invalid_ssl_params): + with pytest.raises(ReconnectError): + with get_client(**invalid_ssl_params) as client: + client.connect([("127.0.0.1", 10801)]) diff --git a/tests/test_examples.py b/tests/test_examples.py index 046eb6d..f90ed17 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -12,40 +12,41 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import glob +import os import subprocess import sys import pytest +from tests.util import get_test_dir, start_ignite_gen SKIP_LIST = [ 'failover.py', # it hangs by design ] -def run_subprocess_34(script: str): - return subprocess.call([ - 'python', - '../examples/{}'.format(script), - ]) +def examples_scripts_gen(): + examples_dir = os.path.join(get_test_dir(), '..', 'examples') + for script in glob.glob1(examples_dir, '*.py'): + if script not in SKIP_LIST: + yield os.path.join(examples_dir, script) -def run_subprocess_35(script: str): - return subprocess.run([ - 'python', - '../examples/{}'.format(script), - ]).returncode +@pytest.fixture(autouse=True) +def server(): + yield from start_ignite_gen(idx=0) # idx=0, because 10800 port is needed for examples. @pytest.mark.examples -def test_examples(): - for script in glob.glob1('../examples', '*.py'): - if script not in SKIP_LIST: - # `subprocess` module was refactored in Python 3.5 - if sys.version_info >= (3, 5): - return_code = run_subprocess_35(script) - else: - return_code = run_subprocess_34(script) - assert return_code == 0 +@pytest.mark.parametrize( + 'example_script', + examples_scripts_gen() +) +def test_examples(example_script): + proc = subprocess.run([ + sys.executable, + example_script + ]) + + assert proc.returncode == 0 diff --git a/tests/util.py b/tests/util.py index 90f0146..af4c324 100644 --- a/tests/util.py +++ b/tests/util.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import contextlib import glob import os +import shutil import jinja2 as jinja2 import psutil @@ -23,6 +24,26 @@ import subprocess import time +from pyignite import Client + + +@contextlib.contextmanager +def get_client(**kwargs): + client = Client(**kwargs) + try: + yield client + finally: + client.close() + + +@contextlib.contextmanager +def get_or_create_cache(client, cache_name): + cache = client.get_or_create_cache(cache_name) + try: + yield cache + finally: + cache.destroy() + def wait_for_condition(condition, interval=0.1, timeout=10, error=None): start = time.time() @@ -111,7 +132,7 @@ def create_config_file(tpl_name, file_name, **kwargs): f.write(template.render(**kwargs)) -def _start_ignite(idx=1, debug=False, use_ssl=False): +def start_ignite(idx=1, debug=False, use_ssl=False, use_auth=False): clear_logs(idx) runner = get_ignite_runner() @@ -122,7 +143,8 @@ def _start_ignite(idx=1, debug=False, use_ssl=False): env["JVM_OPTS"] = "-Djava.net.preferIPv4Stack=true -Xdebug -Xnoagent -Djava.compiler=NONE " \ "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 " - params = {'ignite_instance_idx': str(idx), 'ignite_client_port': 10800 + idx, 'use_ssl': use_ssl} + params = {'ignite_instance_idx': str(idx), 'ignite_client_port': 10800 + idx, 'use_ssl': use_ssl, + 'use_auth': use_auth} create_config_file('log4j.xml.jinja2', f'log4j-{idx}.xml', **params) create_config_file('ignite-config.xml.jinja2', f'ignite-config-{idx}.xml', **params) @@ -140,10 +162,12 @@ def _start_ignite(idx=1, debug=False, use_ssl=False): raise Exception("Failed to start Ignite: timeout while trying to connect") -def start_ignite_gen(idx=1, use_ssl=False): - srv = _start_ignite(idx, use_ssl=use_ssl) - yield srv - kill_process_tree(srv.pid) +def start_ignite_gen(idx=1, use_ssl=False, use_auth=False): + srv = start_ignite(idx, use_ssl=use_ssl, use_auth=use_auth) + try: + yield srv + finally: + kill_process_tree(srv.pid) def get_log_files(idx=1): @@ -151,6 +175,13 @@ def get_log_files(idx=1): return glob.glob(logs_pattern) +def clear_ignite_work_dir(): + for ignite_dir in get_ignite_dirs(): + work_dir = os.path.join(ignite_dir, 'work') + if os.path.exists(work_dir): + shutil.rmtree(work_dir, ignore_errors=True) + + def clear_logs(idx=1): for f in get_log_files(idx): os.remove(f) diff --git a/tox.ini b/tox.ini index 104a705..3ab8dea 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ [tox] skipsdist = True -envlist = py{36,37,38}-{no-ssl,ssl,ssl-password} +envlist = py{36,37,38,39} [testenv] passenv = TEAMCITY_VERSION IGNITE_HOME @@ -26,44 +26,8 @@ deps = recreate = True usedevelop = True commands = - pytest {env:PYTESTARGS:} {posargs} --force-cext + pytest {env:PYTESTARGS:} {posargs} --force-cext --examples -[jenkins] +[testenv:py{36,37,38,39}-jenkins] setenv: PYTESTARGS = --junitxml=junit-{envname}.xml - -[no-ssl] -setenv: - PYTEST_ADDOPTS = --examples - -[ssl] -setenv: - PYTEST_ADDOPTS = --examples --use-ssl=True --ssl-certfile={toxinidir}/tests/config/ssl/client_full.pem --ssl-version=TLSV1_2 - -[ssl-password] -setenv: - PYTEST_ADDOPTS = --examples --use-ssl=True --ssl-certfile={toxinidir}/tests/config/ssl/client_with_pass_full.pem --ssl-keyfile-password=654321 --ssl-version=TLSV1_2 - -[testenv:py{36,37,38}-no-ssl] -setenv: {[no-ssl]setenv} - -[testenv:py{36,37,38}-ssl] -setenv: {[ssl]setenv} - -[testenv:py{36,37,38}-ssl-password] -setenv: {[ssl-password]setenv} - -[testenv:py{36,37,38}-jenkins-no-ssl] -setenv: - {[no-ssl]setenv} - {[jenkins]setenv} - -[testenv:py{36,37,38}-jenkins-ssl] -setenv: - {[ssl]setenv} - {[jenkins]setenv} - -[testenv:py{36,37,38}-jenkins-ssl-password] -setenv: - {[ssl-password]setenv} - {[jenkins]setenv}