diff --git a/README.rst b/README.rst index 97bdb6c..1b0c8aa 100644 --- a/README.rst +++ b/README.rst @@ -315,19 +315,7 @@ Kwargs set properties: Testing ======= The main test mechanism for the package `exec-helpers` is using `tox`. -Test environments available: - -:: - - pep8 - py27 - py34 - py35 - py36 - pypy - pypy3 - pylint - pep257 +Available environments can be collected via `tox -l` CI systems ========== diff --git a/doc/source/SSHClient.rst b/doc/source/SSHClient.rst index 61d8834..4875ea9 100644 --- a/doc/source/SSHClient.rst +++ b/doc/source/SSHClient.rst @@ -118,7 +118,7 @@ API: SSHClient and SSHAuth. :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>' :type log_mask_re: typing.Optional[str] - :rtype: ``typing.Tuple[paramiko.Channel, paramiko.ChannelFile, paramiko.ChannelFile, paramiko.ChannelFile]`` + :rtype: ``typing.Tuple[paramiko.Channel, paramiko.ChannelFile, typing.Optional[paramiko.ChannelFile], typing.Optional[paramiko.ChannelFile]]`` .. versionchanged:: 1.2.0 open_stdout and open_stderr flags .. versionchanged:: 1.2.0 stdin data diff --git a/doc/source/Subprocess.rst b/doc/source/Subprocess.rst index eb0116b..bf3bfd8 100644 --- a/doc/source/Subprocess.rst +++ b/doc/source/Subprocess.rst @@ -56,7 +56,7 @@ API: Subprocess :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>' :type log_mask_re: typing.Optional[str] - :rtype: ``typing.Tuple[subprocess.Popen, None, typing.Optional[io.TextIOWrapper], typing.Optional[io.TextIOWrapper], ]`` + :rtype: ``typing.Tuple[subprocess.Popen, None, typing.Optional[typing.IO], typing.Optional[typing.IO], ]`` .. versionadded:: 1.2.0 diff --git a/exec_helpers/_api.py b/exec_helpers/_api.py index 77cb01f..3283474 100644 --- a/exec_helpers/_api.py +++ b/exec_helpers/_api.py @@ -25,7 +25,7 @@ import logging import re import threading -import typing # noqa # pylint: disable=unused-import +import typing import six # noqa # pylint: disable=unused-import @@ -35,6 +35,9 @@ from exec_helpers import proc_enums from exec_helpers import _log_templates +_type_exit_codes = typing.Union[int, proc_enums.ExitCodes] +_type_expected = typing.Optional[typing.Iterable[_type_exit_codes]] + class ExecHelper(object): """ExecHelper global API.""" @@ -171,9 +174,9 @@ def execute_async( def _exec_command( self, command, # type: str - interface, # type: typing.Any, - stdout, # type: typing.Any, - stderr, # type: typing.Any, + interface, # type: typing.Any + stdout, # type: typing.Any + stderr, # type: typing.Any timeout, # type: int verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] @@ -227,10 +230,10 @@ def execute( """ with self.lock: ( - iface, # type: typing.Any + iface, _, - stderr, # type: typing.Any - stdout, # type: typing.Any + stderr, + stdout, ) = self.execute_async( command, verbose=verbose, diff --git a/exec_helpers/_ssh_client_base.py b/exec_helpers/_ssh_client_base.py index 4e83dd2..d9a24a7 100644 --- a/exec_helpers/_ssh_client_base.py +++ b/exec_helpers/_ssh_client_base.py @@ -65,8 +65,8 @@ _type_execute_async = typing.Tuple[ paramiko.Channel, paramiko.ChannelFile, - paramiko.ChannelFile, - paramiko.ChannelFile + typing.Optional[paramiko.ChannelFile], + typing.Optional[paramiko.ChannelFile] ] CPYTHON = 'CPython' == platform.python_implementation() @@ -101,7 +101,7 @@ class _MemorizedSSH(type): duplicates is possible. """ - __cache = {} + __cache = {} # type: typing.Dict[typing.Tuple[str, int], SSHClientBase] @classmethod def __prepare__( @@ -622,7 +622,7 @@ def poll_pipes( :type stop: Event :type channel: paramiko.channel.Channel """ - while not stop.isSet(): + while not stop.is_set(): time.sleep(0.1) if stdout or stderr: poll_streams(result=result) @@ -662,7 +662,7 @@ def poll_pipes( concurrent.futures.wait([future], timeout) # Process closed? - if stop_event.isSet(): + if stop_event.is_set(): stop_event.clear() interface.close() return result @@ -764,7 +764,7 @@ def execute_together( remotes, # type: typing.Iterable[SSHClientBase] command, # type: str timeout=constants.DEFAULT_TIMEOUT, # type: typing.Optional[int] - expected=None, # type: typing.Optional[typing.Iterable[]] + expected=None, # type: typing.Optional[typing.Iterable[int]] raise_on_err=True, # type: bool **kwargs ): # type: (...) -> _type_multiple_results @@ -782,10 +782,8 @@ def execute_together( :type raise_on_err: bool :return: dictionary {(hostname, port): result} :rtype: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] - :raises ParallelCallProcessError: - Unexpected any code at lest on one target - :raises ParallelCallExceptions: - At lest one exception raised during execution (including timeout) + :raises ParallelCallProcessError: Unexpected any code at lest on one target + :raises ParallelCallExceptions: At lest one exception raised during execution (including timeout) .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd @@ -796,11 +794,14 @@ def get_result( ): # type: (...) -> exec_result.ExecResult """Get result from remote call.""" ( - chan, # type: paramiko.channel.Channel + chan, _, - stderr, # type: paramiko.channel.ChannelFile - stdout, # type: paramiko.channel.ChannelFile - ) = remote.execute_async(command, **kwargs) + stderr, + stdout, + ) = remote.execute_async( + command, + **kwargs + ) # type: _type_execute_async chan.status_event.wait(timeout) exit_code = chan.recv_exit_status() @@ -832,19 +833,19 @@ def get_result( futures[remote] = get_result(remote) ( - _, # type: typing.Set[concurrent.futures.Future] - not_done, # type: typing.Set[concurrent.futures.Future] + _, + not_done, ) = concurrent.futures.wait( list(futures.values()), timeout=timeout - ) + ) # type: typing.Set[concurrent.futures.Future], typing.Set[concurrent.futures.Future] for future in not_done: future.cancel() for ( - remote, # type: SSHClientBase - future, # type: concurrent.futures.Future - ) in futures.items(): + remote, + future, + ) in futures.items(): # type: SSHClientBase, concurrent.futures.Future try: result = future.result() results[(remote.hostname, remote.port)] = result diff --git a/exec_helpers/exceptions.py b/exec_helpers/exceptions.py index ac714a2..28a486e 100644 --- a/exec_helpers/exceptions.py +++ b/exec_helpers/exceptions.py @@ -71,7 +71,7 @@ def __init__( self, result=None, # type: exec_result.ExecResult expected=None, # type: typing.Optional[typing.List[_type_exit_codes]] - ): + ): # type: (...) -> None """Exception for error on process calls. :param result: execution result @@ -136,10 +136,10 @@ def __init__( self, command, # type: str exceptions, # type: typing.Dict[typing.Tuple[str, int], Exception] - errors, # type: _type_multiple_results, - results, # type: _type_multiple_results, + errors, # type: _type_multiple_results + results, # type: _type_multiple_results expected=None, # type: typing.Optional[typing.List[_type_exit_codes]] - ): + ): # type: (...) -> None """Exception raised during parallel call as result of exceptions. :param command: command @@ -190,10 +190,10 @@ class ParallelCallProcessError(ExecCalledProcessError): def __init__( self, command, # type: str - errors, # type: _type_multiple_results, - results, # type: _type_multiple_results, + errors, # type: _type_multiple_results + results, # type: _type_multiple_results expected=None, # type: typing.Optional[typing.List[_type_exit_codes]] - ): + ): # type: (...) -> None """Exception during parallel execution. :param command: command diff --git a/exec_helpers/exec_result.py b/exec_helpers/exec_result.py index a14499f..64b66e3 100644 --- a/exec_helpers/exec_result.py +++ b/exec_helpers/exec_result.py @@ -55,7 +55,7 @@ def __init__( stdout=None, # type: typing.Optional[typing.Iterable[bytes]] stderr=None, # type: typing.Optional[typing.Iterable[bytes]] exit_code=proc_enums.ExitCodes.EX_INVALID # type: _type_exit_codes - ): + ): # type: (...) -> None """Command execution result. :param cmd: command @@ -77,8 +77,8 @@ def __init__( elif isinstance(stdin, bytearray): stdin = self._get_str_from_bin(stdin) self.__stdin = stdin - self.__stdout = tuple(stdout) if stdout is not None else () - self.__stderr = tuple(stderr) if stderr is not None else () + self.__stdout = tuple(stdout) if stdout is not None else () # type: typing.Tuple[bytes] + self.__stderr = tuple(stderr) if stderr is not None else () # type: typing.Tuple[bytes] self.__exit_code = None self.__timestamp = None @@ -99,7 +99,7 @@ def lock(self): # type: () -> threading.RLock return self.__lock @property - def timestamp(self): # type: () -> typing.Optional(datetime.datetime) + def timestamp(self): # type: () -> typing.Optional[datetime.datetime] """Timestamp. :rtype: typing.Optional(datetime.datetime) @@ -133,10 +133,10 @@ def _get_str_from_bin(src): # type: (bytearray) -> str def _get_brief(cls, data): # type: (typing.Tuple[bytes]) -> str """Get brief output: 7 lines maximum (3 first + ... + 3 last). - :type data: typing.List[bytes] + :type data: typing.Tuple[bytes] :rtype: str """ - src = data if len(data) <= 7 else data[:3] + (b'...\n',) + data[-3:] + src = data if len(data) <= 7 else data[:3] + (b'...\n',) + data[-3:] # type: typing.Tuple[bytes] return cls._get_str_from_bin( cls._get_bytearray_from_array(src) ) diff --git a/exec_helpers/proc_enums.py b/exec_helpers/proc_enums.py index df25a5c..ce0a533 100644 --- a/exec_helpers/proc_enums.py +++ b/exec_helpers/proc_enums.py @@ -169,7 +169,7 @@ def exit_code_to_enum(code): # type: (_type_exit_codes) -> _type_exit_codes def exit_codes_to_enums( - codes=None # type: typing.Optional[typing.List[_type_exit_codes]] + codes=None # type: typing.Optional[typing.Iterable[_type_exit_codes]] ): # type: (...) -> typing.List[_type_exit_codes] """Convert integer exit codes to enums.""" if codes is None: diff --git a/exec_helpers/ssh_auth.py b/exec_helpers/ssh_auth.py index 6d128d9..6d751eb 100644 --- a/exec_helpers/ssh_auth.py +++ b/exec_helpers/ssh_auth.py @@ -52,10 +52,10 @@ def __init__( username=None, # type: typing.Optional[str] password=None, # type: typing.Optional[str] key=None, # type: typing.Optional[paramiko.RSAKey] - keys=None, # type: typing.Optional[typing.Iterable[paramiko.RSAKey]], + keys=None, # type: typing.Optional[typing.Iterable[paramiko.RSAKey]] key_filename=None, # type: typing.Union[typing.List[str], str, None] passphrase=None, # type: typing.Optional[str] - ): + ): # type: (...) -> None """SSH credentials object. Used to authorize SSHClient. @@ -137,7 +137,7 @@ def enter_password(self, tgt): # type: (io.StringIO) -> None :type tgt: file """ # noinspection PyTypeChecker - return tgt.write('{}\n'.format(self.__password)) + tgt.write('{}\n'.format(self.__password)) def connect( self, @@ -166,7 +166,7 @@ def connect( 'password': self.__password, 'key_filename': self.key_filename, 'passphrase': self.__passphrase, - } + } # type: typing.Dict[str, typing.Any] if hostname is not None: kwargs['hostname'] = hostname kwargs['port'] = port diff --git a/exec_helpers/subprocess_runner.py b/exec_helpers/subprocess_runner.py index 4de232d..1059707 100644 --- a/exec_helpers/subprocess_runner.py +++ b/exec_helpers/subprocess_runner.py @@ -22,7 +22,6 @@ import collections import errno -import io import logging import os import select @@ -30,7 +29,7 @@ import subprocess # nosec # Expected usage import threading import time -import typing +import typing # noqa # pylint: disable=unused-import import six import threaded @@ -44,13 +43,6 @@ # noinspection PyUnresolvedReferences devnull = open(os.devnull) # subprocess.DEVNULL is py3.3+ -_type_execute_async = typing.Tuple[ - subprocess.Popen, - None, - typing.Optional[io.TextIOWrapper], - typing.Optional[io.TextIOWrapper], -] - _win = sys.platform == "win32" _posix = 'posix' in sys.builtin_module_names @@ -71,7 +63,7 @@ class SingletonMeta(type): Main goals: not need to implement __new__ in singleton classes """ - _instances = {} + _instances = {} # type: typing.Dict[typing.Type, typing.Any] _lock = threading.RLock() def __call__(cls, *args, **kwargs): @@ -97,7 +89,7 @@ def __prepare__( return collections.OrderedDict() # pragma: no cover -def set_nonblocking_pipe(pipe): # type: (os.pipe) -> None +def set_nonblocking_pipe(pipe): # type: (typing.Any) -> None """Set PIPE unblocked to allow polling of all pipes in parallel.""" descriptor = pipe.fileno() # pragma: no cover @@ -154,9 +146,9 @@ def __init__( def _exec_command( self, command, # type: str - interface, # type: subprocess.Popen, - stdout, # type: typing.Optional[io.TextIOWrapper], - stderr, # type: typing.Optional[io.TextIOWrapper], + interface, # type: subprocess.Popen + stdout, # type: typing.Optional[typing.IO] + stderr, # type: typing.Optional[typing.IO] timeout, # type: int verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] @@ -211,7 +203,7 @@ def poll_streams( verbose=verbose ) - @threaded.threaded(started=True, daemon=True) + @threaded.threaded(started=True) def poll_pipes( result, # type: exec_result.ExecResult stop, # type: threading.Event @@ -221,7 +213,7 @@ def poll_pipes( :type result: ExecResult :type stop: Event """ - while not stop.isSet(): + while not stop.is_set(): time.sleep(0.1) if stdout or stderr: poll_streams(result=result) @@ -262,7 +254,8 @@ def poll_pipes( stop_event.wait(timeout) # Process closed? - if stop_event.isSet(): + if stop_event.is_set(): + poll_thread.join(0.1) stop_event.clear() return result # Kill not ended process and wait for close @@ -295,7 +288,7 @@ def execute_async( verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] **kwargs - ): # type: (...) -> _type_execute_async + ): # type: (...) -> typing.Tuple[subprocess.Popen, None, typing.Optional[typing.IO], typing.Optional[typing.IO], ] """Execute command in async mode and return Popen with IO objects. :param command: Command for execution @@ -314,8 +307,8 @@ def execute_async( :rtype: typing.Tuple[ subprocess.Popen, None, - typing.Optional[io.TextIOWrapper], - typing.Optional[io.TextIOWrapper], + typing.Optional[typing.IO], + typing.Optional[typing.IO], ] .. versionadded:: 1.2.0 diff --git a/setup.py b/setup.py index e145e15..0ac3cbb 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ def _extension(modpath): binding=True, embedsignature=True, overflowcheck=True, + language_level=3, ) ) if cythonize is not None else () diff --git a/test/test_ssh_client.py b/test/test_ssh_client.py index 95c338e..d466a2f 100644 --- a/test/test_ssh_client.py +++ b/test/test_ssh_client.py @@ -924,7 +924,7 @@ def test_execute_timeout( logger.reset_mock() # noinspection PyTypeChecker - result = ssh.execute(command=command, verbose=False, timeout=1) + result = ssh.execute(command=command, verbose=False, timeout=0.1) self.assertEqual( result, @@ -960,7 +960,7 @@ def test_execute_timeout_fail( with self.assertRaises(exec_helpers.ExecHelperTimeoutError): # noinspection PyTypeChecker - ssh.execute(command=command, verbose=False, timeout=1) + ssh.execute(command=command, verbose=False, timeout=0.1) execute_async.assert_called_once_with(command, verbose=False) chan.assert_has_calls((mock.call.status_event.is_set(), )) diff --git a/test/test_subprocess_runner.py b/test/test_subprocess_runner.py index 997d2fc..e241e7d 100644 --- a/test/test_subprocess_runner.py +++ b/test/test_subprocess_runner.py @@ -50,9 +50,6 @@ def fileno(self): return hash(tuple(self.__src)) -# TODO(AStepanov): Cover negative scenarios (timeout) - - @mock.patch('exec_helpers.subprocess_runner.logger', autospec=True) @mock.patch('select.select', autospec=True) @mock.patch('exec_helpers.subprocess_runner.set_nonblocking_pipe', autospec=True) @@ -125,10 +122,7 @@ def test_001_call( # noinspection PyTypeChecker result = runner.execute(command) - self.assertEqual( - result, exp_result - - ) + self.assertEqual(result, exp_result) popen.assert_has_calls(( mock.call( args=[command], @@ -389,10 +383,7 @@ def test_008_execute_mask_global(self, popen, _, select, logger): # noinspection PyTypeChecker result = runner.execute(cmd) - self.assertEqual( - result, exp_result - - ) + self.assertEqual(result, exp_result) popen.assert_has_calls(( mock.call( args=[cmd], @@ -445,10 +436,7 @@ def test_009_execute_mask_local(self, popen, _, select, logger): # noinspection PyTypeChecker result = runner.execute(cmd, log_mask_re=log_mask_re) - self.assertEqual( - result, exp_result - - ) + self.assertEqual(result, exp_result) popen.assert_has_calls(( mock.call( args=[cmd], @@ -503,10 +491,8 @@ def test_004_check_stdin_str( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result + self.assertEqual(result, exp_result) - ) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -544,10 +530,8 @@ def test_005_check_stdin_bytes( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result + self.assertEqual(result, exp_result) - ) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -585,10 +569,8 @@ def test_006_check_stdin_bytearray( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result + self.assertEqual(result, exp_result) - ) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -631,10 +613,8 @@ def test_007_check_stdin_fail_broken_pipe( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result + self.assertEqual(result, exp_result) - ) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -677,10 +657,7 @@ def test_008_check_stdin_fail_closed_win( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result - - ) + self.assertEqual(result, exp_result) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -722,7 +699,7 @@ def test_009_check_stdin_fail_write( with self.assertRaises(OSError): # noinspection PyTypeChecker - runner.execute(print_stdin, stdin=stdin) + runner.execute_async(print_stdin, stdin=stdin) popen_obj.kill.assert_called_once() @unittest.skipIf(six.PY2, 'Not implemented exception') @@ -749,10 +726,8 @@ def test_010_check_stdin_fail_close_pipe( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result + self.assertEqual(result, exp_result) - ) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -795,10 +770,7 @@ def test_011_check_stdin_fail_close_pipe_win( # noinspection PyTypeChecker result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual( - result, exp_result - - ) + self.assertEqual(result, exp_result) popen.assert_has_calls(( mock.call( args=[print_stdin], @@ -840,7 +812,7 @@ def test_012_check_stdin_fail_close( with self.assertRaises(OSError): # noinspection PyTypeChecker - runner.execute(print_stdin, stdin=stdin) + runner.execute_async(print_stdin, stdin=stdin) popen_obj.kill.assert_called_once() @mock.patch('time.sleep', autospec=True) @@ -858,7 +830,7 @@ def test_013_execute_timeout_done( # noinspection PyTypeChecker - res = runner.execute(command, timeout=1) + res = runner.execute(command, timeout=0.1) self.assertEqual(res, exp_result) @@ -877,8 +849,8 @@ def test_013_execute_timeout_done( @mock.patch('exec_helpers.subprocess_runner.logger', autospec=True) +@mock.patch('exec_helpers.subprocess_runner.Subprocess.execute') class TestSubprocessRunnerHelpers(unittest.TestCase): - @mock.patch('exec_helpers.subprocess_runner.Subprocess.execute') def test_001_check_call(self, execute, logger): exit_code = 0 return_value = exec_helpers.ExecResult( @@ -913,7 +885,6 @@ def test_001_check_call(self, execute, logger): runner.check_call(command=command, verbose=verbose, timeout=None) execute.assert_called_once_with(command, verbose, None) - @mock.patch('exec_helpers.subprocess_runner.Subprocess.execute') def test_002_check_call_expected(self, execute, logger): exit_code = 0 return_value = exec_helpers.ExecResult( @@ -952,7 +923,7 @@ def test_002_check_call_expected(self, execute, logger): execute.assert_called_once_with(command, verbose, None) @mock.patch('exec_helpers.subprocess_runner.Subprocess.check_call') - def test_003_check_stderr(self, check_call, logger): + def test_003_check_stderr(self, check_call, _, logger): return_value = exec_helpers.ExecResult( cmd=command, stdout=stdout_list, diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index f1460c7..53a19a5 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,5 +1,5 @@ #!/bin/bash -PYTHON_VERSIONS="cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m" +PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" # Avoid creation of __pycache__/*.py[c|o] export PYTHONDONTWRITEBYTECODE=1 diff --git a/tox.ini b/tox.ini index bee2e03..dd4f1fb 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] minversion = 2.0 -envlist = pep8, pep257, py{27,34,35,36,py,py3}, pylint, bandit, py{27,34,35,36}-nocov, docs, +envlist = pep8, pep257, py{27,34,35,36,py,py3}, pylint, bandit, py{34,35,36}-nocov, docs, skipsdist = True skip_missing_interpreters = True @@ -28,13 +28,6 @@ commands = py.test -vv --junitxml=unit_result.xml --html=report.html --cov-config .coveragerc --cov-report html --cov=exec_helpers {posargs:test} coverage report --fail-under 97 -[testenv:py27-nocov] -usedevelop = False -commands = - python setup.py bdist_wheel - pip install exec_helpers --no-index -f dist - py.test -vv {posargs:test} - [testenv:py34-nocov] usedevelop = False commands = @@ -61,9 +54,9 @@ commands = {posargs:} [tox:travis] 2.7 = install, py27, -3.4 = install, py34, -3.5 = install, py35, -3.6 = install, py36, +3.4 = py34, +3.5 = py35, +3.6 = py36, pypy = install, pypy, pypy3 = install, pypy3,