From 107f2854b8342a9956c61b304f480a314a2469cb Mon Sep 17 00:00:00 2001 From: Alexey Stepanov Date: Mon, 3 Sep 2018 17:43:00 +0200 Subject: [PATCH 1/3] Old is for py27 only. Backport typehints from validated new version * no *.pyi * no cythonize support code * static analisys from python 3.6 without sdist (because of mypy) Signed-off-by: Alexey Stepanov --- .travis.yml | 8 +- README.rst | 21 ++-- doc/source/ExecResult.rst | 8 +- doc/source/SSHClient.rst | 12 +-- doc/source/Subprocess.rst | 8 +- doc/source/exceptions.rst | 2 +- exec_helpers/_ssh_client_base.py | 141 +++++++++++++------------ exec_helpers/_ssh_client_base.pyi | 163 ----------------------------- exec_helpers/api.py | 58 +++++----- exec_helpers/api.pyi | 74 ------------- exec_helpers/exceptions.py | 5 +- exec_helpers/exceptions.pyi | 81 -------------- exec_helpers/exec_result.py | 77 ++++++++------ exec_helpers/exec_result.pyi | 96 ----------------- exec_helpers/ssh_auth.py | 17 ++- exec_helpers/ssh_auth.pyi | 45 -------- exec_helpers/ssh_client.py | 8 +- exec_helpers/subprocess_runner.py | 33 +++--- exec_helpers/subprocess_runner.pyi | 63 ----------- setup.py | 126 +--------------------- tools/build-wheels.sh | 64 ----------- tools/run_docker.sh | 32 ------ tox.ini | 14 +-- 23 files changed, 212 insertions(+), 944 deletions(-) delete mode 100644 exec_helpers/_ssh_client_base.pyi delete mode 100644 exec_helpers/api.pyi delete mode 100644 exec_helpers/exceptions.pyi delete mode 100644 exec_helpers/exec_result.pyi delete mode 100644 exec_helpers/ssh_auth.pyi delete mode 100644 exec_helpers/subprocess_runner.pyi delete mode 100755 tools/build-wheels.sh delete mode 100755 tools/run_docker.sh diff --git a/.travis.yml b/.travis.yml index a2bde38..63f00ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ language: python os: linux python: - &main_python 2.7 -- 3.4 - &pypy pypy install: - &upgrade_python_toolset pip install --upgrade pip setuptools wheel @@ -17,7 +16,7 @@ jobs: fast_finish: true include: - stage: Static analisys - python: 3.4 + python: 3.6 services: [] install: - *upgrade_python_toolset @@ -27,7 +26,7 @@ jobs: after_success: skip - stage: Code style check - python: 3.4 + python: *main_python install: - *upgrade_python_toolset - pip install tox @@ -43,8 +42,7 @@ jobs: services: [] install: - *upgrade_python_toolset - script: - - ./tools/run_docker.sh "exec_helpers" + script: [] before_deploy: - pip install -r build_requirements.txt - python setup.py bdist_wheel diff --git a/README.rst b/README.rst index a3e04bb..9562dbe 100644 --- a/README.rst +++ b/README.rst @@ -43,10 +43,9 @@ Pros: :: Python 2.7 - Python 3.4 PyPy -.. note:: Update to version 2.0+ for usage with python 3.5+. This version is for legacy python and no new features are planned. +.. note:: Update to version 2.0+ for usage with python 3.4+. This version is for legacy python and no new features are planned. This package includes: @@ -127,7 +126,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif result = helper.execute( command, # type: str verbose=False, # type: bool - timeout=1 * 60 * 60, # type: typing.Optional[int] + timeout=1 * 60 * 60, # type: typing.Union[int, float, None] **kwargs ) @@ -137,7 +136,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif result = helper.check_call( command, # type: str verbose=False, # type: bool - timeout=1 * 60 * 60, # type: typing.Optional[int] + timeout=1 * 60 * 60, # type: type: typing.Union[int, float, None] error_info=None, # type: typing.Optional[str] expected=None, # type: typing.Optional[typing.Iterable[int]] raise_on_err=True, # type: bool @@ -149,7 +148,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif result = helper.check_stderr( command, # type: str verbose=False, # type: bool - timeout=1 * 60 * 60, # type: typing.Optional[int] + timeout=1 * 60 * 60, # type: type: typing.Union[int, float, None] error_info=None, # type: typing.Optional[str] raise_on_err=True, # type: bool ) @@ -185,10 +184,10 @@ Execution result object has a set of useful properties: * `stderr` -> `typing.Tuple[bytes]`. Raw stderr output. * `stdout_bin` -> `bytearray`. Binary stdout output. * `stderr_bin` -> `bytearray`. Binary stderr output. -* `stdout_str` -> `six.text_types`. Text representation of output. -* `stderr_str` -> `six.text_types`. Text representation of output. -* `stdout_brief` -> `six.text_types`. Up to 7 lines from stdout (3 first and 3 last if >7 lines). -* `stderr_brief` -> `six.text_types`. Up to 7 lines from stderr (3 first and 3 last if >7 lines). +* `stdout_str` -> `typing.Text`. Text representation of output. +* `stderr_str` -> `typing.Text`. Text representation of output. +* `stdout_brief` -> `typing.Text`. Up to 7 lines from stdout (3 first and 3 last if >7 lines). +* `stderr_brief` -> `typing.Text`. Up to 7 lines from stderr (3 first and 3 last if >7 lines). * `stdout_json` - STDOUT decoded as JSON. @@ -209,7 +208,7 @@ Possible to call commands in parallel on multiple hosts if it's not produce huge results = SSHClient.execute_together( remotes, # type: typing.Iterable[SSHClient] command, # type: str - timeout=1 * 60 * 60, # type: typing.Optional[int] + timeout=1 * 60 * 60, # type: type: typing.Union[int, float, None] expected=None, # type: typing.Optional[typing.Iterable[int]] raise_on_err=True # type: bool ) @@ -227,7 +226,7 @@ For execute through SSH host can be used `execute_through_host` method: command, # type: str auth=None, # type: typing.Optional[SSHAuth] target_port=22, # type: int - timeout=1 * 60 * 60, # type: typing.Optional[int] + timeout=1 * 60 * 60, # type: type: typing.Union[int, float, None] verbose=False, # type: bool get_pty=False, # type: bool ) diff --git a/doc/source/ExecResult.rst b/doc/source/ExecResult.rst index fb8362c..66ffe31 100644 --- a/doc/source/ExecResult.rst +++ b/doc/source/ExecResult.rst @@ -15,7 +15,7 @@ API: ExecResult :param cmd: command :type cmd: ``str`` :param stdin: STDIN - :type stdin: ``typing.Optional[str]`` + :type stdin: ``typing.Union[bytes, str, bytearray, None]`` :param stdout: binary STDOUT :type stdout: ``typing.Optional[typing.Iterable[bytes]]`` :param stderr: binary STDERR @@ -40,17 +40,17 @@ API: ExecResult .. py:attribute:: stdin - ``typing.Text`` + ``typing.Optional[str]`` Stdin input as string. .. py:attribute:: stdout - ``typing.Tuple[bytes]`` + ``typing.Tuple[bytes, ...]`` Stdout output as list of binaries. .. py:attribute:: stderr - ``typing.Tuple[bytes]`` + ``typing.Tuple[bytes, ...]`` Stderr output as list of binaries. .. py:attribute:: stdout_bin diff --git a/doc/source/SSHClient.rst b/doc/source/SSHClient.rst index 37cf9ac..895ea71 100644 --- a/doc/source/SSHClient.rst +++ b/doc/source/SSHClient.rst @@ -128,7 +128,7 @@ API: SSHClient and SSHAuth. :param command: Command for execution :type command: ``str`` :param stdin: pass STDIN text to the process - :type stdin: ``typing.Union[typing.AnyStr, bytearray, None]`` + :type stdin: ``typing.Union[str, bytes, bytearray, None]`` :param open_stdout: open STDOUT stream for read :type open_stdout: bool :param open_stderr: open STDERR stream for read @@ -153,7 +153,7 @@ API: SSHClient and SSHAuth. :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :rtype: ExecResult :raises ExecHelperTimeoutError: Timeout exceeded @@ -168,7 +168,7 @@ API: SSHClient and SSHAuth. :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param error_info: Text for error details, if fail happens :type error_info: ``typing.Optional[str]`` :param expected: expected return codes (0 by default) @@ -190,7 +190,7 @@ API: SSHClient and SSHAuth. :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param error_info: Text for error details, if fail happens :type error_info: ``typing.Optional[str]`` :param raise_on_err: Raise exception on unexpected return code @@ -217,7 +217,7 @@ API: SSHClient and SSHAuth. :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param get_pty: open PTY on target machine :type get_pty: ``bool`` :rtype: ExecResult @@ -234,7 +234,7 @@ API: SSHClient and SSHAuth. :param command: Command for execution :type command: ``str`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param expected: expected return codes (0 by default) :type expected: ``typing.Optional[typing.Iterable[]]`` :param raise_on_err: Raise exception on unexpected return code diff --git a/doc/source/Subprocess.rst b/doc/source/Subprocess.rst index d9b76a5..9f6d10a 100644 --- a/doc/source/Subprocess.rst +++ b/doc/source/Subprocess.rst @@ -46,7 +46,7 @@ API: Subprocess :param command: Command for execution :type command: str :param stdin: pass STDIN text to the process - :type stdin: ``typing.Union[typing.AnyStr, bytearray, None]`` + :type stdin: ``typing.Union[str, bytes, bytearray, None]`` :param open_stdout: open STDOUT stream for read :type open_stdout: ``bool`` :param open_stderr: open STDERR stream for read @@ -69,7 +69,7 @@ API: Subprocess :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :rtype: ExecResult :raises ExecHelperTimeoutError: Timeout exceeded @@ -88,7 +88,7 @@ API: Subprocess :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param error_info: Text for error details, if fail happens :type error_info: ``typing.Optional[str]`` :param expected: expected return codes (0 by default) @@ -111,7 +111,7 @@ API: Subprocess :param verbose: Produce log.info records for command call and output :type verbose: ``bool`` :param timeout: Timeout for command execution. - :type timeout: ``typing.Union[int, None]`` + :type timeout: ``typing.Union[int, float, None]`` :param error_info: Text for error details, if fail happens :type error_info: ``typing.Optional[str]`` :param raise_on_err: Raise exception on unexpected return code diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst index 5d50d24..4226a72 100644 --- a/doc/source/exceptions.rst +++ b/doc/source/exceptions.rst @@ -36,7 +36,7 @@ API: exceptions .. py:attribute:: timeout - ``int`` + ``typing.Union[int, float]`` .. py:attribute:: result diff --git a/exec_helpers/_ssh_client_base.py b/exec_helpers/_ssh_client_base.py index 068a31e..2b86fc7 100644 --- a/exec_helpers/_ssh_client_base.py +++ b/exec_helpers/_ssh_client_base.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import unicode_literals +import abc import base64 import collections # noinspection PyCompatibility @@ -35,8 +36,8 @@ import warnings import advanced_descriptors -import paramiko -import tenacity +import paramiko # type: ignore +import tenacity # type: ignore import threaded import six @@ -64,7 +65,7 @@ CPYTHON = 'CPython' == platform.python_implementation() -class _MemorizedSSH(type): +class _MemorizedSSH(abc.ABCMeta): """Memorize metaclass for SSHClient. This class implements caching and managing of SSHClient connections. @@ -100,7 +101,7 @@ def __prepare__( mcs, # type: typing.Type[_MemorizedSSH] name, # type: str bases, # type: typing.Iterable[typing.Type] - **kwargs # type: typing.Dict + **kwargs # type: typing.Any ): # type: (...) -> collections.OrderedDict # pylint: disable=unused-argument """Metaclass magic for object storage. @@ -108,7 +109,7 @@ def __prepare__( """ return collections.OrderedDict() # pragma: no cover - def __call__( + def __call__( # type: ignore cls, # type: _MemorizedSSH host, # type: str port=22, # type: int @@ -120,14 +121,21 @@ def __call__( ): # type: (...) -> SSHClientBase """Main memorize method: check for cached instance and return it. + :param host: remote hostname :type host: str + :param port: remote ssh port :type port: int - :type username: str - :type password: str - :type private_keys: list - :type auth: ssh_auth.SSHAuth + :param username: remote username. + :type username: typing.Optional[str] + :param password: remote password + :type password: typing.Optional[str] + :param private_keys: private keys for connection + :type private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] + :param auth: credentials for connection + :type auth: typing.Optional[ssh_auth.SSHAuth] + :param verbose: show additional error/warning messages :type verbose: bool - :rtype: SSHClient + :rtype: SSHClientBase """ if (host, port) in cls.__cache: key = host, port @@ -153,7 +161,7 @@ def __call__( # If we have only cache reference and temporary getrefcount # reference: close connection before deletion cls.__cache[key].logger.debug('Closing as unused') - cls.__cache[key].close() + cls.__cache[key].close() # type: ignore del cls.__cache[key] # noinspection PyArgumentList ssh = super( @@ -172,7 +180,7 @@ def clear_cache(mcs): # type: (typing.Type[_MemorizedSSH]) -> None getrefcount is used to check for usage. """ - n_count = 3 if six.PY3 else 4 + n_count = 4 # PY3: cache, ssh, temporary # PY4: cache, values mapping, ssh, temporary for ssh in mcs.__cache.values(): @@ -181,7 +189,7 @@ def clear_cache(mcs): # type: (typing.Type[_MemorizedSSH]) -> None sys.getrefcount(ssh) == n_count ): # pragma: no cover ssh.logger.debug('Closing as unused') - ssh.close() + ssh.close() # type: ignore mcs.__cache = {} @classmethod @@ -189,7 +197,7 @@ def close_connections(mcs): # type: (typing.Type[_MemorizedSSH]) -> None """Close connections for selected or all cached records.""" for ssh in mcs.__cache.values(): if ssh.is_alive: - ssh.close() + ssh.close() # type: ignore class SSHClientBase(six.with_metaclass(_MemorizedSSH, api.ExecHelper)): @@ -216,19 +224,19 @@ def __init__( ): # type: (...) -> None """Context manager for call commands with sudo. - :type ssh: SSHClient - :type enforce: bool + :type ssh: SSHClientBase + :type enforce: typing.Optional[bool] """ self.__ssh = ssh self.__sudo_status = ssh.sudo_mode self.__enforce = enforce - def __enter__(self): + def __enter__(self): # type: () -> None self.__sudo_status = self.__ssh.sudo_mode if self.__enforce is not None: self.__ssh.sudo_mode = self.__enforce - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb): # type: (typing.Any, typing.Any, typing.Any) -> None self.__ssh.sudo_mode = self.__sudo_status class __get_keepalive(object): @@ -247,7 +255,7 @@ def __init__( ): # type: (...) -> None """Context manager for keepalive management. - :type ssh: SSHClient + :type ssh: SSHClientBase :type enforce: bool :param enforce: Keep connection alive after context manager exit """ @@ -255,17 +263,17 @@ def __init__( self.__keepalive_status = ssh.keepalive_mode self.__enforce = enforce - def __enter__(self): + def __enter__(self): # type: () -> None self.__keepalive_status = self.__ssh.keepalive_mode if self.__enforce is not None: self.__ssh.keepalive_mode = self.__enforce self.__ssh.__enter__() - def __exit__(self, exc_type, exc_val, exc_tb): - self.__ssh.__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) + def __exit__(self, exc_type, exc_val, exc_tb): # type: (typing.Any, typing.Any, typing.Any) -> None + self.__ssh.__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) # type: ignore self.__ssh.keepalive_mode = self.__keepalive_status - def __hash__(self): + def __hash__(self): # type: () -> int """Hash for usage as dict keys.""" return hash(( self.__class__, @@ -394,13 +402,13 @@ def _ssh(self): # type: () -> paramiko.SSHClient """ return self.__ssh - @tenacity.retry( + @tenacity.retry( # type: ignore retry=tenacity.retry_if_exception_type(paramiko.SSHException), stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_fixed(3), reraise=True, ) - def __connect(self): + def __connect(self): # type: ()-> None """Main method for connection open.""" with self.lock: self.auth.connect( @@ -408,7 +416,7 @@ def __connect(self): hostname=self.hostname, port=self.port, log=self.__verbose) - def __connect_sftp(self): + def __connect_sftp(self): # type: ()-> None """SFTP connection opener.""" with self.lock: try: @@ -434,7 +442,7 @@ def _sftp(self): # type: () -> paramiko.sftp_client.SFTPClient raise paramiko.SSHException('SFTP connection failed') @advanced_descriptors.SeparateClassMethod - def close(self): + def close(self): # type: ()-> None """Close SSH and SFTP sessions.""" with self.lock: # noinspection PyBroadException @@ -453,7 +461,7 @@ def close(self): ) # noinspection PyMethodParameters - @close.class_method + @close.class_method # type: ignore def close( # pylint: disable=no-self-argument cls # type: typing.Type[SSHClientBase] ): # type: (...) -> None @@ -462,7 +470,7 @@ def close( # pylint: disable=no-self-argument cls.__class__.close_connections() @classmethod - def _clear_cache(cls): + def _clear_cache(cls): # type: (typing.Type[SSHClientBase]) -> None """Enforce clear memorized records.""" warnings.warn( '_clear_cache() is dangerous and not recommended for normal use!', @@ -470,11 +478,11 @@ def _clear_cache(cls): ) _MemorizedSSH.clear_cache() - def __del__(self): + def __del__(self): # type: ()-> None """Destructor helper: close channel and threads BEFORE closing others. - Due to threading in paramiko, default destructor could generate asserts - on close, so we calling channel close before closing main ssh object. + Due to threading in paramiko, default destructor could generate asserts on close, + so we calling channel close before closing main ssh object. """ try: self.__ssh.close() @@ -487,7 +495,7 @@ def __del__(self): ) self.__sftp = None - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb): # type: (typing.Any, typing.Any, typing.Any) -> None """Exit context manager. .. versionchanged:: 1.0.0 disconnect enforced on close @@ -495,7 +503,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): .. versionchanged:: 1.2.1 disconnect enforced on close only not in keepalive mode """ if not self.__keepalive_mode: - self.close() + self.close() # type: ignore super(SSHClientBase, self).__exit__(exc_type, exc_val, exc_tb) @property @@ -533,7 +541,7 @@ def keepalive_mode(self, mode): # type: (bool) -> None def reconnect(self): # type: () -> None """Reconnect SSH session.""" with self.lock: - self.close() + self.close() # type: ignore self.__ssh = paramiko.SSHClient() self.__ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -548,6 +556,7 @@ def sudo( :param enforce: Enforce sudo enabled or disabled. By default: None :type enforce: typing.Optional[bool] + :rtype: typing.ContextManager """ return self.__get_sudo(ssh=self, enforce=enforce) @@ -559,6 +568,7 @@ def keepalive( :param enforce: Enforce keepalive enabled or disabled. :type enforce: bool + :rtype: typing.ContextManager .. Note:: Enter and exit ssh context manager is produced as well. .. versionadded:: 1.2.1 @@ -568,19 +578,19 @@ def keepalive( def execute_async( self, command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] + stdin=None, # type: typing.Union[bytes, str, bytearray, None] open_stdout=True, # type: bool open_stderr=True, # type: bool verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> _type_execute_async """Execute command in async mode and return channel with IO objects. :param command: Command for execution :type command: str :param stdin: pass STDIN text to the process - :type stdin: typing.Union[typing.AnyStr, bytearray, None] + :type stdin: typing.Union[bytes, str, bytearray, None] :param open_stdout: open STDOUT stream for read :type open_stdout: bool :param open_stderr: open STDERR stream for read @@ -606,7 +616,7 @@ def execute_async( log_mask_re=log_mask_re ) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) @@ -631,6 +641,7 @@ def execute_async( cmd = "sudo -S bash -c 'eval \"$(base64 -d <(echo \"{0}\"))\"'".format(encoded_cmd) chan.exec_command(cmd) # nosec # Sanitize on caller side if stdout.channel.closed is False: + # noinspection PyTypeChecker self.auth.enter_password(_stdin) _stdin.flush() else: @@ -649,20 +660,20 @@ def _exec_command( self, command, # type: str interface, # type: paramiko.channel.Channel - stdout, # type: paramiko.channel.ChannelFile - stderr, # type: paramiko.channel.ChannelFile - timeout, # type: typing.Union[int, None] + stdout, # type: typing.Optional[paramiko.ChannelFile] + stderr, # type: typing.Optional[paramiko.ChannelFile] + timeout, # type: typing.Union[int, float, None] verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Get exit status from channel with timeout. :type command: str :type interface: paramiko.channel.Channel - :type stdout: paramiko.channel.ChannelFile - :type stderr: paramiko.channel.ChannelFile - :type timeout: typing.Union[int, None] + :type stdout: typing.Optional[paramiko.ChannelFile] + :type stderr: typing.Optional[paramiko.ChannelFile] + :type timeout: typing.Union[int, float, None] :type verbose: bool :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>' @@ -672,7 +683,7 @@ def _exec_command( .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - def poll_streams(): + def poll_streams(): # type: () -> None """Poll FIFO buffers if data available.""" if stdout and interface.recv_ready(): result.read_stdout( @@ -687,7 +698,7 @@ def poll_streams(): verbose=verbose ) - @threaded.threadpooled + @threaded.threadpooled # type: ignore def poll_pipes(stop, ): # type: (threading.Event) -> None """Polling task for FIFO buffers. @@ -724,6 +735,7 @@ def poll_pipes(stop, ): # type: (threading.Event) -> None stop_event = threading.Event() # pylint: disable=assignment-from-no-return + # noinspection PyNoneFunctionAssignment future = poll_pipes(stop=stop_event) # type: concurrent.futures.Future # pylint: enable=assignment-from-no-return @@ -743,7 +755,7 @@ def poll_pipes(stop, ): # type: (threading.Event) -> None timeout=timeout ) self.logger.debug(wait_err_msg) - raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) + raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) # type: ignore def execute_through_host( self, @@ -752,9 +764,9 @@ def execute_through_host( auth=None, # type: typing.Optional[ssh_auth.SSHAuth] target_port=22, # type: int verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] + timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, float, None] get_pty=False, # type: bool - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Execute command on remote host through currently connected host. @@ -769,7 +781,7 @@ def execute_through_host( :param verbose: Produce log.info records for command call and output :type verbose: bool :param timeout: Timeout for command execution. - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :param get_pty: open PTY on target machine :type get_pty: bool :rtype: ExecResult @@ -782,7 +794,7 @@ def execute_through_host( cmd=command, log_mask_re=kwargs.get('log_mask_re', None) ) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) @@ -830,10 +842,10 @@ def execute_together( cls, remotes, # type: typing.Iterable[SSHClientBase] command, # type: str - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] + timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, float, None] expected=None, # type: typing.Optional[typing.Iterable[int]] raise_on_err=True, # type: bool - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] """Execute command on multiple remotes in async mode. @@ -842,7 +854,7 @@ def execute_together( :param command: Command for execution :type command: str :param timeout: Timeout for command execution. - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :param expected: expected return codes (0 by default) :type expected: typing.Optional[typing.Iterable[]] :param raise_on_err: Raise exception on unexpected return code @@ -855,8 +867,8 @@ def execute_together( .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - @threaded.threadpooled - def get_result(): # type: () -> exec_result.ExecResult + @threaded.threadpooled # type: ignore + def get_result(remote): # type: (SSHClientBase) -> exec_result.ExecResult """Get result from remote call.""" ( chan, @@ -889,14 +901,11 @@ def get_result(): # type: () -> exec_result.ExecResult expected = expected or [proc_enums.ExitCodes.EX_OK] expected = proc_enums.exit_codes_to_enums(expected) - futures = {} + futures = {remote: get_result(remote) for remote in set(remotes)} # Use distinct remotes results = {} errors = {} raised_exceptions = {} - for remote in set(remotes): # Use distinct remotes - futures[remote] = get_result() - ( _, not_done, @@ -910,7 +919,7 @@ def get_result(): # type: () -> exec_result.ExecResult for ( remote, - future, + future, # type: ignore ) in futures.items(): # type: SSHClientBase, concurrent.futures.Future try: result = future.result() @@ -977,7 +986,7 @@ def utime( .. versionadded:: 1.0.0 """ - return self._sftp.utime(path, times) # pragma: no cover + return self._sftp.utime(path, times) # type: ignore # pragma: no cover def isfile(self, path): # type: (str) -> bool """Check, that path is file using SFTP session. @@ -987,7 +996,7 @@ def isfile(self, path): # type: (str) -> bool """ try: attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFREG != 0 + return attrs.st_mode & stat.S_IFREG != 0 # type: ignore except IOError: return False @@ -999,6 +1008,6 @@ def isdir(self, path): # type: (str) -> bool """ try: attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFDIR != 0 + return attrs.st_mode & stat.S_IFDIR != 0 # type: ignore except IOError: return False diff --git a/exec_helpers/_ssh_client_base.pyi b/exec_helpers/_ssh_client_base.pyi deleted file mode 100644 index 61ecce9..0000000 --- a/exec_helpers/_ssh_client_base.pyi +++ /dev/null @@ -1,163 +0,0 @@ -import collections -import typing - -import paramiko # type: ignore - -from exec_helpers import exec_result, ssh_auth, api - -CPYTHON: bool = ... - - -class _MemorizedSSH(type): - @classmethod - def __prepare__( - mcs: typing.Type[_MemorizedSSH], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Dict - ) -> collections.OrderedDict: ... - - def __call__( # type: ignore - cls: _MemorizedSSH, - host: str, - port: int = ..., - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - verbose: bool = ..., - ) -> SSHClientBase: ... - - @classmethod - def clear_cache(mcs: typing.Type[_MemorizedSSH]) -> None: ... - - @classmethod - def close_connections(mcs: typing.Type[_MemorizedSSH]) -> None: ... - - -class SSHClientBase(api.ExecHelper, metaclass=_MemorizedSSH): - def __hash__(self) -> int: ... - - def __init__( - self, - host: str, - port: int = ..., - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - verbose: bool = ..., - ) -> None: ... - - @property - def auth(self) -> ssh_auth.SSHAuth: ... - - @property - def hostname(self) -> str: ... - - @property - def port(self) -> int: ... - - @property - def is_alive(self) -> bool: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... - - @property - def _ssh(self) -> paramiko.SSHClient: ... - - @property - def _sftp(self) -> paramiko.sftp_client.SFTPClient: ... - - @classmethod - def close(cls: typing.Union[SSHClientBase, typing.Type[SSHClientBase]]) -> None: ... - - @classmethod - def _clear_cache(cls) -> None: ... - - def __del__(self) -> None: ... - - def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: ... - - @property - def sudo_mode(self) -> bool: ... - - @sudo_mode.setter - def sudo_mode(self, mode: bool) -> None: ... - - @property - def keepalive_mode(self) -> bool: ... - - @keepalive_mode.setter - def keepalive_mode(self, mode: bool) -> None: ... - - def reconnect(self) -> None: ... - - def sudo(self, enforce: typing.Optional[bool] = ...) -> typing.ContextManager: ... - - def keepalive(self, enforce: bool = ...) -> typing.ContextManager: ... - - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[ - paramiko.Channel, - paramiko.ChannelFile, - typing.Optional[paramiko.ChannelFile], - typing.Optional[paramiko.ChannelFile], - ]: ... - - def _exec_command( - self, - command: str, - interface: paramiko.channel.Channel, - stdout: paramiko.channel.ChannelFile, - stderr: paramiko.channel.ChannelFile, - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - def execute_through_host( - self, - hostname: str, - command: str, - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - target_port: int = ..., - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - get_pty: bool = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - @classmethod - def execute_together( - cls, - remotes: typing.Iterable[SSHClientBase], - command: str, - timeout: typing.Union[int, None] = ..., - expected: typing.Optional[typing.Iterable[int]] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Dict - ) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult]: ... - - def open(self, path: str, mode: str = ...) -> paramiko.SFTPFile: ... - - def exists(self, path: str) -> bool: ... - - def stat(self, path: str) -> paramiko.sftp_attr.SFTPAttributes: ... - - def utime(self, path: str, times: typing.Optional[typing.Tuple[int, int]] = ...) -> None: ... - - def isfile(self, path: str) -> bool: ... - - def isdir(self, path: str) -> bool: ... diff --git a/exec_helpers/api.py b/exec_helpers/api.py index 8c55bb4..ef77373 100644 --- a/exec_helpers/api.py +++ b/exec_helpers/api.py @@ -23,18 +23,21 @@ from __future__ import division from __future__ import unicode_literals +import abc import logging import re import threading import typing # noqa # pylint: disable=unused-import +import six + from exec_helpers import constants from exec_helpers import exceptions from exec_helpers import exec_result # noqa # pylint: disable=unused-import from exec_helpers import proc_enums -class ExecHelper(object): +class ExecHelper(six.with_metaclass(abc.ABCMeta, object)): """ExecHelper global API.""" __slots__ = ( @@ -74,7 +77,7 @@ def lock(self): # type: () -> threading.RLock """ return self.__lock - def __enter__(self): + def __enter__(self): # type: () -> ExecHelper """Get context manager. .. versionchanged:: 1.1.0 lock on enter @@ -82,9 +85,9 @@ def __enter__(self): self.lock.acquire() return self - def __exit__(self, exc_type, exc_val, exc_tb): # pragma: no cover + def __exit__(self, exc_type, exc_val, exc_tb): # type: (typing.Any, typing.Any, typing.Any) -> None """Context manager usage.""" - self.lock.release() + self.lock.release() # pragma: no cover def _mask_command( self, @@ -130,22 +133,23 @@ def mask(text, rules): # type: (str, str) -> str return cmd + @abc.abstractmethod def execute_async( self, command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] + stdin=None, # type: typing.Union[bytes, str, bytearray, None] open_stdout=True, # type: bool open_stderr=True, # type: bool verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any,] """Execute command in async mode and return remote interface with IO objects. :param command: Command for execution :type command: str :param stdin: pass STDIN text to the process - :type stdin: typing.Union[typing.AnyStr, bytearray, None] + :type stdin: typing.Union[bytes, str, bytearray, None] :param open_stdout: open STDOUT stream for read :type open_stdout: bool :param open_stderr: open STDERR stream for read @@ -155,28 +159,24 @@ def execute_async( :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>' :type log_mask_re: typing.Optional[str] - :rtype: typing.Tuple[ - typing.Any, - typing.Any, - typing.Any, - typing.Any, - ] + :rtype: typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any] .. versionchanged:: 1.2.0 open_stdout and open_stderr flags .. versionchanged:: 1.2.0 stdin data """ raise NotImplementedError # pragma: no cover + @abc.abstractmethod def _exec_command( self, command, # type: str interface, # type: typing.Any stdout, # type: typing.Any stderr, # type: typing.Any - timeout, # type: typing.Union[int, None] + timeout, # type: typing.Union[int, float, None] verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Get exit status from channel with timeout. @@ -189,7 +189,7 @@ def _exec_command( :param stderr: STDERR pipe or file-like object :type stderr: typing.Any :param timeout: Timeout for command execution - :type timeout: int + :type timeout: typing.Union[int, float, None] :param verbose: produce verbose log record on command call :type verbose: bool :param log_mask_re: regex lookup rule to mask command for logger. @@ -206,19 +206,17 @@ def execute( self, command, # type: str verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - **kwargs + timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, float, None] + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Execute command and wait for return code. - Timeout limitation: read tick is 100 ms. - :param command: Command for execution :type command: str :param verbose: Produce log.info records for command call and output :type verbose: bool :param timeout: Timeout for command execution. - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :rtype: ExecResult :raises ExecHelperTimeoutError: Timeout exceeded @@ -246,7 +244,7 @@ def execute( **kwargs ) message = "Command {result.cmd!r} exit code: {result.exit_code!s}".format(result=result) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=message ) @@ -256,22 +254,20 @@ def check_call( self, command, # type: str verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] + timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, float, None] error_info=None, # type: typing.Optional[str] expected=None, # type: typing.Optional[typing.Iterable[typing.Union[int, proc_enums.ExitCodes]]] raise_on_err=True, # type: bool - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Execute command and check for return code. - Timeout limitation: read tick is 100 ms. - :param command: Command for execution :type command: str :param verbose: Produce log.info records for command call and output :type verbose: bool :param timeout: Timeout for command execution. - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :param error_info: Text for error details, if fail happens :type error_info: typing.Optional[str] :param expected: expected return codes (0 by default) @@ -306,21 +302,19 @@ def check_stderr( self, command, # type: str verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] + timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, float, None] error_info=None, # type: typing.Optional[str] raise_on_err=True, # type: bool - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Execute command expecting return code 0 and empty STDERR. - Timeout limitation: read tick is 100 ms. - :param command: Command for execution :type command: str :param verbose: Produce log.info records for command call and output :type verbose: bool :param timeout: Timeout for command execution. - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :param error_info: Text for error details, if fail happens :type error_info: typing.Optional[str] :param raise_on_err: Raise exception on unexpected return code diff --git a/exec_helpers/api.pyi b/exec_helpers/api.pyi deleted file mode 100644 index ccbcec1..0000000 --- a/exec_helpers/api.pyi +++ /dev/null @@ -1,74 +0,0 @@ -import logging -import threading -import typing - -from exec_helpers import exec_result, proc_enums - -class ExecHelper: - log_mask_re: typing.Optional[str] = ... - - def __init__(self, logger: logging.Logger, log_mask_re: typing.Optional[str] = ...) -> None: ... - - @property - def logger(self) -> logging.Logger: ... - - @property - def lock(self) -> threading.RLock: ... - - def __enter__(self) -> ExecHelper: ... - - def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: ... - - def _mask_command(self, cmd: str, log_mask_re: typing.Optional[str] = ...) -> str: ... - - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any]: ... - - def _exec_command( - self, - command: str, - interface: typing.Any, - stdout: typing.Any, - stderr: typing.Any, - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - def execute( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - **kwargs: typing.Type - ) -> exec_result.ExecResult: ... - - def check_call( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - error_info: typing.Optional[str] = ..., - expected: typing.Optional[typing.Iterable[typing.Union[int, proc_enums.ExitCodes]]] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Type - ) -> exec_result.ExecResult: ... - - def check_stderr( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - error_info: typing.Optional[str] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... diff --git a/exec_helpers/exceptions.py b/exec_helpers/exceptions.py index fc01aa7..87d07b5 100644 --- a/exec_helpers/exceptions.py +++ b/exec_helpers/exceptions.py @@ -18,11 +18,14 @@ from __future__ import division from __future__ import unicode_literals -import typing # noqa # pylint: disable=unused-import +import typing from exec_helpers import proc_enums from exec_helpers import _log_templates +if typing.TYPE_CHECKING: # pragma: no cover + from exec_helpers import exec_result # noqa: F401 # pylint: disable=cyclic-import, unused-import + __all__ = ( 'ExecHelperError', 'ExecHelperTimeoutError', diff --git a/exec_helpers/exceptions.pyi b/exec_helpers/exceptions.pyi deleted file mode 100644 index 3df60f5..0000000 --- a/exec_helpers/exceptions.pyi +++ /dev/null @@ -1,81 +0,0 @@ -import typing -from exec_helpers import proc_enums, exec_result - -class ExecHelperError(Exception): ... - -class DeserializeValueError(ExecHelperError, ValueError): ... - -class ExecCalledProcessError(ExecHelperError): ... - -class ExecHelperTimeoutError(ExecCalledProcessError): - - result: exec_result.ExecResult = ... - timeout: int = ... - - def __init__(self, result: exec_result.ExecResult, timeout: typing.Union[int, float]) -> None: ... - - @property - def cmd(self) -> str: ... - - @property - def stdout(self) -> typing.Text: ... - - @property - def stderr(self) -> typing.Text: ... - - -class CalledProcessError(ExecCalledProcessError): - - result: exec_result.ExecResult = ... - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - - def __init__( - self, - result: exec_result.ExecResult, - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... - - @property - def returncode(self) -> typing.Union[int, proc_enums.ExitCodes]: ... - - @property - def cmd(self) -> str: ... - - @property - def stdout(self) -> typing.Text: ... - - @property - def stderr(self) -> typing.Text: ... - - -class ParallelCallExceptions(ExecCalledProcessError): - - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - cmd: str = ... - exceptions: typing.Dict[typing.Tuple[str, int], Exception] = ... - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - - def __init__( - self, - command: str, - exceptions: typing.Dict[typing.Tuple[str, int], Exception], - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... - -class ParallelCallProcessError(ExecCalledProcessError): - - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - cmd: str = ... - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - - def __init__( - self, - command: str, - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... diff --git a/exec_helpers/exec_result.py b/exec_helpers/exec_result.py index fb9130d..7d7a7b0 100644 --- a/exec_helpers/exec_result.py +++ b/exec_helpers/exec_result.py @@ -50,7 +50,7 @@ class ExecResult(object): def __init__( self, cmd, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] + stdin=None, # type: typing.Union[bytes, str, bytearray, None] stdout=None, # type: typing.Optional[typing.Iterable[bytes]] stderr=None, # type: typing.Optional[typing.Iterable[bytes]] exit_code=proc_enums.ExitCodes.EX_INVALID # type: typing.Union[int, proc_enums.ExitCodes] @@ -60,7 +60,7 @@ def __init__( :param cmd: command :type cmd: str :param stdin: string STDIN - :type stdin: typing.Union[typing.AnyStr, bytearray, None] + :type stdin: typing.Union[bytes, str, bytearray, None] :param stdout: binary STDOUT :type stdout: typing.Optional[typing.Iterable[bytes]] :param stderr: binary STDERR @@ -76,10 +76,18 @@ def __init__( elif isinstance(stdin, bytearray): stdin = self._get_str_from_bin(stdin) self.__stdin = stdin # type: typing.Optional[typing.Text] - self.__stdout = tuple(stdout) if stdout is not None else () # type: typing.Tuple[bytes] - self.__stderr = tuple(stderr) if stderr is not None else () # type: typing.Tuple[bytes] - self.__exit_code = None + if stdout is not None: + self.__stdout = tuple(stdout) # type: typing.Tuple[bytes, ...] + else: + self.__stdout = () + + if stderr is not None: + self.__stderr = tuple(stderr) # type: typing.Tuple[bytes, ...] + else: + self.__stderr = () + + self.__exit_code = proc_enums.ExitCodes.EX_INVALID # type: typing.Union[int, proc_enums.ExitCodes] self.__timestamp = None self.exit_code = exit_code @@ -101,7 +109,7 @@ def lock(self): # type: () -> threading.RLock def timestamp(self): # type: () -> typing.Optional[datetime.datetime] """Timestamp. - :rtype: typing.Optional(datetime.datetime) + :rtype: typing.Optional[datetime.datetime] """ return self.__timestamp @@ -117,7 +125,7 @@ def _get_bytearray_from_array( return bytearray(b''.join(src)) @staticmethod - def _get_str_from_bin(src): # type: (bytearray) -> typing.Text + def _get_str_from_bin(src): # type: (bytearray) -> str """Join data in list to the string, with python 2&3 compatibility. :type src: bytearray @@ -135,7 +143,10 @@ def _get_brief(cls, data): # type: (typing.Tuple[bytes]) -> typing.Text :type data: typing.Tuple[bytes] :rtype: str """ - src = data if len(data) <= 7 else data[:3] + (b'...\n',) + data[-3:] # type: typing.Tuple[bytes] + if len(data) <= 7: + src = data # type: typing.Tuple[bytes, ...] + else: + src = data[:3] + (b'...\n',) + data[-3:] return cls._get_str_from_bin( cls._get_bytearray_from_array(src) ) @@ -157,7 +168,7 @@ def stdin(self): # type: () -> typing.Optional[typing.Text] return self.__stdin @property - def stdout(self): # type: () -> typing.Tuple[bytes] + def stdout(self): # type: () -> typing.Tuple[bytes, ...] """Stdout output as list of binaries. :rtype: typing.Tuple[bytes] @@ -165,10 +176,10 @@ def stdout(self): # type: () -> typing.Tuple[bytes] return self.__stdout @property - def stderr(self): # type: () -> typing.Tuple[bytes] + def stderr(self): # type: () -> typing.Tuple[bytes, ...] """Stderr output as list of binaries. - :rtype: typing.Tuple[bytes] + :rtype: typing.Tuple[bytes, ...] """ return self.__stderr @@ -183,7 +194,7 @@ def __poll_stream( for line in src: dst.append(line) if log: - log.log( + log.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=line.decode('utf-8', errors='backslashreplace').rstrip() ) @@ -208,10 +219,11 @@ def read_stdout( .. versionchanged:: 1.2.0 - src can be None """ - if self.timestamp: - raise RuntimeError('Final exit code received.') if not src: return + if self.timestamp: + raise RuntimeError('Final exit code received.') + with self.lock: self.__stdout_str = self.__stdout_brief = None self.__stdout += tuple(self.__poll_stream(src, log, verbose)) @@ -225,7 +237,7 @@ def read_stderr( """Read stderr file-like object to stdout. :param src: source - :type src: typing.Iterable + :type src: typing.Optional[typing.Iterable] :param log: logger :type log: typing.Optional[logging.Logger] :param verbose: use log.info instead of log.debug @@ -233,10 +245,11 @@ def read_stderr( .. versionchanged:: 1.2.0 - src can be None """ - if self.timestamp: - raise RuntimeError('Final exit code received.') if not src: return + if self.timestamp: + raise RuntimeError('Final exit code received.') + with self.lock: self.__stderr_str = self.__stderr_brief = None self.__stderr += tuple(self.__poll_stream(src, log, verbose)) @@ -269,8 +282,8 @@ def stdout_str(self): # type: () -> typing.Text """ with self.lock: if self.__stdout_str is None: - self.__stdout_str = self._get_str_from_bin(self.stdout_bin) - return self.__stdout_str + self.__stdout_str = self._get_str_from_bin(self.stdout_bin) # type: ignore + return self.__stdout_str # type: ignore @property def stderr_str(self): # type: () -> typing.Text @@ -280,8 +293,8 @@ def stderr_str(self): # type: () -> typing.Text """ with self.lock: if self.__stderr_str is None: - self.__stderr_str = self._get_str_from_bin(self.stderr_bin) - return self.__stderr_str + self.__stderr_str = self._get_str_from_bin(self.stderr_bin) # type: ignore + return self.__stderr_str # type: ignore @property def stdout_brief(self): # type: () -> typing.Text @@ -291,8 +304,8 @@ def stdout_brief(self): # type: () -> typing.Text """ with self.lock: if self.__stdout_brief is None: - self.__stdout_brief = self._get_brief(self.stdout) - return self.__stdout_brief + self.__stdout_brief = self._get_brief(self.stdout) # type: ignore + return self.__stdout_brief # type: ignore @property def stderr_brief(self): # type: () -> typing.Text @@ -302,8 +315,8 @@ def stderr_brief(self): # type: () -> typing.Text """ with self.lock: if self.__stderr_brief is None: - self.__stderr_brief = self._get_brief(self.stderr) - return self.__stderr_brief + self.__stderr_brief = self._get_brief(self.stderr) # type: ignore + return self.__stderr_brief # type: ignore @property def exit_code(self): # type: () -> typing.Union[int, proc_enums.ExitCodes] @@ -317,7 +330,7 @@ def exit_code(self): # type: () -> typing.Union[int, proc_enums.ExitCodes] def exit_code(self, new_val): # type: (typing.Union[int, proc_enums.ExitCodes]) -> None """Return(exit) code of command. - :type new_val: int + :type new_val: typing.Union[int, proc_enums.ExitCodes] If valid exit code is set - object became read-only. """ if self.timestamp: @@ -327,13 +340,13 @@ def exit_code(self, new_val): # type: (typing.Union[int, proc_enums.ExitCodes]) with self.lock: self.__exit_code = proc_enums.exit_code_to_enum(new_val) if self.__exit_code != proc_enums.ExitCodes.EX_INVALID: - self.__timestamp = datetime.datetime.utcnow() + self.__timestamp = datetime.datetime.utcnow() # type: ignore def __deserialize(self, fmt): # type: (str) -> typing.Any """Deserialize stdout as data format. :type fmt: str - :rtype: object + :rtype: typing.Any :raises NotImplementedError: fmt deserialization not implemented :raises DeserializeValueError: Not valid source format """ @@ -359,7 +372,7 @@ def __deserialize(self, fmt): # type: (str) -> typing.Any def stdout_json(self): # type: () -> typing.Any """JSON from stdout. - :rtype: object + :rtype: typing.Any """ with self.lock: return self.__deserialize(fmt='json') @@ -368,12 +381,12 @@ def stdout_json(self): # type: () -> typing.Any def stdout_yaml(self): # type: () -> typing.Any """YAML from stdout. - :rtype: Union(list, dict, None) + :rtype: typing.Any """ with self.lock: return self.__deserialize(fmt='yaml') - def __dir__(self): + def __dir__(self): # type: () -> typing.List[str] """Override dir for IDE and as source for getitem checks.""" return [ 'cmd', 'stdout', 'stderr', 'exit_code', @@ -428,7 +441,7 @@ def __ne__(self, other): # type: (typing.Any) -> bool """Comparision.""" return not self.__eq__(other) - def __hash__(self): + def __hash__(self): # type: () -> int """Hash for usage as dict key and in sets.""" return hash( ( diff --git a/exec_helpers/exec_result.pyi b/exec_helpers/exec_result.pyi deleted file mode 100644 index 88b515c..0000000 --- a/exec_helpers/exec_result.pyi +++ /dev/null @@ -1,96 +0,0 @@ -import datetime -import logging -import threading -import typing -from exec_helpers import proc_enums - -class ExecResult: - def __init__( - self, - cmd: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - stdout: typing.Optional[typing.Iterable[bytes]] = ..., - stderr: typing.Optional[typing.Iterable[bytes]] = ..., - exit_code: typing.Union[int, proc_enums.ExitCodes] = ..., - ) -> None: ... - - @property - def lock(self) -> threading.RLock: ... - - @property - def timestamp(self) -> typing.Optional[datetime.datetime]: ... - - @staticmethod - def _get_bytearray_from_array(src: typing.Iterable[bytes]) -> bytearray: ... - - @staticmethod - def _get_str_from_bin(src: bytearray) -> typing.Text: ... - - @classmethod - def _get_brief(cls, data: typing.Tuple[bytes]) -> typing.Text: ... - - @property - def cmd(self) -> str: ... - - @property - def stdin(self) -> typing.Optional[typing.Text]: ... - - @property - def stdout(self) -> typing.Tuple[bytes]: ... - - @property - def stderr(self) -> typing.Tuple[bytes]: ... - - def read_stdout( - self, - src: typing.Optional[typing.Iterable] = ..., - log: typing.Optional[logging.Logger] = ..., - verbose: bool = ..., - ) -> None: ... - - def read_stderr( - self, - src: typing.Optional[typing.Iterable] = ..., - log: typing.Optional[logging.Logger] = ..., - verbose: bool = ..., - ) -> None: ... - - @property - def stdout_bin(self) -> bytearray: ... - - @property - def stderr_bin(self) -> bytearray: ... - - @property - def stdout_str(self) -> typing.Text: ... - - @property - def stderr_str(self) -> typing.Text: ... - - @property - def stdout_brief(self) -> typing.Text: ... - - @property - def stderr_brief(self) -> typing.Text: ... - - @property - def exit_code(self) -> typing.Union[int, proc_enums.ExitCodes]: ... - - @exit_code.setter - def exit_code(self, new_val: typing.Union[int, proc_enums.ExitCodes]) -> None: ... - - @property - def stdout_json(self) -> typing.Any: ... - - @property - def stdout_yaml(self) -> typing.Any: ... - - def __getitem__(self, item: str) -> typing.Any: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... - - def __eq__(self, other: typing.Any) -> bool: ... - - def __ne__(self, other: typing.Any) -> bool: ... diff --git a/exec_helpers/ssh_auth.py b/exec_helpers/ssh_auth.py index e270ee2..514d176 100644 --- a/exec_helpers/ssh_auth.py +++ b/exec_helpers/ssh_auth.py @@ -21,11 +21,10 @@ from __future__ import unicode_literals import copy -import io # noqa # pylint: disable=unused-import import logging import typing # noqa # pylint: disable=unused-import -import paramiko +import paramiko # type: ignore __all__ = ('SSHAuth', ) @@ -114,16 +113,14 @@ def public_key(self): # type: () -> typing.Optional[str] return self.__get_public_key(self.__key) @property - def key_filename( - self - ): # type: () -> typing.Union[typing.List[str], str, None] + def key_filename(self): # type: () -> typing.Union[typing.List[str], str, None] """Key filename(s). .. versionadded:: 1.0.0 """ return copy.deepcopy(self.__key_filename) - def enter_password(self, tgt): # type: (io.StringIO) -> None + def enter_password(self, tgt): # type: (typing.IO) -> None """Enter password to STDIN. Note: required for 'sudo' call @@ -197,7 +194,7 @@ def connect( logger.exception(msg) raise paramiko.AuthenticationException(msg) - def __hash__(self): + def __hash__(self): # type: () -> int """Hash for usage as dict keys and comparison.""" return hash(( self.__class__, @@ -222,7 +219,7 @@ def __ne__(self, other): # type: (typing.Any) -> bool def __deepcopy__(self, memo): # type: (typing.Any) -> SSHAuth """Helper for copy.deepcopy.""" - return self.__class__( + return self.__class__( # type: ignore username=self.username, password=self.__password, key=self.__key, @@ -231,7 +228,7 @@ def __deepcopy__(self, memo): # type: (typing.Any) -> SSHAuth def __copy__(self): # type: () -> SSHAuth """Copy self.""" - return self.__class__( + return self.__class__( # type: ignore username=self.username, password=self.__password, key=self.__key, @@ -244,7 +241,7 @@ def __repr__(self): # type: (...) -> str None if self.__key is None else ''.format(self.public_key) ) - _keys = [] + _keys = [] # type: typing.List[typing.Union[str, None]] for k in self.__keys: if k == self.__key: continue diff --git a/exec_helpers/ssh_auth.pyi b/exec_helpers/ssh_auth.pyi deleted file mode 100644 index f0d0ad4..0000000 --- a/exec_helpers/ssh_auth.pyi +++ /dev/null @@ -1,45 +0,0 @@ -import io -import paramiko # type: ignore -import typing - -class SSHAuth: - def __init__( - self, - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - key: typing.Optional[paramiko.RSAKey] = ..., - keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - key_filename: typing.Union[typing.List[str], str, None] = ..., - passphrase: typing.Optional[str] = ..., - ) -> None: ... - - @property - def username(self) -> typing.Optional[str]: ... - - @property - def public_key(self) -> typing.Optional[str]: ... - - @property - def key_filename(self) -> typing.Union[typing.List[str], str, None]: ... - - def enter_password(self, tgt: io.StringIO) -> None: ... - - def connect( - self, - client: typing.Union[paramiko.SSHClient, paramiko.Transport], - hostname: typing.Optional[str] = ..., - port: int = ..., - log: bool = ..., - ) -> None: ... - - def __eq__(self, other: typing.Any) -> bool: ... - - def __ne__(self, other: typing.Any) -> bool: ... - - def __deepcopy__(self, memo: typing.Any) -> SSHAuth: ... - - def __copy__(self) -> SSHAuth: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... diff --git a/exec_helpers/ssh_client.py b/exec_helpers/ssh_client.py index 3232596..ec076ea 100644 --- a/exec_helpers/ssh_client.py +++ b/exec_helpers/ssh_client.py @@ -116,11 +116,7 @@ def download( if self.exists(destination): self._sftp.get(destination, target) else: - self.logger.debug( - "Can't download %s because it doesn't exist", destination - ) + self.logger.debug("Can't download %s because it doesn't exist", destination) else: - self.logger.debug( - "Can't download %s because it is a directory", destination - ) + self.logger.debug("Can't download %s because it is a directory", destination) return os.path.exists(target) diff --git a/exec_helpers/subprocess_runner.py b/exec_helpers/subprocess_runner.py index ba3513f..89f022f 100644 --- a/exec_helpers/subprocess_runner.py +++ b/exec_helpers/subprocess_runner.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import unicode_literals +import abc import collections # noinspection PyCompatibility import concurrent.futures @@ -43,7 +44,7 @@ devnull = open(os.devnull) # subprocess.DEVNULL is py3.3+ -class SingletonMeta(type): +class SingletonMeta(abc.ABCMeta): """Metaclass for Singleton. Main goals: not need to implement __new__ in singleton classes @@ -52,7 +53,7 @@ class SingletonMeta(type): _instances = {} # type: typing.Dict[typing.Type, typing.Any] _lock = threading.RLock() # type: threading.RLock - def __call__(cls, *args, **kwargs): + def __call__(cls, *args, **kwargs): # type: (SingletonMeta, typing.Any, typing.Any) -> typing.Any """Singleton.""" with cls._lock: if cls not in cls._instances: @@ -65,7 +66,7 @@ def __prepare__( mcs, name, # type: str bases, # type: typing.Iterable[typing.Type] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> collections.OrderedDict # pylint: disable=unused-argument """Metaclass magic for object storage. @@ -100,10 +101,10 @@ def _exec_command( interface, # type: subprocess.Popen stdout, # type: typing.Optional[typing.IO] stderr, # type: typing.Optional[typing.IO] - timeout, # type: typing.Union[int, None] + timeout, # type: typing.Union[int, float, None] verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> exec_result.ExecResult """Get exit status from channel with timeout. @@ -112,11 +113,11 @@ def _exec_command( :param interface: Control interface :type interface: subprocess.Popen :param stdout: STDOUT pipe or file-like object - :type stdout: typing.Any + :type stdout: typing.Optional[typing.IO] :param stderr: STDERR pipe or file-like object - :type stderr: typing.Any + :type stderr: typing.Optional[typing.IO] :param timeout: Timeout for command execution - :type timeout: typing.Union[int, None] + :type timeout: typing.Union[int, float, None] :param verbose: produce verbose log record on command call :type verbose: bool :param log_mask_re: regex lookup rule to mask command for logger. @@ -127,8 +128,8 @@ def _exec_command( .. versionadded:: 1.2.0 """ - @threaded.threadpooled - def poll_stdout(): + @threaded.threadpooled # type: ignore + def poll_stdout(): # type: () -> None """Sync stdout poll.""" result.read_stdout( src=stdout, @@ -137,8 +138,8 @@ def poll_stdout(): ) interface.wait() # wait for the end of execution - @threaded.threadpooled - def poll_stderr(): + @threaded.threadpooled # type: ignore + def poll_stderr(): # type: () -> None """Sync stderr poll.""" result.read_stderr( src=stderr, @@ -185,19 +186,19 @@ def poll_stderr(): def execute_async( self, command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] + stdin=None, # type: typing.Union[str, bytes, bytearray, None] open_stdout=True, # type: bool open_stderr=True, # type: bool verbose=False, # type: bool log_mask_re=None, # type: typing.Optional[str] - **kwargs + **kwargs # type: typing.Any ): # type: (...) -> typing.Tuple[subprocess.Popen, None, typing.Optional[typing.IO], typing.Optional[typing.IO], ] """Execute command in async mode and return Popen with IO objects. :param command: Command for execution :type command: str :param stdin: pass STDIN text to the process - :type stdin: typing.Union[typing.AnyStr, bytearray, None] + :type stdin: typing.Union[str, bytes, bytearray, None] :param open_stdout: open STDOUT stream for read :type open_stdout: bool :param open_stderr: open STDERR stream for read @@ -218,7 +219,7 @@ def execute_async( """ cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) diff --git a/exec_helpers/subprocess_runner.pyi b/exec_helpers/subprocess_runner.pyi deleted file mode 100644 index 4d6c2b7..0000000 --- a/exec_helpers/subprocess_runner.pyi +++ /dev/null @@ -1,63 +0,0 @@ -import collections -import logging -import subprocess -import threading -import typing - -from exec_helpers import exec_result, api - -logger: logging.Logger -devnull: typing.IO - -class SingletonMeta(type): - _instances: typing.Dict[typing.Type, typing.Any] = ... - _lock: threading.RLock = ... - - def __call__(cls: SingletonMeta, *args: typing.Tuple, **kwargs: typing.Dict) -> typing.Any: ... - - @classmethod - def __prepare__( - mcs: typing.Type[SingletonMeta], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Dict - ) -> collections.OrderedDict: ... - -class Subprocess(api.ExecHelper, metaclass=SingletonMeta): - def __init__(self, log_mask_re: typing.Optional[str] = ...) -> None: ... - - def _exec_command( - self, - command: str, - interface: subprocess.Popen, - stdout: typing.Optional[typing.IO], - stderr: typing.Optional[typing.IO], - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - @typing.overload # type: ignore - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[subprocess.Popen, None, None, None]: ... - - @typing.overload - def execute_async( - self, - command: str, - stdin: None = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[subprocess.Popen, None, typing.IO, typing.IO]: ... diff --git a/setup.py b/setup.py index aadfbc0..2380c92 100644 --- a/setup.py +++ b/setup.py @@ -23,18 +23,9 @@ import ast import collections -from distutils.command import build_ext -import distutils.errors -import glob import os.path -import shutil import sys -try: - from Cython.Build import cythonize -except ImportError: - cythonize = None - import setuptools PY3 = sys.version_info[:2] > (2, 7) @@ -54,92 +45,6 @@ long_description = f.read() -def _extension(modpath): - """Make setuptools.Extension.""" - source_path = modpath.replace('.', '/') + '.py' - return setuptools.Extension( - modpath if PY3 else modpath.encode('utf-8'), - [source_path if PY3 else source_path.encode('utf-8')] - ) - - -requires_optimization = [ - _extension('exec_helpers.api'), - _extension('exec_helpers.constants'), - _extension('exec_helpers._log_templates'), - _extension('exec_helpers.exceptions'), - _extension('exec_helpers.exec_result'), - _extension('exec_helpers.proc_enums'), - _extension('exec_helpers._ssh_client_base'), - _extension('exec_helpers.ssh_auth'), - _extension('exec_helpers.ssh_client'), - _extension('exec_helpers.subprocess_runner'), -] - -if 'win32' != sys.platform: - requires_optimization.append( - _extension('exec_helpers.__init__') - ) - -ext_modules = cythonize( - requires_optimization, - compiler_directives=dict( - always_allow_keywords=True, - binding=True, - embedsignature=True, - overflowcheck=True, - language_level=3, - ) -) if cythonize is not None and PY3 else [] - - -class BuildFailed(Exception): - """For install clear scripts.""" - pass - - -class AllowFailRepair(build_ext.build_ext): - """This class allows C extension building to fail and repairs init.""" - - def run(self): - """Run.""" - try: - build_ext.build_ext.run(self) - - # Copy __init__.py back to repair package. - build_dir = os.path.abspath(self.build_lib) - root_dir = os.path.abspath(os.path.join(__file__, '..')) - target_dir = build_dir if not self.inplace else root_dir - - src_files = ( - os.path.join('exec_helpers', '__init__.py'), - ) - - for src_file in src_files: - src = os.path.join(root_dir, src_file) - dst = os.path.join(target_dir, src_file) - - if src != dst: - shutil.copyfile(src, dst) - except ( - distutils.errors.DistutilsPlatformError, - getattr(globals()['__builtins__'], 'FileNotFoundError', OSError) - ): - raise BuildFailed() - - def build_extension(self, ext): - """build_extension.""" - try: - build_ext.build_ext.build_extension(self, ext) - except ( - distutils.errors.CCompilerError, - distutils.errors.DistutilsExecError, - distutils.errors.DistutilsPlatformError, - ValueError - ): - raise BuildFailed() - - # noinspection PyUnresolvedReferences def get_simple_vars_from_src(src): """Get simple (string/number/boolean and None) assigned values from source. @@ -191,8 +96,6 @@ def get_simple_vars_from_src(src): ast.Str, ast.Num, ast.List, ast.Set, ast.Dict, ast.Tuple ) - if PY3: - ast_data += (ast.Bytes, ast.NameConstant,) tree = ast.parse(src) @@ -236,8 +139,6 @@ def get_simple_vars_from_src(src): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', @@ -264,7 +165,7 @@ def get_simple_vars_from_src(src): long_description=long_description, classifiers=classifiers, keywords=keywords, - python_requires='>=2.7.5,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<3.5', + python_requires='>=2.7.5,<3.0', # While setuptools cannot deal with pre-installed incompatible versions, # setting a lower bound is not harmful - it makes error messages cleaner. DO # NOT set an upper bound on setuptools, as that will lead to uninstallable @@ -276,27 +177,8 @@ def get_simple_vars_from_src(src): "!=36.2.0", install_requires=required, package_data={ - str('exec_helpers'): [ - os.path.basename(filename) - for filename in glob.glob(os.path.join('exec_helpers', '*.pyi')) - ] + [ - 'py.typed' - ], + str('exec_helpers'): ['py.typed'], }, ) -if cythonize is not None: - setup_args['ext_modules'] = ext_modules - setup_args['cmdclass'] = dict(build_ext=AllowFailRepair) - -try: - setuptools.setup(**setup_args) -except BuildFailed: - print( - '*' * 80 + '\n' - '* Build Failed!\n' - '* Use clear scripts version.\n' - '*' * 80 + '\n' - ) - del setup_args['ext_modules'] - del setup_args['cmdclass'] - setuptools.setup(**setup_args) + +setuptools.setup(**setup_args) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh deleted file mode 100755 index 4c357cc..0000000 --- a/tools/build-wheels.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -PYTHON_VERSIONS="cp34-cp34m" - -# Avoid creation of __pycache__/*.py[c|o] -export PYTHONDONTWRITEBYTECODE=1 - -package_name="$1" -if [ -z "$package_name" ] -then - &>2 echo "Please pass package name as a first argument of this script ($0)" - exit 1 -fi - -arch=`uname -m` - -# Clean-up -rm -rf /io/.tox -rm -rf /io/*.egg-info -rm -rf /io/.pytest_cache -find -name *.py[co] -delete - -echo -echo -echo "Compile wheels" -for PYTHON in ${PYTHON_VERSIONS}; do - /opt/python/${PYTHON}/bin/pip install -U pip setuptools wheel - /opt/python/${PYTHON}/bin/pip install -r /io/build_requirements.txt - /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/ - cd /io - /opt/python/${PYTHON}/bin/python setup.py bdist_wheel -done - -echo -echo -echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/${package_name}*${arch}.whl; do - echo "Repairing $whl..." - auditwheel repair "$whl" -w /io/dist/ -done - -echo "Cleanup OS specific wheels" -rm -fv /io/dist/*-linux_*.whl - -echo -echo -echo "Install packages and test" -echo "dist directory:" -ls /io/dist - -for PYTHON in ${PYTHON_VERSIONS}; do - echo - echo -n "Test $PYTHON: $package_name " - /opt/python/${PYTHON}/bin/python -c "import platform;print(platform.platform())" - /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist - /opt/python/${PYTHON}/bin/pip install pytest - /opt/python/${PYTHON}/bin/py.test -vv /io/test -done - -find /io/dist/ -type f -not -name "*$package_name*" -delete -rm -rf /io/build -rm -rf /io/*.egg-info -rm -rf /io/.pytest_cache -chmod -v a+rwx /io/dist -chmod -v a+rw /io/dist/* diff --git a/tools/run_docker.sh b/tools/run_docker.sh deleted file mode 100755 index 1ea8c3a..0000000 --- a/tools/run_docker.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -package_name="$1" -if [ -z "$package_name" ] -then - &>2 echo "Please pass package name as a first argument of this script ($0)" - exit 1 -fi - -manylinux1_image_prefix="quay.io/pypa/manylinux1_" -dock_ext_args="" -declare -A docker_pull_pids=() # This syntax requires at least bash v4 - -for arch in x86_64 i686 -do - docker pull "${manylinux1_image_prefix}${arch}" & - docker_pull_pids[$arch]=$! -done - -for arch in x86_64 i686 -do - echo - echo - arch_pull_pid=${docker_pull_pids[$arch]} - echo waiting for docker pull pid $arch_pull_pid to complete downloading container for $arch arch... - wait $arch_pull_pid # await for docker image for current arch to be pulled from hub - [ $arch == "i686" ] && dock_ext_args="linux32" - - echo Building wheel for $arch arch - docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" - - dock_ext_args="" # Reset docker args, just in case -done diff --git a/tox.ini b/tox.ini index 8c0dde2..e78d442 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] minversion = 2.0 -envlist = pep8, pep257, py{27,34,py}, pylint, bandit, py34-nocov, mypy, docs +envlist = pep8, pep257, py{27,py}, pylint, bandit, mypy, docs skipsdist = True skip_missing_interpreters = True @@ -20,7 +20,6 @@ deps = pytest-cov pytest-html pytest-sugar - py34-nocov: Cython -r{toxinidir}/CI_REQUIREMENTS.txt py{27,py}: mock @@ -29,19 +28,11 @@ commands = py.test --junitxml=unit_result.xml --cov-report html --self-contained-html --html=report.html --cov-config .coveragerc --cov=exec_helpers {posargs:test} coverage report --fail-under 97 -[testenv:py34-nocov] -usedevelop = False -commands = - python setup.py bdist_wheel - pip install exec_helpers --no-index -f dist - py.test -vvv {posargs:test} - [testenv:venv] commands = {posargs:} [tox:travis] 2.7 = install, py27, -3.4 = py34, pypy = install, pypy, [testenv:pep8] @@ -110,11 +101,13 @@ count = True max-line-length = 120 [testenv:docs] +usedevelop = False deps = sphinx commands = python setup.py build_sphinx [testenv:bandit] +usedevelop = False deps = bandit commands = bandit -r exec_helpers @@ -125,6 +118,7 @@ deps = commands = pipdeptree [testenv:mypy] +usedevelop = False deps = mypy>=0.620 -r{toxinidir}/CI_REQUIREMENTS.txt From da5ffcfb1ba7586bca31c0306d67a76424ab13f1 Mon Sep 17 00:00:00 2001 From: Alexey Stepanov Date: Tue, 4 Sep 2018 12:23:00 +0200 Subject: [PATCH 2/3] Remove tests, which are expected no python 2.7 compatible Signed-off-by: Alexey Stepanov --- exec_helpers/exceptions.py | 12 ++--- test/test_subprocess_runner.py | 84 ---------------------------------- 2 files changed, 3 insertions(+), 93 deletions(-) diff --git a/exec_helpers/exceptions.py b/exec_helpers/exceptions.py index 87d07b5..c3c9f19 100644 --- a/exec_helpers/exceptions.py +++ b/exec_helpers/exceptions.py @@ -140,9 +140,7 @@ def __init__( super(CalledProcessError, self).__init__(message) @property - def returncode( - self - ): # type: () -> typing.Union[int, proc_enums.ExitCodes] + def returncode(self): # type: () -> typing.Union[int, proc_enums.ExitCodes] """Command return code.""" return self.result.exit_code @@ -192,9 +190,7 @@ def __init__( :param results: all results :type results: typing.Dict[typing.Tuple[str, int], ExecResult] :param expected: expected return codes - :type expected: typing.Optional[typing.List[ - typing.List[typing.Union[int, proc_enums.ExitCodes]] - ] + :type expected: typing.Optional[typing.List[typing.List[typing.Union[int, proc_enums.ExitCodes]]] """ expected = expected or [proc_enums.ExitCodes.EX_OK] self.expected = proc_enums.exit_codes_to_enums(expected) @@ -244,9 +240,7 @@ def __init__( :param results: all results :type results: typing.Dict[typing.Tuple[str, int], ExecResult] :param expected: expected return codes - :type expected: typing.Optional[typing.List[ - typing.List[typing.Union[int, proc_enums.ExitCodes]] - ] + :type expected: typing.Optional[typing.List[typing.List[typing.Union[int, proc_enums.ExitCodes]]] """ expected = expected or [proc_enums.ExitCodes.EX_OK] self.expected = proc_enums.exit_codes_to_enums(expected) diff --git a/test/test_subprocess_runner.py b/test/test_subprocess_runner.py index 1999214..967274e 100644 --- a/test/test_subprocess_runner.py +++ b/test/test_subprocess_runner.py @@ -550,48 +550,6 @@ def test_006_check_stdin_bytearray( mock.call.close() ]) - @unittest.skipIf(six.PY2, 'Not implemented exception') - def test_007_check_stdin_fail_broken_pipe( - self, - popen, # type: mock.MagicMock - logger # type: mock.MagicMock - ): # type: (...) -> None - stdin = b'this is a line' - - popen_obj, exp_result = self.prepare_close(popen, cmd=print_stdin, stdout_override=[stdin]) - - pipe_err = BrokenPipeError() - pipe_err.errno = errno.EPIPE - - stdin_mock = mock.Mock() - stdin_mock.attach_mock(mock.Mock(side_effect=pipe_err), 'write') - popen_obj.attach_mock(stdin_mock, 'stdin') - - runner = exec_helpers.Subprocess() - - # noinspection PyTypeChecker - result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual(result, exp_result) - - popen.assert_has_calls(( - mock.call( - args=[print_stdin], - cwd=None, - env=None, - shell=True, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=False, - ), - )) - - stdin_mock.assert_has_calls([ - mock.call.write(stdin), - mock.call.close() - ]) - logger.warning.assert_called_once_with('STDIN Send failed: broken PIPE') - def test_008_check_stdin_fail_closed_win( self, popen, # type: mock.MagicMock @@ -654,48 +612,6 @@ def test_009_check_stdin_fail_write( runner.execute_async(print_stdin, stdin=stdin) popen_obj.kill.assert_called_once() - @unittest.skipIf(six.PY2, 'Not implemented exception') - def test_010_check_stdin_fail_close_pipe( - self, - popen, # type: mock.MagicMock - logger # type: mock.MagicMock - ): # type: (...) -> None - stdin = b'this is a line' - - popen_obj, exp_result = self.prepare_close(popen, cmd=print_stdin, stdout_override=[stdin]) - - pipe_err = BrokenPipeError() - pipe_err.errno = errno.EPIPE - - stdin_mock = mock.Mock() - stdin_mock.attach_mock(mock.Mock(side_effect=pipe_err), 'close') - popen_obj.attach_mock(stdin_mock, 'stdin') - - runner = exec_helpers.Subprocess() - - # noinspection PyTypeChecker - result = runner.execute(print_stdin, stdin=stdin) - self.assertEqual(result, exp_result) - - popen.assert_has_calls(( - mock.call( - args=[print_stdin], - cwd=None, - env=None, - shell=True, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=False, - ), - )) - - stdin_mock.assert_has_calls([ - mock.call.write(stdin), - mock.call.close() - ]) - logger.warning.assert_not_called() - def test_011_check_stdin_fail_close_pipe_win( self, popen, # type: mock.MagicMock From d7d99a31fa12a15798be12ca8b977c99249962d5 Mon Sep 17 00:00:00 2001 From: Alexey Stepanov Date: Tue, 4 Sep 2018 12:37:52 +0200 Subject: [PATCH 3/3] remove useless import Signed-off-by: Alexey Stepanov --- test/test_subprocess_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_subprocess_runner.py b/test/test_subprocess_runner.py index 967274e..3ab78f0 100644 --- a/test/test_subprocess_runner.py +++ b/test/test_subprocess_runner.py @@ -26,7 +26,6 @@ import unittest import mock -import six import exec_helpers from exec_helpers import subprocess_runner