diff --git a/.travis.yml b/.travis.yml index 04d2c15..a641be9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,10 @@ sudo: false language: python os: linux python: -- 2.7 -- 3.4 - 3.5 - 3.6 - &mainstream_python 3.7-dev -- &pypy pypy -- pypy3.5 +- &pypy pypy3.5 install: - &upgrade_python_toolset pip install --upgrade pip setuptools wheel - pip install tox-travis diff --git a/CI_REQUIREMENTS.txt b/CI_REQUIREMENTS.txt index 08d9d53..709ad69 100644 --- a/CI_REQUIREMENTS.txt +++ b/CI_REQUIREMENTS.txt @@ -1,4 +1,2 @@ mock # no assert_called_once in py35 -futures>=3.1; python_version == "2.7" -enum34>=1.1; python_version == "2.7" -r requirements.txt diff --git a/README.rst b/README.rst index de9cf16..e3ca5c4 100644 --- a/README.rst +++ b/README.rst @@ -42,14 +42,13 @@ Pros: :: - Python 2.7 - Python 3.4 Python 3.5 Python 3.6 Python 3.7 - PyPy PyPy3 3.5+ +.. note:: For Python 2.7, 3.4 and PyPy please use versions 1.x.x + This package includes: * `SSHClient` - historically the first one helper, which used for SSH connections and requires memorization diff --git a/appveyor.yml b/appveyor.yml index d3965a7..d2a0f11 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,22 +4,6 @@ environment: secure: TCKGf77kkVeo2Pbd+lQY5Q== matrix: - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7.x" # currently 2.7.11 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" # currently 2.7.11 - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5.x" # currently 3.5.1 PYTHON_ARCH: "32" diff --git a/exec_helpers/__init__.py b/exec_helpers/__init__.py index d5c9590..34b2e3f 100644 --- a/exec_helpers/__init__.py +++ b/exec_helpers/__init__.py @@ -14,8 +14,6 @@ """Execution helpers for simplified usage of subprocess and ssh.""" -from __future__ import absolute_import - from .proc_enums import ExitCodes from .exceptions import ( @@ -48,7 +46,7 @@ 'ExecResult', ) -__version__ = '1.3.6' +__version__ = '2.0.0' __author__ = "Alexey Stepanov" __author_email__ = 'penguinolog@gmail.com' __maintainers__ = { diff --git a/exec_helpers/_log_templates.py b/exec_helpers/_log_templates.py index 56cda0f..c0d9712 100644 --- a/exec_helpers/_log_templates.py +++ b/exec_helpers/_log_templates.py @@ -16,10 +16,6 @@ """Text templates for logging.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - CMD_EXEC = "Executing command:\n{cmd!r}\n" CMD_WAIT_ERROR = ( diff --git a/exec_helpers/_ssh_client_base.py b/exec_helpers/_ssh_client_base.py index 068a31e..fbf578d 100644 --- a/exec_helpers/_ssh_client_base.py +++ b/exec_helpers/_ssh_client_base.py @@ -16,13 +16,8 @@ """SSH client helper based on Paramiko. Base class.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import base64 import collections -# noinspection PyCompatibility import concurrent.futures import copy import logging @@ -35,10 +30,9 @@ import warnings import advanced_descriptors -import paramiko -import tenacity +import paramiko # type: ignore +import tenacity # type: ignore import threaded -import six from exec_helpers import api from exec_helpers import constants @@ -96,28 +90,28 @@ class _MemorizedSSH(type): __cache = {} # type: typing.Dict[typing.Tuple[str, int], SSHClientBase] @classmethod - def __prepare__( - mcs, # type: typing.Type[_MemorizedSSH] - name, # type: str - bases, # type: typing.Iterable[typing.Type] - **kwargs # type: typing.Dict - ): # type: (...) -> collections.OrderedDict # pylint: disable=unused-argument + def __prepare__( # pylint: disable=unused-argument + mcs: typing.Type['_MemorizedSSH'], + name: str, + bases: typing.Iterable[typing.Type], + **kwargs: typing.Dict + ) -> collections.OrderedDict: """Metaclass magic for object storage. .. versionadded:: 1.2.0 """ return collections.OrderedDict() # pragma: no cover - def __call__( - cls, # type: _MemorizedSSH - host, # type: str - port=22, # type: int - username=None, # type: typing.Optional[str] - password=None, # type: typing.Optional[str] - private_keys=None, # type: typing.Optional[typing.Iterable[paramiko.RSAKey]] - auth=None, # type: typing.Optional[ssh_auth.SSHAuth] - verbose=True, # type: bool - ): # type: (...) -> SSHClientBase + def __call__( # type: ignore + cls: '_MemorizedSSH', + host: str, + port: int = 22, + username: typing.Optional[str] = None, + password: typing.Optional[str] = None, + private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = None, + auth: typing.Optional[ssh_auth.SSHAuth] = None, + verbose: bool = True, + ) -> 'SSHClientBase': """Main memorize method: check for cached instance and return it. :type host: str @@ -153,7 +147,7 @@ def __call__( # If we have only cache reference and temporary getrefcount # reference: close connection before deletion cls.__cache[key].logger.debug('Closing as unused') - cls.__cache[key].close() + cls.__cache[key].close() # type: ignore del cls.__cache[key] # noinspection PyArgumentList ssh = super( @@ -167,12 +161,12 @@ def __call__( return ssh @classmethod - def clear_cache(mcs): # type: (typing.Type[_MemorizedSSH]) -> None + def clear_cache(mcs: typing.Type['_MemorizedSSH']) -> None: """Clear cached connections for initialize new instance on next call. getrefcount is used to check for usage. """ - n_count = 3 if six.PY3 else 4 + n_count = 3 # PY3: cache, ssh, temporary # PY4: cache, values mapping, ssh, temporary for ssh in mcs.__cache.values(): @@ -181,18 +175,18 @@ def clear_cache(mcs): # type: (typing.Type[_MemorizedSSH]) -> None sys.getrefcount(ssh) == n_count ): # pragma: no cover ssh.logger.debug('Closing as unused') - ssh.close() + ssh.close() # type: ignore mcs.__cache = {} @classmethod - def close_connections(mcs): # type: (typing.Type[_MemorizedSSH]) -> None + def close_connections(mcs: typing.Type['_MemorizedSSH']) -> None: """Close connections for selected or all cached records.""" for ssh in mcs.__cache.values(): if ssh.is_alive: - ssh.close() + ssh.close() # type: ignore -class SSHClientBase(six.with_metaclass(_MemorizedSSH, api.ExecHelper)): +class SSHClientBase(api.ExecHelper, metaclass=_MemorizedSSH): """SSH Client helper.""" __slots__ = ( @@ -200,7 +194,7 @@ class SSHClientBase(six.with_metaclass(_MemorizedSSH, api.ExecHelper)): '__sudo_mode', '__keepalive_mode', '__verbose', ) - class __get_sudo(object): + class __get_sudo: """Context manager for call commands with sudo.""" __slots__ = ( @@ -211,9 +205,9 @@ class __get_sudo(object): def __init__( self, - ssh, # type: SSHClientBase - enforce=None # type: typing.Optional[bool] - ): # type: (...) -> None + ssh: 'SSHClientBase', + enforce: typing.Optional[bool] = None + ) -> None: """Context manager for call commands with sudo. :type ssh: SSHClient @@ -223,15 +217,15 @@ def __init__( self.__sudo_status = ssh.sudo_mode self.__enforce = enforce - def __enter__(self): + def __enter__(self) -> None: self.__sudo_status = self.__ssh.sudo_mode if self.__enforce is not None: self.__ssh.sudo_mode = self.__enforce - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: self.__ssh.sudo_mode = self.__sudo_status - class __get_keepalive(object): + class __get_keepalive: """Context manager for keepalive management.""" __slots__ = ( @@ -242,9 +236,9 @@ class __get_keepalive(object): def __init__( self, - ssh, # type: SSHClientBase - enforce=True # type: bool - ): # type: (...) -> None + ssh: 'SSHClientBase', + enforce: bool = True + ) -> None: """Context manager for keepalive management. :type ssh: SSHClient @@ -255,17 +249,17 @@ def __init__( self.__keepalive_status = ssh.keepalive_mode self.__enforce = enforce - def __enter__(self): + def __enter__(self) -> None: self.__keepalive_status = self.__ssh.keepalive_mode if self.__enforce is not None: self.__ssh.keepalive_mode = self.__enforce self.__ssh.__enter__() - def __exit__(self, exc_type, exc_val, exc_tb): - self.__ssh.__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) + def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: + self.__ssh.__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) # type: ignore self.__ssh.keepalive_mode = self.__keepalive_status - def __hash__(self): + def __hash__(self) -> int: """Hash for usage as dict keys.""" return hash(( self.__class__, @@ -275,14 +269,14 @@ def __hash__(self): def __init__( self, - host, # type: str - port=22, # type: int - username=None, # type: typing.Optional[str] - password=None, # type: typing.Optional[str] - private_keys=None, # type: typing.Optional[typing.Iterable[paramiko.RSAKey]] - auth=None, # type: typing.Optional[ssh_auth.SSHAuth] - verbose=True, # type: bool - ): # type: (...) -> None + host: str, + port: int = 22, + username: typing.Optional[str] = None, + password: typing.Optional[str] = None, + private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = None, + auth: typing.Optional[ssh_auth.SSHAuth] = None, + verbose: bool = True, + ) -> None: """SSHClient helper. :param host: remote hostname @@ -333,7 +327,7 @@ def __init__( self.__connect() @property - def auth(self): # type: () -> ssh_auth.SSHAuth + def auth(self) -> ssh_auth.SSHAuth: """Internal authorisation object. Attention: this public property is mainly for inheritance, @@ -346,7 +340,7 @@ def auth(self): # type: () -> ssh_auth.SSHAuth return self.__auth @property - def hostname(self): # type: () -> str + def hostname(self) -> str: """Connected remote host name. :rtype: str @@ -354,7 +348,7 @@ def hostname(self): # type: () -> str return self.__hostname @property - def port(self): # type: () -> int + def port(self) -> int: """Connected remote port number. :rtype: int @@ -362,21 +356,21 @@ def port(self): # type: () -> int return self.__port @property - def is_alive(self): # type: () -> bool + def is_alive(self) -> bool: """Paramiko status: ready to use|reconnect required. :rtype: bool """ return self.__ssh.get_transport() is not None - def __repr__(self): # type: () -> str + def __repr__(self) -> str: """Representation for debug purposes.""" return '{cls}(host={host}, port={port}, auth={auth!r})'.format( cls=self.__class__.__name__, host=self.hostname, port=self.port, auth=self.auth ) - def __str__(self): # type: () -> str # pragma: no cover + def __str__(self) -> str: # pragma: no cover """Representation for debug purposes.""" return '{cls}(host={host}, port={port}) for user {user}'.format( cls=self.__class__.__name__, host=self.hostname, port=self.port, @@ -384,7 +378,7 @@ def __str__(self): # type: () -> str # pragma: no cover ) @property - def _ssh(self): # type: () -> paramiko.SSHClient + def _ssh(self) -> paramiko.SSHClient: """ssh client object getter for inheritance support only. Attention: ssh client object creation and change @@ -394,13 +388,13 @@ def _ssh(self): # type: () -> paramiko.SSHClient """ return self.__ssh - @tenacity.retry( + @tenacity.retry( # type: ignore retry=tenacity.retry_if_exception_type(paramiko.SSHException), stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_fixed(3), reraise=True, ) - def __connect(self): + def __connect(self) -> None: """Main method for connection open.""" with self.lock: self.auth.connect( @@ -408,7 +402,7 @@ def __connect(self): hostname=self.hostname, port=self.port, log=self.__verbose) - def __connect_sftp(self): + def __connect_sftp(self) -> None: """SFTP connection opener.""" with self.lock: try: @@ -419,7 +413,7 @@ def __connect_sftp(self): ) @property - def _sftp(self): # type: () -> paramiko.sftp_client.SFTPClient + def _sftp(self) -> paramiko.sftp_client.SFTPClient: """SFTP channel access for inheritance. :rtype: paramiko.sftp_client.SFTPClient @@ -434,7 +428,7 @@ def _sftp(self): # type: () -> paramiko.sftp_client.SFTPClient raise paramiko.SSHException('SFTP connection failed') @advanced_descriptors.SeparateClassMethod - def close(self): + def close(self) -> None: """Close SSH and SFTP sessions.""" with self.lock: # noinspection PyBroadException @@ -453,16 +447,14 @@ def close(self): ) # noinspection PyMethodParameters - @close.class_method - def close( # pylint: disable=no-self-argument - cls # type: typing.Type[SSHClientBase] - ): # type: (...) -> None + @close.class_method # type: ignore + def close(cls: typing.Type['SSHClientBase']) -> None: # pylint: disable=no-self-argument """Close all memorized SSH and SFTP sessions.""" # noinspection PyUnresolvedReferences cls.__class__.close_connections() @classmethod - def _clear_cache(cls): + def _clear_cache(cls: typing.Type['SSHClientBase']) -> None: """Enforce clear memorized records.""" warnings.warn( '_clear_cache() is dangerous and not recommended for normal use!', @@ -470,7 +462,7 @@ def _clear_cache(cls): ) _MemorizedSSH.clear_cache() - def __del__(self): + def __del__(self) -> None: """Destructor helper: close channel and threads BEFORE closing others. Due to threading in paramiko, default destructor could generate asserts @@ -487,7 +479,7 @@ def __del__(self): ) self.__sftp = None - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: """Exit context manager. .. versionchanged:: 1.0.0 disconnect enforced on close @@ -495,11 +487,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): .. versionchanged:: 1.2.1 disconnect enforced on close only not in keepalive mode """ if not self.__keepalive_mode: - self.close() + self.close() # type: ignore super(SSHClientBase, self).__exit__(exc_type, exc_val, exc_tb) @property - def sudo_mode(self): # type: () -> bool + def sudo_mode(self) -> bool: """Persistent sudo mode for connection object. :rtype: bool @@ -507,7 +499,7 @@ def sudo_mode(self): # type: () -> bool return self.__sudo_mode @sudo_mode.setter - def sudo_mode(self, mode): # type: (bool) -> None + def sudo_mode(self, mode: bool) -> None: """Persistent sudo mode change for connection object. :type mode: bool @@ -515,7 +507,7 @@ def sudo_mode(self, mode): # type: (bool) -> None self.__sudo_mode = bool(mode) @property - def keepalive_mode(self): # type: () -> bool + def keepalive_mode(self) -> bool: """Persistent keepalive mode for connection object. :rtype: bool @@ -523,17 +515,17 @@ def keepalive_mode(self): # type: () -> bool return self.__keepalive_mode @keepalive_mode.setter - def keepalive_mode(self, mode): # type: (bool) -> None + def keepalive_mode(self, mode: bool) -> None: """Persistent keepalive mode change for connection object. :type mode: bool """ self.__keepalive_mode = bool(mode) - def reconnect(self): # type: () -> None + def reconnect(self) -> None: """Reconnect SSH session.""" with self.lock: - self.close() + self.close() # type: ignore self.__ssh = paramiko.SSHClient() self.__ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -542,8 +534,8 @@ def reconnect(self): # type: () -> None def sudo( self, - enforce=None # type: typing.Optional[bool] - ): # type: (...) -> typing.ContextManager + enforce: typing.Optional[bool] = None + ) -> 'typing.ContextManager': """Call contextmanager for sudo mode change. :param enforce: Enforce sudo enabled or disabled. By default: None @@ -553,8 +545,8 @@ def sudo( def keepalive( self, - enforce=True # type: bool - ): # type: (...) -> typing.ContextManager + enforce: bool = True + ) -> 'typing.ContextManager': """Call contextmanager with keepalive mode change. :param enforce: Enforce keepalive enabled or disabled. @@ -567,14 +559,14 @@ def keepalive( def execute_async( self, - command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] - open_stdout=True, # type: bool - open_stderr=True, # type: bool - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> _type_execute_async + command: str, + stdin: typing.Union[bytes, str, bytearray, None] = None, + open_stdout: bool = True, + open_stderr: bool = True, + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> _type_execute_async: """Execute command in async mode and return channel with IO objects. :param command: Command for execution @@ -606,7 +598,7 @@ def execute_async( log_mask_re=log_mask_re ) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) @@ -647,21 +639,21 @@ def execute_async( def _exec_command( self, - command, # type: str - interface, # type: paramiko.channel.Channel - stdout, # type: paramiko.channel.ChannelFile - stderr, # type: paramiko.channel.ChannelFile - timeout, # type: typing.Union[int, None] - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + interface: paramiko.channel.Channel, + stdout: typing.Optional[paramiko.ChannelFile], + stderr: typing.Optional[paramiko.ChannelFile], + timeout: typing.Union[int, float, None], + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Get exit status from channel with timeout. :type command: str :type interface: paramiko.channel.Channel - :type stdout: paramiko.channel.ChannelFile - :type stderr: paramiko.channel.ChannelFile + :type stdout: typing.Optional[paramiko.ChannelFile] + :type stderr: typing.Optional[paramiko.ChannelFile] :type timeout: typing.Union[int, None] :type verbose: bool :param log_mask_re: regex lookup rule to mask command for logger. @@ -672,7 +664,7 @@ def _exec_command( .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - def poll_streams(): + def poll_streams() -> None: """Poll FIFO buffers if data available.""" if stdout and interface.recv_ready(): result.read_stdout( @@ -687,8 +679,8 @@ def poll_streams(): verbose=verbose ) - @threaded.threadpooled - def poll_pipes(stop, ): # type: (threading.Event) -> None + @threaded.threadpooled # type: ignore + def poll_pipes(stop: threading.Event) -> None: """Polling task for FIFO buffers. :type stop: Event @@ -743,19 +735,19 @@ def poll_pipes(stop, ): # type: (threading.Event) -> None timeout=timeout ) self.logger.debug(wait_err_msg) - raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) + raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) # type: ignore def execute_through_host( self, - hostname, # type: str - command, # type: str - auth=None, # type: typing.Optional[ssh_auth.SSHAuth] - target_port=22, # type: int - verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - get_pty=False, # type: bool - **kwargs - ): # type: (...) -> exec_result.ExecResult + hostname: str, + command: str, + auth: typing.Optional[ssh_auth.SSHAuth] = None, + target_port: int = 22, + verbose: bool = False, + timeout: typing.Union[int, None] = constants.DEFAULT_TIMEOUT, + get_pty: bool = False, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Execute command on remote host through currently connected host. :param hostname: target hostname @@ -778,11 +770,11 @@ def execute_through_host( .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - cmd_for_log = self._mask_command( + cmd_for_log = self._mask_command( # type: ignore cmd=command, log_mask_re=kwargs.get('log_mask_re', None) ) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) @@ -816,7 +808,7 @@ def execute_through_host( channel.exec_command(command) # nosec # Sanitize on caller side # noinspection PyDictCreation - result = self._exec_command( + result = self._exec_command( # type: ignore command, channel, stdout, stderr, timeout, verbose=verbose, log_mask_re=kwargs.get('log_mask_re', None), ) @@ -828,13 +820,13 @@ def execute_through_host( @classmethod def execute_together( cls, - remotes, # type: typing.Iterable[SSHClientBase] - command, # type: str - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - expected=None, # type: typing.Optional[typing.Iterable[int]] - raise_on_err=True, # type: bool - **kwargs - ): # type: (...) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] + remotes: typing.Iterable['SSHClientBase'], + command: str, + timeout: typing.Union[int, None] = constants.DEFAULT_TIMEOUT, + expected: typing.Optional[typing.Iterable[int]] = None, + raise_on_err: bool = True, + **kwargs: typing.Dict + ) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult]: """Execute command on multiple remotes in async mode. :param remotes: Connections to execute on @@ -855,15 +847,15 @@ def execute_together( .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - @threaded.threadpooled - def get_result(): # type: () -> exec_result.ExecResult + @threaded.threadpooled # type: ignore + def get_result(remote: 'SSHClientBase') -> exec_result.ExecResult: """Get result from remote call.""" ( chan, _, stderr, stdout, - ) = remote.execute_async( + ) = remote.execute_async( # type: ignore command, **kwargs ) # type: _type_execute_async @@ -872,7 +864,7 @@ def get_result(): # type: () -> exec_result.ExecResult exit_code = chan.recv_exit_status() # pylint: disable=protected-access - cmd_for_log = remote._mask_command( + cmd_for_log = remote._mask_command( # type: ignore cmd=command, log_mask_re=kwargs.get('log_mask_re', None) ) @@ -889,14 +881,11 @@ def get_result(): # type: () -> exec_result.ExecResult expected = expected or [proc_enums.ExitCodes.EX_OK] expected = proc_enums.exit_codes_to_enums(expected) - futures = {} + futures = {remote: get_result(remote) for remote in set(remotes)} # Use distinct remotes results = {} errors = {} raised_exceptions = {} - for remote in set(remotes): # Use distinct remotes - futures[remote] = get_result() - ( _, not_done, @@ -910,7 +899,7 @@ def get_result(): # type: () -> exec_result.ExecResult for ( remote, - future, + future, # type: ignore ) in futures.items(): # type: SSHClientBase, concurrent.futures.Future try: result = future.result() @@ -934,7 +923,7 @@ def get_result(): # type: () -> exec_result.ExecResult ) return results - def open(self, path, mode='r'): # type: (str, str) -> paramiko.SFTPFile + def open(self, path: str, mode: str = 'r') -> paramiko.SFTPFile: """Open file on remote using SFTP session. :type path: str @@ -943,7 +932,7 @@ def open(self, path, mode='r'): # type: (str, str) -> paramiko.SFTPFile """ return self._sftp.open(path, mode) # pragma: no cover - def exists(self, path): # type: (str) -> bool + def exists(self, path: str) -> bool: """Check for file existence using SFTP session. :type path: str @@ -955,7 +944,7 @@ def exists(self, path): # type: (str) -> bool except IOError: return False - def stat(self, path): # type: (str) -> paramiko.sftp_attr.SFTPAttributes + def stat(self, path: str) -> paramiko.sftp_attr.SFTPAttributes: """Get stat info for path with following symlinks. :type path: str @@ -965,9 +954,9 @@ def stat(self, path): # type: (str) -> paramiko.sftp_attr.SFTPAttributes def utime( self, - path, # type: str - times=None # type: typing.Optional[typing.Tuple[int, int]] - ): # type: (...) -> None + path: str, + times: typing.Optional[typing.Tuple[int, int]] = None + ) -> None: """Set atime, mtime. :param path: filesystem object path @@ -977,9 +966,9 @@ def utime( .. versionadded:: 1.0.0 """ - return self._sftp.utime(path, times) # pragma: no cover + self._sftp.utime(path, times) # pragma: no cover - def isfile(self, path): # type: (str) -> bool + def isfile(self, path: str) -> bool: """Check, that path is file using SFTP session. :type path: str @@ -987,11 +976,11 @@ def isfile(self, path): # type: (str) -> bool """ try: attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFREG != 0 + return attrs.st_mode & stat.S_IFREG != 0 # type: ignore except IOError: return False - def isdir(self, path): # type: (str) -> bool + def isdir(self, path: str) -> bool: """Check, that path is directory using SFTP session. :type path: str @@ -999,6 +988,6 @@ def isdir(self, path): # type: (str) -> bool """ try: attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFDIR != 0 + return attrs.st_mode & stat.S_IFDIR != 0 # type: ignore except IOError: return False diff --git a/exec_helpers/_ssh_client_base.pyi b/exec_helpers/_ssh_client_base.pyi deleted file mode 100644 index 61ecce9..0000000 --- a/exec_helpers/_ssh_client_base.pyi +++ /dev/null @@ -1,163 +0,0 @@ -import collections -import typing - -import paramiko # type: ignore - -from exec_helpers import exec_result, ssh_auth, api - -CPYTHON: bool = ... - - -class _MemorizedSSH(type): - @classmethod - def __prepare__( - mcs: typing.Type[_MemorizedSSH], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Dict - ) -> collections.OrderedDict: ... - - def __call__( # type: ignore - cls: _MemorizedSSH, - host: str, - port: int = ..., - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - verbose: bool = ..., - ) -> SSHClientBase: ... - - @classmethod - def clear_cache(mcs: typing.Type[_MemorizedSSH]) -> None: ... - - @classmethod - def close_connections(mcs: typing.Type[_MemorizedSSH]) -> None: ... - - -class SSHClientBase(api.ExecHelper, metaclass=_MemorizedSSH): - def __hash__(self) -> int: ... - - def __init__( - self, - host: str, - port: int = ..., - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - verbose: bool = ..., - ) -> None: ... - - @property - def auth(self) -> ssh_auth.SSHAuth: ... - - @property - def hostname(self) -> str: ... - - @property - def port(self) -> int: ... - - @property - def is_alive(self) -> bool: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... - - @property - def _ssh(self) -> paramiko.SSHClient: ... - - @property - def _sftp(self) -> paramiko.sftp_client.SFTPClient: ... - - @classmethod - def close(cls: typing.Union[SSHClientBase, typing.Type[SSHClientBase]]) -> None: ... - - @classmethod - def _clear_cache(cls) -> None: ... - - def __del__(self) -> None: ... - - def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: ... - - @property - def sudo_mode(self) -> bool: ... - - @sudo_mode.setter - def sudo_mode(self, mode: bool) -> None: ... - - @property - def keepalive_mode(self) -> bool: ... - - @keepalive_mode.setter - def keepalive_mode(self, mode: bool) -> None: ... - - def reconnect(self) -> None: ... - - def sudo(self, enforce: typing.Optional[bool] = ...) -> typing.ContextManager: ... - - def keepalive(self, enforce: bool = ...) -> typing.ContextManager: ... - - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[ - paramiko.Channel, - paramiko.ChannelFile, - typing.Optional[paramiko.ChannelFile], - typing.Optional[paramiko.ChannelFile], - ]: ... - - def _exec_command( - self, - command: str, - interface: paramiko.channel.Channel, - stdout: paramiko.channel.ChannelFile, - stderr: paramiko.channel.ChannelFile, - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - def execute_through_host( - self, - hostname: str, - command: str, - auth: typing.Optional[ssh_auth.SSHAuth] = ..., - target_port: int = ..., - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - get_pty: bool = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - @classmethod - def execute_together( - cls, - remotes: typing.Iterable[SSHClientBase], - command: str, - timeout: typing.Union[int, None] = ..., - expected: typing.Optional[typing.Iterable[int]] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Dict - ) -> typing.Dict[typing.Tuple[str, int], exec_result.ExecResult]: ... - - def open(self, path: str, mode: str = ...) -> paramiko.SFTPFile: ... - - def exists(self, path: str) -> bool: ... - - def stat(self, path: str) -> paramiko.sftp_attr.SFTPAttributes: ... - - def utime(self, path: str, times: typing.Optional[typing.Tuple[int, int]] = ...) -> None: ... - - def isfile(self, path: str) -> bool: ... - - def isdir(self, path: str) -> bool: ... diff --git a/exec_helpers/api.py b/exec_helpers/api.py index 8c55bb4..16941b2 100644 --- a/exec_helpers/api.py +++ b/exec_helpers/api.py @@ -19,14 +19,10 @@ .. versionchanged:: 1.3.5 make API public to use as interface """ -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import re import threading -import typing # noqa # pylint: disable=unused-import +import typing from exec_helpers import constants from exec_helpers import exceptions @@ -34,7 +30,7 @@ from exec_helpers import proc_enums -class ExecHelper(object): +class ExecHelper: """ExecHelper global API.""" __slots__ = ( @@ -45,9 +41,9 @@ class ExecHelper(object): def __init__( self, - logger, # type: logging.Logger - log_mask_re=None, # type: typing.Optional[str] - ): # type: (...) -> None + logger: logging.Logger, + log_mask_re: typing.Optional[str] = None, + ) -> None: """ExecHelper global API. :param log_mask_re: regex lookup rule to mask command for logger. @@ -62,19 +58,19 @@ def __init__( self.log_mask_re = log_mask_re @property - def logger(self): # type: () -> logging.Logger + def logger(self) -> logging.Logger: """Instance logger access.""" return self.__logger @property - def lock(self): # type: () -> threading.RLock + def lock(self) -> threading.RLock: """Lock. :rtype: threading.RLock """ return self.__lock - def __enter__(self): + def __enter__(self) -> 'ExecHelper': """Get context manager. .. versionchanged:: 1.1.0 lock on enter @@ -82,15 +78,15 @@ def __enter__(self): self.lock.acquire() return self - def __exit__(self, exc_type, exc_val, exc_tb): # pragma: no cover + def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: # pragma: no cover """Context manager usage.""" self.lock.release() def _mask_command( self, - cmd, # type: str - log_mask_re=None, # type: typing.Optional[str] - ): # type: (...) -> str + cmd: str, + log_mask_re: typing.Optional[str] = None, + ) -> str: """Log command with masking and return parsed cmd. :type cmd: str @@ -100,7 +96,7 @@ def _mask_command( .. versionadded:: 1.2.0 """ - def mask(text, rules): # type: (str, str) -> str + def mask(text: str, rules: str) -> str: """Mask part of text using rules.""" indexes = [0] # Start of the line @@ -132,14 +128,14 @@ def mask(text, rules): # type: (str, str) -> str def execute_async( self, - command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] - open_stdout=True, # type: bool - open_stderr=True, # type: bool - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any,] + command: str, + stdin: typing.Union[bytes, str, bytearray, None] = None, + open_stdout: bool = True, + open_stderr: bool = True, + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any]: """Execute command in async mode and return remote interface with IO objects. :param command: Command for execution @@ -155,12 +151,7 @@ def execute_async( :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>' :type log_mask_re: typing.Optional[str] - :rtype: typing.Tuple[ - typing.Any, - typing.Any, - typing.Any, - typing.Any, - ] + :rtype: typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any] .. versionchanged:: 1.2.0 open_stdout and open_stderr flags .. versionchanged:: 1.2.0 stdin data @@ -169,15 +160,15 @@ def execute_async( def _exec_command( self, - command, # type: str - interface, # type: typing.Any - stdout, # type: typing.Any - stderr, # type: typing.Any - timeout, # type: typing.Union[int, None] - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + interface: typing.Any, + stdout: typing.Any, + stderr: typing.Any, + timeout: typing.Union[int, float, None], + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Get exit status from channel with timeout. :param command: Command for execution @@ -204,11 +195,11 @@ def _exec_command( def execute( self, - command, # type: str - verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + verbose: bool = False, + timeout: typing.Union[int, None] = constants.DEFAULT_TIMEOUT, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Execute command and wait for return code. Timeout limitation: read tick is 100 ms. @@ -230,13 +221,13 @@ def execute( _, stderr, stdout, - ) = self.execute_async( + ) = self.execute_async( # type: ignore command, verbose=verbose, **kwargs ) - result = self._exec_command( + result = self._exec_command( # type: ignore command=command, interface=iface, stdout=stdout, @@ -246,7 +237,7 @@ def execute( **kwargs ) message = "Command {result.cmd!r} exit code: {result.exit_code!s}".format(result=result) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=message ) @@ -254,14 +245,14 @@ def execute( def check_call( self, - command, # type: str - verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - error_info=None, # type: typing.Optional[str] - expected=None, # type: typing.Optional[typing.Iterable[typing.Union[int, proc_enums.ExitCodes]]] - raise_on_err=True, # type: bool - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + verbose: bool = False, + timeout: typing.Union[int, None] = constants.DEFAULT_TIMEOUT, + error_info: typing.Optional[str] = None, + expected: typing.Optional[typing.Iterable[typing.Union[int, proc_enums.ExitCodes]]] = None, + raise_on_err: bool = True, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Execute command and check for return code. Timeout limitation: read tick is 100 ms. @@ -304,13 +295,13 @@ def check_call( def check_stderr( self, - command, # type: str - verbose=False, # type: bool - timeout=constants.DEFAULT_TIMEOUT, # type: typing.Union[int, None] - error_info=None, # type: typing.Optional[str] - raise_on_err=True, # type: bool - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + verbose: bool = False, + timeout: typing.Union[int, None] = constants.DEFAULT_TIMEOUT, + error_info: typing.Optional[str] = None, + raise_on_err: bool = True, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Execute command expecting return code 0 and empty STDERR. Timeout limitation: read tick is 100 ms. @@ -343,7 +334,7 @@ def check_stderr( )) self.logger.error(message) if raise_on_err: - raise exceptions.CalledProcessError( + raise exceptions.CalledProcessError( # type: ignore result=ret, expected=kwargs.get('expected'), ) diff --git a/exec_helpers/api.pyi b/exec_helpers/api.pyi deleted file mode 100644 index ccbcec1..0000000 --- a/exec_helpers/api.pyi +++ /dev/null @@ -1,74 +0,0 @@ -import logging -import threading -import typing - -from exec_helpers import exec_result, proc_enums - -class ExecHelper: - log_mask_re: typing.Optional[str] = ... - - def __init__(self, logger: logging.Logger, log_mask_re: typing.Optional[str] = ...) -> None: ... - - @property - def logger(self) -> logging.Logger: ... - - @property - def lock(self) -> threading.RLock: ... - - def __enter__(self) -> ExecHelper: ... - - def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: ... - - def _mask_command(self, cmd: str, log_mask_re: typing.Optional[str] = ...) -> str: ... - - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[typing.Any, typing.Any, typing.Any, typing.Any]: ... - - def _exec_command( - self, - command: str, - interface: typing.Any, - stdout: typing.Any, - stderr: typing.Any, - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - def execute( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - **kwargs: typing.Type - ) -> exec_result.ExecResult: ... - - def check_call( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - error_info: typing.Optional[str] = ..., - expected: typing.Optional[typing.Iterable[typing.Union[int, proc_enums.ExitCodes]]] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Type - ) -> exec_result.ExecResult: ... - - def check_stderr( - self, - command: str, - verbose: bool = ..., - timeout: typing.Union[int, None] = ..., - error_info: typing.Optional[str] = ..., - raise_on_err: bool = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... diff --git a/exec_helpers/constants.py b/exec_helpers/constants.py index 6254ff7..d017283 100644 --- a/exec_helpers/constants.py +++ b/exec_helpers/constants.py @@ -14,11 +14,6 @@ """Global constants.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - - MINUTE = 60 HOUR = 60 * MINUTE diff --git a/exec_helpers/exceptions.py b/exec_helpers/exceptions.py index fc01aa7..7b00430 100644 --- a/exec_helpers/exceptions.py +++ b/exec_helpers/exceptions.py @@ -14,15 +14,14 @@ """Package specific exceptions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -import typing # noqa # pylint: disable=unused-import +import typing from exec_helpers import proc_enums from exec_helpers import _log_templates +if typing.TYPE_CHECKING: + from exec_helpers import exec_result # noqa: F401 # pylint: disable=cyclic-import + __all__ = ( 'ExecHelperError', 'ExecHelperTimeoutError', @@ -65,9 +64,9 @@ class ExecHelperTimeoutError(ExecCalledProcessError): def __init__( self, - result, # type: exec_result.ExecResult - timeout, # type: typing.Union[int, float] - ): # type: (...) -> None + result: 'exec_result.ExecResult', + timeout: typing.Union[int, float], + ) -> None: """Exception for error on process calls. :param result: execution result @@ -84,17 +83,17 @@ def __init__( super(ExecHelperTimeoutError, self).__init__(message) @property - def cmd(self): # type: () -> str + def cmd(self) -> str: """Failed command.""" return self.result.cmd @property - def stdout(self): # type: () -> typing.Text + def stdout(self) -> str: """Command stdout.""" return self.result.stdout_str @property - def stderr(self): # type: () -> typing.Text + def stderr(self) -> str: """Command stderr.""" return self.result.stderr_str @@ -109,9 +108,9 @@ class CalledProcessError(ExecCalledProcessError): def __init__( self, - result, # type: exec_result.ExecResult - expected=None, # type: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] - ): # type: (...) -> None + result: 'exec_result.ExecResult', + expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, + ) -> None: """Exception for error on process calls. :param result: execution result @@ -139,22 +138,22 @@ def __init__( @property def returncode( self - ): # type: () -> typing.Union[int, proc_enums.ExitCodes] + ) -> typing.Union[int, proc_enums.ExitCodes]: """Command return code.""" return self.result.exit_code @property - def cmd(self): # type: () -> str + def cmd(self) -> str: """Failed command.""" return self.result.cmd @property - def stdout(self): # type: () -> typing.Text + def stdout(self) -> str: """Command stdout.""" return self.result.stdout_str @property - def stderr(self): # type: () -> typing.Text + def stderr(self) -> str: """Command stderr.""" return self.result.stderr_str @@ -172,12 +171,12 @@ class ParallelCallExceptions(ExecCalledProcessError): def __init__( self, - command, # type: str - exceptions, # type: typing.Dict[typing.Tuple[str, int], Exception] - errors, # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] - results, # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] - expected=None, # type: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] - ): # type: (...) -> None + command: str, + exceptions: typing.Dict[typing.Tuple[str, int], Exception], + errors: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + results: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, + ) -> None: """Exception raised during parallel call as result of exceptions. :param command: command @@ -227,11 +226,11 @@ class ParallelCallProcessError(ExecCalledProcessError): def __init__( self, - command, # type: str - errors, # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] - results, # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] - expected=None, # type: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] - ): # type: (...) -> None + command: str, + errors: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + results: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, + ) -> None: """Exception during parallel execution. :param command: command diff --git a/exec_helpers/exceptions.pyi b/exec_helpers/exceptions.pyi deleted file mode 100644 index 3df60f5..0000000 --- a/exec_helpers/exceptions.pyi +++ /dev/null @@ -1,81 +0,0 @@ -import typing -from exec_helpers import proc_enums, exec_result - -class ExecHelperError(Exception): ... - -class DeserializeValueError(ExecHelperError, ValueError): ... - -class ExecCalledProcessError(ExecHelperError): ... - -class ExecHelperTimeoutError(ExecCalledProcessError): - - result: exec_result.ExecResult = ... - timeout: int = ... - - def __init__(self, result: exec_result.ExecResult, timeout: typing.Union[int, float]) -> None: ... - - @property - def cmd(self) -> str: ... - - @property - def stdout(self) -> typing.Text: ... - - @property - def stderr(self) -> typing.Text: ... - - -class CalledProcessError(ExecCalledProcessError): - - result: exec_result.ExecResult = ... - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - - def __init__( - self, - result: exec_result.ExecResult, - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... - - @property - def returncode(self) -> typing.Union[int, proc_enums.ExitCodes]: ... - - @property - def cmd(self) -> str: ... - - @property - def stdout(self) -> typing.Text: ... - - @property - def stderr(self) -> typing.Text: ... - - -class ParallelCallExceptions(ExecCalledProcessError): - - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - cmd: str = ... - exceptions: typing.Dict[typing.Tuple[str, int], Exception] = ... - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - - def __init__( - self, - command: str, - exceptions: typing.Dict[typing.Tuple[str, int], Exception], - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... - -class ParallelCallProcessError(ExecCalledProcessError): - - expected: typing.List[typing.Union[int, proc_enums.ExitCodes]] = ... - cmd: str = ... - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult] = ... - - def __init__( - self, - command: str, - errors: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - results: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult], - expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = ..., - ) -> None: ... diff --git a/exec_helpers/exec_result.py b/exec_helpers/exec_result.py index fb9130d..ccc0ef5 100644 --- a/exec_helpers/exec_result.py +++ b/exec_helpers/exec_result.py @@ -16,20 +16,15 @@ """Execution result.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import datetime import json import logging import threading -import typing # noqa # pylint: disable=unused-import +import typing -import six import yaml -from exec_helpers import exceptions +from exec_helpers import exceptions # pylint: disable=cyclic-import from exec_helpers import proc_enums __all__ = ('ExecResult', ) @@ -37,7 +32,7 @@ logger = logging.getLogger(__name__) -class ExecResult(object): +class ExecResult: """Execution result.""" __slots__ = [ @@ -50,11 +45,11 @@ class ExecResult(object): def __init__( self, cmd, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] - stdout=None, # type: typing.Optional[typing.Iterable[bytes]] - stderr=None, # type: typing.Optional[typing.Iterable[bytes]] - exit_code=proc_enums.ExitCodes.EX_INVALID # type: typing.Union[int, proc_enums.ExitCodes] - ): # type: (...) -> None + stdin: typing.Union[bytes, str, bytearray, None] = None, + stdout: typing.Optional[typing.Iterable[bytes]] = None, + stderr: typing.Optional[typing.Iterable[bytes]] = None, + exit_code: typing.Union[int, proc_enums.ExitCodes] = proc_enums.ExitCodes.EX_INVALID + ) -> None: """Command execution result. :param cmd: command @@ -71,15 +66,23 @@ def __init__( self.__lock = threading.RLock() self.__cmd = cmd - if isinstance(stdin, six.binary_type): + if isinstance(stdin, bytes): stdin = self._get_str_from_bin(bytearray(stdin)) elif isinstance(stdin, bytearray): stdin = self._get_str_from_bin(stdin) - self.__stdin = stdin # type: typing.Optional[typing.Text] - self.__stdout = tuple(stdout) if stdout is not None else () # type: typing.Tuple[bytes] - self.__stderr = tuple(stderr) if stderr is not None else () # type: typing.Tuple[bytes] + self.__stdin = stdin # type: typing.Optional[str] + + if stdout is not None: + self.__stdout = tuple(stdout) # type: typing.Tuple[bytes, ...] + else: + self.__stdout = () - self.__exit_code = None + if stderr is not None: + self.__stderr = tuple(stderr) # type: typing.Tuple[bytes, ...] + else: + self.__stderr = () + + self.__exit_code = proc_enums.ExitCodes.EX_INVALID # type: typing.Union[int, proc_enums.ExitCodes] self.__timestamp = None self.exit_code = exit_code @@ -90,7 +93,7 @@ def __init__( self.__stderr_brief = None @property - def lock(self): # type: () -> threading.RLock + def lock(self) -> threading.RLock: """Lock object for thread-safe operation. :rtype: threading.RLock @@ -98,7 +101,7 @@ def lock(self): # type: () -> threading.RLock return self.__lock @property - def timestamp(self): # type: () -> typing.Optional[datetime.datetime] + def timestamp(self) -> typing.Optional[datetime.datetime]: """Timestamp. :rtype: typing.Optional(datetime.datetime) @@ -106,9 +109,7 @@ def timestamp(self): # type: () -> typing.Optional[datetime.datetime] return self.__timestamp @staticmethod - def _get_bytearray_from_array( - src # type: typing.Iterable[bytes] - ): # type: (...) -> bytearray + def _get_bytearray_from_array(src: typing.Iterable[bytes]) -> bytearray: """Get bytearray from array of bytes blocks. :type src: typing.List[bytes] @@ -117,7 +118,7 @@ def _get_bytearray_from_array( return bytearray(b''.join(src)) @staticmethod - def _get_str_from_bin(src): # type: (bytearray) -> typing.Text + def _get_str_from_bin(src: bytearray) -> str: """Join data in list to the string, with python 2&3 compatibility. :type src: bytearray @@ -129,19 +130,22 @@ def _get_str_from_bin(src): # type: (bytearray) -> typing.Text ) @classmethod - def _get_brief(cls, data): # type: (typing.Tuple[bytes]) -> typing.Text + def _get_brief(cls, data: typing.Tuple[bytes, ...]) -> str: """Get brief output: 7 lines maximum (3 first + ... + 3 last). - :type data: typing.Tuple[bytes] + :type data: typing.Tuple[bytes, ...] :rtype: str """ - src = data if len(data) <= 7 else data[:3] + (b'...\n',) + data[-3:] # type: typing.Tuple[bytes] + if len(data) <= 7: + src = data # type: typing.Tuple[bytes, ...] + else: + src = data[:3] + (b'...\n',) + data[-3:] return cls._get_str_from_bin( cls._get_bytearray_from_array(src) ) @property - def cmd(self): # type: () -> typing.Text + def cmd(self): # type: () -> str """Executed command. :rtype: str @@ -149,41 +153,41 @@ def cmd(self): # type: () -> typing.Text return self.__cmd @property - def stdin(self): # type: () -> typing.Optional[typing.Text] + def stdin(self) -> typing.Optional[str]: """Stdin input as string. - :rtype: typing.Optional[typing.Text] + :rtype: typing.Optional[str] """ return self.__stdin @property - def stdout(self): # type: () -> typing.Tuple[bytes] + def stdout(self) -> typing.Tuple[bytes, ...]: """Stdout output as list of binaries. - :rtype: typing.Tuple[bytes] + :rtype: typing.Tuple[bytes, ...] """ return self.__stdout @property - def stderr(self): # type: () -> typing.Tuple[bytes] + def stderr(self) -> typing.Tuple[bytes, ...]: """Stderr output as list of binaries. - :rtype: typing.Tuple[bytes] + :rtype: typing.Tuple[bytes, ...] """ return self.__stderr @staticmethod def __poll_stream( - src, # type: typing.Iterable[bytes] - log=None, # type: typing.Optional[logging.Logger] - verbose=False # type: bool - ): # type: (...) -> typing.List[bytes] + src: typing.Iterable[bytes], + log: typing.Optional[logging.Logger] = None, + verbose: bool = False + ) -> typing.List[bytes]: dst = [] try: for line in src: dst.append(line) if log: - log.log( + log.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=line.decode('utf-8', errors='backslashreplace').rstrip() ) @@ -193,10 +197,10 @@ def __poll_stream( def read_stdout( self, - src=None, # type: typing.Optional[typing.Iterable] - log=None, # type: typing.Optional[logging.Logger] - verbose=False # type: bool - ): # type: (...) -> None + src: typing.Optional[typing.Iterable] = None, + log: typing.Optional[logging.Logger] = None, + verbose: bool = False + ) -> None: """Read stdout file-like object to stdout. :param src: source @@ -208,20 +212,21 @@ def read_stdout( .. versionchanged:: 1.2.0 - src can be None """ - if self.timestamp: - raise RuntimeError('Final exit code received.') if not src: return + if self.timestamp: + raise RuntimeError('Final exit code received.') + with self.lock: self.__stdout_str = self.__stdout_brief = None self.__stdout += tuple(self.__poll_stream(src, log, verbose)) def read_stderr( self, - src=None, # type: typing.Optional[typing.Iterable] - log=None, # type: typing.Optional[logging.Logger] - verbose=False # type: bool - ): # type: (...) -> None + src: typing.Optional[typing.Iterable] = None, + log: typing.Optional[logging.Logger] = None, + verbose: bool = False + ) -> None: """Read stderr file-like object to stdout. :param src: source @@ -233,16 +238,17 @@ def read_stderr( .. versionchanged:: 1.2.0 - src can be None """ - if self.timestamp: - raise RuntimeError('Final exit code received.') if not src: return + if self.timestamp: + raise RuntimeError('Final exit code received.') + with self.lock: self.__stderr_str = self.__stderr_brief = None self.__stderr += tuple(self.__poll_stream(src, log, verbose)) @property - def stdout_bin(self): # type: () -> bytearray + def stdout_bin(self) -> bytearray: """Stdout in binary format. Sometimes logging is used to log binary objects too (example: Session), @@ -253,7 +259,7 @@ def stdout_bin(self): # type: () -> bytearray return self._get_bytearray_from_array(self.stdout) @property - def stderr_bin(self): # type: () -> bytearray + def stderr_bin(self) -> bytearray: """Stderr in binary format. :rtype: bytearray @@ -262,51 +268,51 @@ def stderr_bin(self): # type: () -> bytearray return self._get_bytearray_from_array(self.stderr) @property - def stdout_str(self): # type: () -> typing.Text + def stdout_str(self) -> str: """Stdout output as string. :rtype: str """ with self.lock: if self.__stdout_str is None: - self.__stdout_str = self._get_str_from_bin(self.stdout_bin) - return self.__stdout_str + self.__stdout_str = self._get_str_from_bin(self.stdout_bin) # type: ignore + return self.__stdout_str # type: ignore @property - def stderr_str(self): # type: () -> typing.Text + def stderr_str(self) -> str: """Stderr output as string. :rtype: str """ with self.lock: if self.__stderr_str is None: - self.__stderr_str = self._get_str_from_bin(self.stderr_bin) - return self.__stderr_str + self.__stderr_str = self._get_str_from_bin(self.stderr_bin) # type: ignore + return self.__stderr_str # type: ignore @property - def stdout_brief(self): # type: () -> typing.Text + def stdout_brief(self) -> str: """Brief stdout output (mostly for exceptions). :rtype: str """ with self.lock: if self.__stdout_brief is None: - self.__stdout_brief = self._get_brief(self.stdout) - return self.__stdout_brief + self.__stdout_brief = self._get_brief(self.stdout) # type: ignore + return self.__stdout_brief # type: ignore @property - def stderr_brief(self): # type: () -> typing.Text + def stderr_brief(self) -> str: """Brief stderr output (mostly for exceptions). :rtype: str """ with self.lock: if self.__stderr_brief is None: - self.__stderr_brief = self._get_brief(self.stderr) - return self.__stderr_brief + self.__stderr_brief = self._get_brief(self.stderr) # type: ignore + return self.__stderr_brief # type: ignore @property - def exit_code(self): # type: () -> typing.Union[int, proc_enums.ExitCodes] + def exit_code(self) -> typing.Union[int, proc_enums.ExitCodes]: """Return(exit) code of command. :rtype: typing.Union[int, proc_enums.ExitCodes] @@ -314,7 +320,7 @@ def exit_code(self): # type: () -> typing.Union[int, proc_enums.ExitCodes] return self.__exit_code @exit_code.setter - def exit_code(self, new_val): # type: (typing.Union[int, proc_enums.ExitCodes]) -> None + def exit_code(self, new_val: typing.Union[int, proc_enums.ExitCodes]) -> None: """Return(exit) code of command. :type new_val: int @@ -322,14 +328,14 @@ def exit_code(self, new_val): # type: (typing.Union[int, proc_enums.ExitCodes]) """ if self.timestamp: raise RuntimeError('Exit code is already received.') - if not isinstance(new_val, six.integer_types): + if not isinstance(new_val, int): raise TypeError('Exit code is strictly int') with self.lock: self.__exit_code = proc_enums.exit_code_to_enum(new_val) if self.__exit_code != proc_enums.ExitCodes.EX_INVALID: - self.__timestamp = datetime.datetime.utcnow() + self.__timestamp = datetime.datetime.utcnow() # type: ignore - def __deserialize(self, fmt): # type: (str) -> typing.Any + def __deserialize(self, fmt: str) -> typing.Any: """Deserialize stdout as data format. :type fmt: str @@ -338,7 +344,7 @@ def __deserialize(self, fmt): # type: (str) -> typing.Any :raises DeserializeValueError: Not valid source format """ try: - if fmt == 'json': + if fmt == 'json': # pylint: disable=no-else-return return json.loads(self.stdout_str, encoding='utf-8') elif fmt == 'yaml': return yaml.safe_load(self.stdout_str) @@ -347,7 +353,7 @@ def __deserialize(self, fmt): # type: (str) -> typing.Any " stdout is not valid {fmt}:\n" '{{stdout!r}}\n'.format( fmt=fmt)) - logger.exception(self.cmd + tmpl.format(stdout=self.stdout_str)) + logger.exception(self.cmd + tmpl.format(stdout=self.stdout_str)) # pylint: disable=logging-not-lazy raise exceptions.DeserializeValueError( self.cmd + tmpl.format(stdout=self.stdout_brief) ) @@ -356,7 +362,7 @@ def __deserialize(self, fmt): # type: (str) -> typing.Any raise NotImplementedError(msg) @property - def stdout_json(self): # type: () -> typing.Any + def stdout_json(self) -> typing.Any: """JSON from stdout. :rtype: object @@ -365,7 +371,7 @@ def stdout_json(self): # type: () -> typing.Any return self.__deserialize(fmt='json') @property - def stdout_yaml(self): # type: () -> typing.Any + def stdout_yaml(self) -> typing.Any: """YAML from stdout. :rtype: Union(list, dict, None) @@ -373,7 +379,7 @@ def stdout_yaml(self): # type: () -> typing.Any with self.lock: return self.__deserialize(fmt='yaml') - def __dir__(self): + def __dir__(self) -> typing.List[str]: """Override dir for IDE and as source for getitem checks.""" return [ 'cmd', 'stdout', 'stderr', 'exit_code', @@ -383,7 +389,7 @@ def __dir__(self): 'lock' ] - def __getitem__(self, item): # type: (str) -> typing.Any + def __getitem__(self, item: str) -> typing.Any: """Dict like get data.""" if item in dir(self): return getattr(self, item) @@ -393,7 +399,7 @@ def __getitem__(self, item): # type: (str) -> typing.Any ) ) - def __repr__(self): # type: () -> str + def __repr__(self) -> str: """Representation for debugging.""" return ( '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, ' @@ -405,7 +411,7 @@ def __repr__(self): # type: () -> str exit_code=self.exit_code )) - def __str__(self): # type: () -> str + def __str__(self) -> str: """Representation for logging.""" return ( "{cls}(\n\tcmd={cmd!r}," @@ -420,15 +426,15 @@ def __str__(self): # type: () -> str ) ) - def __eq__(self, other): # type: (typing.Any) -> bool + def __eq__(self, other: typing.Any) -> bool: """Comparision.""" return hash(self) == hash(other) - def __ne__(self, other): # type: (typing.Any) -> bool + def __ne__(self, other: typing.Any) -> bool: """Comparision.""" return not self.__eq__(other) - def __hash__(self): + def __hash__(self) -> int: """Hash for usage as dict key and in sets.""" return hash( ( diff --git a/exec_helpers/exec_result.pyi b/exec_helpers/exec_result.pyi deleted file mode 100644 index 88b515c..0000000 --- a/exec_helpers/exec_result.pyi +++ /dev/null @@ -1,96 +0,0 @@ -import datetime -import logging -import threading -import typing -from exec_helpers import proc_enums - -class ExecResult: - def __init__( - self, - cmd: str, - stdin: typing.Union[typing.AnyStr, bytearray, None] = ..., - stdout: typing.Optional[typing.Iterable[bytes]] = ..., - stderr: typing.Optional[typing.Iterable[bytes]] = ..., - exit_code: typing.Union[int, proc_enums.ExitCodes] = ..., - ) -> None: ... - - @property - def lock(self) -> threading.RLock: ... - - @property - def timestamp(self) -> typing.Optional[datetime.datetime]: ... - - @staticmethod - def _get_bytearray_from_array(src: typing.Iterable[bytes]) -> bytearray: ... - - @staticmethod - def _get_str_from_bin(src: bytearray) -> typing.Text: ... - - @classmethod - def _get_brief(cls, data: typing.Tuple[bytes]) -> typing.Text: ... - - @property - def cmd(self) -> str: ... - - @property - def stdin(self) -> typing.Optional[typing.Text]: ... - - @property - def stdout(self) -> typing.Tuple[bytes]: ... - - @property - def stderr(self) -> typing.Tuple[bytes]: ... - - def read_stdout( - self, - src: typing.Optional[typing.Iterable] = ..., - log: typing.Optional[logging.Logger] = ..., - verbose: bool = ..., - ) -> None: ... - - def read_stderr( - self, - src: typing.Optional[typing.Iterable] = ..., - log: typing.Optional[logging.Logger] = ..., - verbose: bool = ..., - ) -> None: ... - - @property - def stdout_bin(self) -> bytearray: ... - - @property - def stderr_bin(self) -> bytearray: ... - - @property - def stdout_str(self) -> typing.Text: ... - - @property - def stderr_str(self) -> typing.Text: ... - - @property - def stdout_brief(self) -> typing.Text: ... - - @property - def stderr_brief(self) -> typing.Text: ... - - @property - def exit_code(self) -> typing.Union[int, proc_enums.ExitCodes]: ... - - @exit_code.setter - def exit_code(self, new_val: typing.Union[int, proc_enums.ExitCodes]) -> None: ... - - @property - def stdout_json(self) -> typing.Any: ... - - @property - def stdout_yaml(self) -> typing.Any: ... - - def __getitem__(self, item: str) -> typing.Any: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... - - def __eq__(self, other: typing.Any) -> bool: ... - - def __ne__(self, other: typing.Any) -> bool: ... diff --git a/exec_helpers/proc_enums.py b/exec_helpers/proc_enums.py index 00fa4f4..41b785d 100644 --- a/exec_helpers/proc_enums.py +++ b/exec_helpers/proc_enums.py @@ -19,15 +19,9 @@ Linux signals, Linux & bash return codes. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import enum import typing # noqa # pylint: disable=unused-import -import six - __all__ = ( 'SigNum', 'ExitCodes', @@ -72,7 +66,7 @@ class SigNum(enum.IntEnum): SIGPWR = 30 # Power failure restart (System V). SIGSYS = 31 # Bad system call. - def __str__(self): # type: () -> str + def __str__(self) -> str: """Representation for logs.""" return "{name}<{value:d}(0x{value:02X})>".format( # pragma: no cover name=self.name, @@ -80,15 +74,8 @@ def __str__(self): # type: () -> str ) -if six.PY3: # pragma: no cover - digit_type = int -else: # pragma: no cover - # noinspection PyUnresolvedReferences - digit_type = long # noqa # pylint: disable=undefined-variable, long-builtin - - @enum.unique -class ExitCodes(digit_type, enum.Enum): +class ExitCodes(int, enum.Enum): """Linux & bash exit codes.""" EX_OK = 0 # successful termination @@ -150,7 +137,7 @@ class ExitCodes(digit_type, enum.Enum): EX_SIGPWR = 128 + SigNum.SIGPWR EX_SIGSYS = 128 + SigNum.SIGSYS - def __str__(self): # type: () -> str + def __str__(self) -> str: """Representation for logs.""" return "{name}<{value:d}(0x{value:02X})>".format( name=self.name, @@ -158,7 +145,7 @@ def __str__(self): # type: () -> str ) -def exit_code_to_enum(code): # type: (typing.Union[int, ExitCodes]) -> typing.Union[int, ExitCodes] +def exit_code_to_enum(code: typing.Union[int, ExitCodes]) -> typing.Union[int, ExitCodes]: """Convert exit code to enum if possible.""" if isinstance(code, int) and code in ExitCodes.__members__.values(): return ExitCodes(code) @@ -166,8 +153,8 @@ def exit_code_to_enum(code): # type: (typing.Union[int, ExitCodes]) -> typing.U def exit_codes_to_enums( - codes=None # type: typing.Optional[typing.Iterable[typing.Union[int, ExitCodes]]] -): # type: (...) -> typing.List[typing.Union[int, ExitCodes]] + codes: typing.Optional[typing.Iterable[typing.Union[int, ExitCodes]]] = None +) -> typing.List[typing.Union[int, ExitCodes]]: """Convert integer exit codes to enums.""" if codes is None: return [ExitCodes.EX_OK] diff --git a/exec_helpers/ssh_auth.py b/exec_helpers/ssh_auth.py index e270ee2..99f2e32 100644 --- a/exec_helpers/ssh_auth.py +++ b/exec_helpers/ssh_auth.py @@ -16,16 +16,11 @@ """SSH client credentials class.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import copy -import io # noqa # pylint: disable=unused-import import logging -import typing # noqa # pylint: disable=unused-import +import typing -import paramiko +import paramiko # type: ignore __all__ = ('SSHAuth', ) @@ -34,7 +29,7 @@ logging.getLogger('iso8601').setLevel(logging.WARNING) -class SSHAuth(object): +class SSHAuth: """SSH Authorization object.""" __slots__ = ( @@ -44,13 +39,13 @@ class SSHAuth(object): def __init__( self, - username=None, # type: typing.Optional[str] - password=None, # type: typing.Optional[str] - key=None, # type: typing.Optional[paramiko.RSAKey] - keys=None, # type: typing.Optional[typing.Iterable[paramiko.RSAKey]] - key_filename=None, # type: typing.Union[typing.List[str], str, None] - passphrase=None, # type: typing.Optional[str] - ): # type: (...) -> None + username: typing.Optional[str] = None, + password: typing.Optional[str] = None, + key: typing.Optional[paramiko.RSAKey] = None, + keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = None, + key_filename: typing.Union[typing.List[str], str, None] = None, + passphrase: typing.Optional[str] = None, + ) -> None: """SSH credentials object. Used to authorize SSHClient. @@ -88,7 +83,7 @@ def __init__( self.__passphrase = passphrase @property - def username(self): # type: () -> typing.Optional[str] + def username(self) -> typing.Optional[str]: """Username for auth. :rtype: str @@ -96,7 +91,7 @@ def username(self): # type: () -> typing.Optional[str] return self.__username @staticmethod - def __get_public_key(key): # type: (typing.Union[paramiko.RSAKey, None]) -> typing.Optional[str] + def __get_public_key(key: typing.Union[paramiko.RSAKey, None])-> typing.Optional[str]: """Internal method for get public key from private. :type key: paramiko.RSAKey @@ -106,7 +101,7 @@ def __get_public_key(key): # type: (typing.Union[paramiko.RSAKey, None]) -> typ return '{0} {1}'.format(key.get_name(), key.get_base64()) @property - def public_key(self): # type: () -> typing.Optional[str] + def public_key(self) -> typing.Optional[str]: """public key for stored private key if presents else None. :rtype: str @@ -114,16 +109,14 @@ def public_key(self): # type: () -> typing.Optional[str] return self.__get_public_key(self.__key) @property - def key_filename( - self - ): # type: () -> typing.Union[typing.List[str], str, None] + def key_filename(self) -> typing.Union[typing.List[str], str, None]: """Key filename(s). .. versionadded:: 1.0.0 """ return copy.deepcopy(self.__key_filename) - def enter_password(self, tgt): # type: (io.StringIO) -> None + def enter_password(self, tgt: typing.IO) -> None: """Enter password to STDIN. Note: required for 'sudo' call @@ -136,11 +129,11 @@ def enter_password(self, tgt): # type: (io.StringIO) -> None def connect( self, - client, # type: typing.Union[paramiko.SSHClient, paramiko.Transport] - hostname=None, # type: typing.Optional[str] - port=22, # type: int - log=True, # type: bool - ): # type: (...) -> None + client: typing.Union[paramiko.SSHClient, paramiko.Transport], + hostname: typing.Optional[str] = None, + port: int = 22, + log: bool = True, + ) -> None: """Connect SSH client object using credentials. :param client: SSH Client (low level) @@ -197,7 +190,7 @@ def connect( logger.exception(msg) raise paramiko.AuthenticationException(msg) - def __hash__(self): + def __hash__(self) -> int: """Hash for usage as dict keys and comparison.""" return hash(( self.__class__, @@ -212,39 +205,39 @@ def __hash__(self): self.__passphrase )) - def __eq__(self, other): # type: (typing.Any) -> bool + def __eq__(self, other: typing.Any) -> bool: """Comparison helper.""" return hash(self) == hash(other) - def __ne__(self, other): # type: (typing.Any) -> bool + def __ne__(self, other: typing.Any) -> bool: """Comparison helper.""" return not self.__eq__(other) - def __deepcopy__(self, memo): # type: (typing.Any) -> SSHAuth + def __deepcopy__(self, memo: typing.Any) -> 'SSHAuth': """Helper for copy.deepcopy.""" - return self.__class__( + return self.__class__( # type: ignore username=self.username, password=self.__password, key=self.__key, keys=copy.deepcopy(self.__keys) ) - def __copy__(self): # type: () -> SSHAuth + def __copy__(self) -> 'SSHAuth': """Copy self.""" - return self.__class__( + return self.__class__( # type: ignore username=self.username, password=self.__password, key=self.__key, keys=self.__keys ) - def __repr__(self): # type: (...) -> str + def __repr__(self) -> str: """Representation for debug purposes.""" _key = ( None if self.__key is None else ''.format(self.public_key) ) - _keys = [] + _keys = [] # type: typing.List[typing.Union[str, None]] for k in self.__keys: if k == self.__key: continue @@ -268,7 +261,7 @@ def __repr__(self): # type: (...) -> str keys=_keys) ) - def __str__(self): # type: (...) -> str + def __str__(self) -> str: """Representation for debug purposes.""" return ( '{cls} for {username}'.format( diff --git a/exec_helpers/ssh_auth.pyi b/exec_helpers/ssh_auth.pyi deleted file mode 100644 index f0d0ad4..0000000 --- a/exec_helpers/ssh_auth.pyi +++ /dev/null @@ -1,45 +0,0 @@ -import io -import paramiko # type: ignore -import typing - -class SSHAuth: - def __init__( - self, - username: typing.Optional[str] = ..., - password: typing.Optional[str] = ..., - key: typing.Optional[paramiko.RSAKey] = ..., - keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = ..., - key_filename: typing.Union[typing.List[str], str, None] = ..., - passphrase: typing.Optional[str] = ..., - ) -> None: ... - - @property - def username(self) -> typing.Optional[str]: ... - - @property - def public_key(self) -> typing.Optional[str]: ... - - @property - def key_filename(self) -> typing.Union[typing.List[str], str, None]: ... - - def enter_password(self, tgt: io.StringIO) -> None: ... - - def connect( - self, - client: typing.Union[paramiko.SSHClient, paramiko.Transport], - hostname: typing.Optional[str] = ..., - port: int = ..., - log: bool = ..., - ) -> None: ... - - def __eq__(self, other: typing.Any) -> bool: ... - - def __ne__(self, other: typing.Any) -> bool: ... - - def __deepcopy__(self, memo: typing.Any) -> SSHAuth: ... - - def __copy__(self) -> SSHAuth: ... - - def __repr__(self) -> str: ... - - def __str__(self) -> str: ... diff --git a/exec_helpers/ssh_client.py b/exec_helpers/ssh_client.py index 3232596..a76f9eb 100644 --- a/exec_helpers/ssh_client.py +++ b/exec_helpers/ssh_client.py @@ -16,10 +16,6 @@ """SSH client helper based on Paramiko. Extended API helpers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import os import posixpath @@ -39,11 +35,11 @@ class SSHClient(SSHClientBase): __slots__ = () @staticmethod - def _path_esc(path): # type: (str) -> str + def _path_esc(path: str) -> str: """Escape space character in the path.""" return path.replace(' ', '\ ') - def mkdir(self, path): # type: (str) -> None + def mkdir(self, path: str) -> None: """run 'mkdir -p path' on remote. :type path: str @@ -53,7 +49,7 @@ def mkdir(self, path): # type: (str) -> None # noinspection PyTypeChecker self.execute("mkdir -p {}\n".format(self._path_esc(path))) - def rm_rf(self, path): # type: (str) -> None + def rm_rf(self, path: str) -> None: """run 'rm -rf path' on remote. :type path: str @@ -61,7 +57,7 @@ def rm_rf(self, path): # type: (str) -> None # noinspection PyTypeChecker self.execute("rm -rf {}".format(self._path_esc(path))) - def upload(self, source, target): # type: (str, str) -> None + def upload(self, source: str, target: str) -> None: """Upload file(s) from source to target using SFTP session. :type source: str @@ -92,11 +88,7 @@ def upload(self, source, target): # type: (str, str) -> None self._sftp.unlink(remote_path) self._sftp.put(local_path, remote_path) - def download( - self, - destination, # type: str - target # type: str - ): # type: (...) -> bool + def download(self, destination: str, target: str) -> bool: """Download file(s) to target from destination. :type destination: str diff --git a/exec_helpers/subprocess_runner.py b/exec_helpers/subprocess_runner.py index 4f9c5d0..af4653a 100644 --- a/exec_helpers/subprocess_runner.py +++ b/exec_helpers/subprocess_runner.py @@ -16,21 +16,14 @@ """Python subprocess.Popen wrapper.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import collections -# noinspection PyCompatibility import concurrent.futures import errno import logging -import os import subprocess # nosec # Expected usage import threading -import typing # noqa # pylint: disable=unused-import +import typing -import six import threaded from exec_helpers import api @@ -39,8 +32,6 @@ from exec_helpers import _log_templates logger = logging.getLogger(__name__) # type: logging.Logger -# noinspection PyUnresolvedReferences -devnull = open(os.devnull) # subprocess.DEVNULL is py3.3+ class SingletonMeta(type): @@ -52,7 +43,7 @@ class SingletonMeta(type): _instances = {} # type: typing.Dict[typing.Type, typing.Any] _lock = threading.RLock() # type: threading.RLock - def __call__(cls, *args, **kwargs): + def __call__(cls: 'SingletonMeta', *args: typing.Tuple, **kwargs: typing.Dict) -> typing.Any: """Singleton.""" with cls._lock: if cls not in cls._instances: @@ -61,12 +52,12 @@ def __call__(cls, *args, **kwargs): return cls._instances[cls] @classmethod - def __prepare__( - mcs, - name, # type: str - bases, # type: typing.Iterable[typing.Type] - **kwargs - ): # type: (...) -> collections.OrderedDict # pylint: disable=unused-argument + def __prepare__( # pylint: disable=unused-argument + mcs: typing.Type['SingletonMeta'], + name: str, + bases: typing.Iterable[typing.Type], + **kwargs: typing.Dict + ) -> collections.OrderedDict: """Metaclass magic for object storage. .. versionadded:: 1.2.0 @@ -74,13 +65,10 @@ def __prepare__( return collections.OrderedDict() # pragma: no cover -class Subprocess(six.with_metaclass(SingletonMeta, api.ExecHelper)): +class Subprocess(api.ExecHelper, metaclass=SingletonMeta): """Subprocess helper with timeouts and lock-free FIFO.""" - def __init__( - self, - log_mask_re=None, # type: typing.Optional[str] - ): # type: (...) -> None + def __init__(self, log_mask_re: typing.Optional[str] = None) -> None: """Subprocess helper with timeouts and lock-free FIFO. For excluding race-conditions we allow to run 1 command simultaneously @@ -96,15 +84,15 @@ def __init__( def _exec_command( self, - command, # type: str - interface, # type: subprocess.Popen - stdout, # type: typing.Optional[typing.IO] - stderr, # type: typing.Optional[typing.IO] - timeout, # type: typing.Union[int, None] - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> exec_result.ExecResult + command: str, + interface: subprocess.Popen, + stdout: typing.Optional[typing.IO], + stderr: typing.Optional[typing.IO], + timeout: typing.Union[int, float, None], + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> exec_result.ExecResult: """Get exit status from channel with timeout. :param command: Command for execution @@ -127,8 +115,8 @@ def _exec_command( .. versionadded:: 1.2.0 """ - @threaded.threadpooled - def poll_stdout(): + @threaded.threadpooled # type: ignore + def poll_stdout() -> None: """Sync stdout poll.""" result.read_stdout( src=stdout, @@ -136,8 +124,8 @@ def poll_stdout(): verbose=verbose ) - @threaded.threadpooled - def poll_stderr(): + @threaded.threadpooled # type: ignore + def poll_stderr() -> None: """Sync stderr poll.""" result.read_stderr( src=stderr, @@ -181,16 +169,47 @@ def poll_stderr(): logger.debug(wait_err_msg) raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) + # pylint: disable=function-redefined, unused-argument + # noinspection PyMethodOverriding + @typing.overload # type: ignore + def execute_async( # pylint: disable=signature-differs + self, + command: str, + stdin: typing.Union[bytes, str, bytearray], + open_stdout: bool = True, + open_stderr: bool = True, + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> typing.Tuple[subprocess.Popen, None, None, None]: + """Overload: with stdin.""" + pass + + @typing.overload # noqa: F811 def execute_async( self, - command, # type: str - stdin=None, # type: typing.Union[typing.AnyStr, bytearray, None] - open_stdout=True, # type: bool - open_stderr=True, # type: bool - verbose=False, # type: bool - log_mask_re=None, # type: typing.Optional[str] - **kwargs - ): # type: (...) -> typing.Tuple[subprocess.Popen, None, typing.Optional[typing.IO], typing.Optional[typing.IO], ] + command: str, + stdin: None = None, + open_stdout: bool = True, + open_stderr: bool = True, + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> typing.Tuple[subprocess.Popen, None, typing.IO, typing.IO]: + """Overload: no stdin.""" + pass + + # pylint: enable=unused-argument + def execute_async( # type: ignore # noqa: F811 + self, + command: str, + stdin: typing.Union[typing.AnyStr, bytearray, None] = None, + open_stdout: bool = True, + open_stderr: bool = True, + verbose: bool = False, + log_mask_re: typing.Optional[str] = None, + **kwargs: typing.Dict + ) -> typing.Tuple[subprocess.Popen, None, typing.Optional[typing.IO], typing.Optional[typing.IO]]: """Execute command in async mode and return Popen with IO objects. :param command: Command for execution @@ -217,15 +236,15 @@ def execute_async( """ cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) - self.logger.log( + self.logger.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) - process = subprocess.Popen( + process = subprocess.Popen( # type: ignore args=[command], - stdout=subprocess.PIPE if open_stdout else devnull, - stderr=subprocess.PIPE if open_stderr else devnull, + stdout=subprocess.PIPE if open_stdout else subprocess.DEVNULL, + stderr=subprocess.PIPE if open_stderr else subprocess.DEVNULL, stdin=subprocess.PIPE, shell=True, cwd=kwargs.get('cwd', None), @@ -234,10 +253,10 @@ def execute_async( ) if stdin is not None: - if isinstance(stdin, six.text_type): - stdin = stdin.encode(encoding='utf-8') + if isinstance(stdin, str): + stdin = stdin.encode(encoding='utf-8') # type: ignore elif isinstance(stdin, bytearray): - stdin = bytes(stdin) + stdin = bytes(stdin) # type: ignore try: process.stdin.write(stdin) except OSError as exc: @@ -261,3 +280,5 @@ def execute_async( raise return process, None, process.stderr, process.stdout + + # pylint: enable=function-redefined diff --git a/exec_helpers/subprocess_runner.pyi b/exec_helpers/subprocess_runner.pyi deleted file mode 100644 index 4d6c2b7..0000000 --- a/exec_helpers/subprocess_runner.pyi +++ /dev/null @@ -1,63 +0,0 @@ -import collections -import logging -import subprocess -import threading -import typing - -from exec_helpers import exec_result, api - -logger: logging.Logger -devnull: typing.IO - -class SingletonMeta(type): - _instances: typing.Dict[typing.Type, typing.Any] = ... - _lock: threading.RLock = ... - - def __call__(cls: SingletonMeta, *args: typing.Tuple, **kwargs: typing.Dict) -> typing.Any: ... - - @classmethod - def __prepare__( - mcs: typing.Type[SingletonMeta], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Dict - ) -> collections.OrderedDict: ... - -class Subprocess(api.ExecHelper, metaclass=SingletonMeta): - def __init__(self, log_mask_re: typing.Optional[str] = ...) -> None: ... - - def _exec_command( - self, - command: str, - interface: subprocess.Popen, - stdout: typing.Optional[typing.IO], - stderr: typing.Optional[typing.IO], - timeout: typing.Union[int, None], - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> exec_result.ExecResult: ... - - @typing.overload # type: ignore - def execute_async( - self, - command: str, - stdin: typing.Union[typing.AnyStr, bytearray] = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[subprocess.Popen, None, None, None]: ... - - @typing.overload - def execute_async( - self, - command: str, - stdin: None = ..., - open_stdout: bool = ..., - open_stderr: bool = ..., - verbose: bool = ..., - log_mask_re: typing.Optional[str] = ..., - **kwargs: typing.Dict - ) -> typing.Tuple[subprocess.Popen, None, typing.IO, typing.IO]: ... diff --git a/setup.py b/setup.py index d959910..6cab15d 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,6 @@ import setuptools -PY3 = sys.version_info[:2] > (2, 7) - with open( os.path.join( os.path.dirname(__file__), @@ -57,10 +55,7 @@ def _extension(modpath): """Make setuptools.Extension.""" source_path = modpath.replace('.', '/') + '.py' - return setuptools.Extension( - modpath if PY3 else modpath.encode('utf-8'), - [source_path if PY3 else source_path.encode('utf-8')] - ) + return setuptools.Extension(modpath, [source_path]) requires_optimization = [ @@ -90,7 +85,7 @@ def _extension(modpath): overflowcheck=True, language_level=3, ) -) if cythonize is not None and PY3 else [] +) if cythonize is not None else [] class BuildFailed(Exception): @@ -189,10 +184,9 @@ def get_simple_vars_from_src(src): """ ast_data = ( ast.Str, ast.Num, - ast.List, ast.Set, ast.Dict, ast.Tuple + ast.List, ast.Set, ast.Dict, ast.Tuple, + ast.Bytes, ast.NameConstant, ) - if PY3: - ast_data += (ast.Bytes, ast.NameConstant,) tree = ast.parse(src) @@ -234,10 +228,7 @@ def get_simple_vars_from_src(src): 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -267,7 +258,7 @@ def get_simple_vars_from_src(src): long_description=long_description, classifiers=classifiers, keywords=keywords, - python_requires='>=2.7.5,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', + python_requires='>=3.5', # While setuptools cannot deal with pre-installed incompatible versions, # setting a lower bound is not harmful - it makes error messages cleaner. DO # NOT set an upper bound on setuptools, as that will lead to uninstallable @@ -277,12 +268,6 @@ def get_simple_vars_from_src(src): setup_requires="setuptools >= 21.0.0,!=24.0.0," "!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2," "!=36.2.0", - extras_require={ - ':python_version == "2.7"': [ - 'futures>=1.0', - 'enum34>=1.1' - ], - }, install_requires=required, package_data={ str('exec_helpers'): [ diff --git a/test/test_subprocess_runner.py b/test/test_subprocess_runner.py index 1999214..0314d55 100644 --- a/test/test_subprocess_runner.py +++ b/test/test_subprocess_runner.py @@ -245,7 +245,7 @@ def test_005_execute_no_stdout( shell=True, stderr=subprocess.PIPE, stdin=subprocess.PIPE, - stdout=subprocess_runner.devnull, + stdout=subprocess.DEVNULL, universal_newlines=False, ), )) @@ -284,7 +284,7 @@ def test_006_execute_no_stderr( cwd=None, env=None, shell=True, - stderr=subprocess_runner.devnull, + stderr=subprocess.DEVNULL, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=False, @@ -330,9 +330,9 @@ def test_007_execute_no_stdout_stderr( cwd=None, env=None, shell=True, - stderr=subprocess_runner.devnull, + stderr=subprocess.DEVNULL, stdin=subprocess.PIPE, - stdout=subprocess_runner.devnull, + stdout=subprocess.DEVNULL, universal_newlines=False, ), )) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 032245f..0bc72ec 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,5 +1,5 @@ #!/bin/bash -PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m" +PYTHON_VERSIONS="cp35-cp35m cp36-cp36m cp37-cp37m" # Avoid creation of __pycache__/*.py[c|o] export PYTHONDONTWRITEBYTECODE=1 diff --git a/tox.ini b/tox.ini index f0bb51b..f51dade 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] minversion = 2.0 -envlist = pep8, pep257, py{27,34,35,36,37,py,py3}, pylint, bandit, py{34,35,36,37}-nocov, mypy, docs +envlist = pep8, pylint, mypy, bandit, pep257, py{35,36,37,py3}, docs, py{35,36,37}-nocov skipsdist = True skip_missing_interpreters = True @@ -20,22 +20,14 @@ deps = pytest-cov pytest-html pytest-sugar - py{27,34,35,36}-nocov: Cython + py{35,36}-nocov: Cython -r{toxinidir}/CI_REQUIREMENTS.txt - py{27,py}: mock commands = pip freeze py.test --junitxml=unit_result.xml --cov-report html --self-contained-html --html=report.html --cov-config .coveragerc --cov=exec_helpers {posargs:test} coverage report --fail-under 97 -[testenv:py34-nocov] -usedevelop = False -commands = - python setup.py bdist_wheel - pip install exec_helpers --no-index -f dist - py.test -vvv {posargs:test} - [testenv:py35-nocov] usedevelop = False commands = @@ -61,18 +53,14 @@ commands = commands = {posargs:} [tox:travis] -2.7 = install, py27, -3.4 = py34, 3.5 = py35, 3.6 = py36, 3.7 = py37, -pypy = install, pypy, pypy3 = install, pypy3, [testenv:pep8] deps = flake8 - flake8-future-import usedevelop = False commands = flake8 @@ -90,7 +78,7 @@ commands = pip install ./ -vvv -U [testenv:pylint] usedevelop = False deps = - pylint<2 + pylint>=2 -r{toxinidir}/CI_REQUIREMENTS.txt commands = pylint exec_helpers @@ -106,29 +94,7 @@ exclude = build, __init__.py, docs -ignore = - # Expected - # __future__ import "division" present - FI50 - # __future__ import "absolute_import" present - FI51 - # Setup only: - # __future__ import "print_function" present - FI53 - # __future__ import "unicode_literals" present - FI54 - - # Expected missind (not used or required for not supported versions): - # __future__ import "with_statement" missing - FI12 - # __future__ import "print_function" missing - FI13 - # __future__ import "generator_stop" missing - FI15 - # __future__ import "nested_scopes" missing - FI16 - # __future__ import "generators" missing - FI17 + show-pep8 = True show-source = True count = True