diff --git a/README.rst b/README.rst index 1f33084..a4d4560 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,8 @@ exec-helpers :target: https://pypi.python.org/pypi/exec-helpers .. image:: https://img.shields.io/github/license/python-useful-helpers/exec-helpers.svg :target: https://raw.githubusercontent.com/python-useful-helpers/exec-helpers/master/LICENSE +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black Execution helpers for simplified usage of subprocess and ssh. Why another subprocess wrapper and why no clear `paramiko`? diff --git a/exec_helpers/__init__.py b/exec_helpers/__init__.py index 82b6450..2fa526c 100644 --- a/exec_helpers/__init__.py +++ b/exec_helpers/__init__.py @@ -22,7 +22,7 @@ CalledProcessError, ParallelCallProcessError, ParallelCallExceptions, - ExecHelperTimeoutError + ExecHelperTimeoutError, ) from .exec_result import ExecResult @@ -33,32 +33,30 @@ from .subprocess_runner import Subprocess, SubprocessExecuteAsyncResult # nosec # Expected __all__ = ( - 'ExecHelperError', - 'ExecCalledProcessError', - 'CalledProcessError', - 'ParallelCallExceptions', - 'ParallelCallProcessError', - 'ExecHelperTimeoutError', - 'ExecHelper', - 'SSHClient', - 'SshExecuteAsyncResult', - 'SSHAuth', - 'Subprocess', - 'SubprocessExecuteAsyncResult', - 'ExitCodes', - 'ExecResult', + "ExecHelperError", + "ExecCalledProcessError", + "CalledProcessError", + "ParallelCallExceptions", + "ParallelCallProcessError", + "ExecHelperTimeoutError", + "ExecHelper", + "SSHClient", + "SshExecuteAsyncResult", + "SSHAuth", + "Subprocess", + "SubprocessExecuteAsyncResult", + "ExitCodes", + "ExecResult", ) -__version__ = '2.0.2' +__version__ = "2.0.2" __author__ = "Alexey Stepanov" -__author_email__ = 'penguinolog@gmail.com' +__author_email__ = "penguinolog@gmail.com" __maintainers__ = { - 'Alexey Stepanov': 'penguinolog@gmail.com', - 'Antonio Esposito': 'esposito.cloud@gmail.com', - 'Dennis Dmitriev': 'dis-xcom@gmail.com', + "Alexey Stepanov": "penguinolog@gmail.com", + "Antonio Esposito": "esposito.cloud@gmail.com", + "Dennis Dmitriev": "dis-xcom@gmail.com", } -__url__ = 'https://github.com/python-useful-helpers/exec-helpers' -__description__ = ( - "Execution helpers for simplified usage of subprocess and ssh." -) +__url__ = "https://github.com/python-useful-helpers/exec-helpers" +__description__ = "Execution helpers for simplified usage of subprocess and ssh." __license__ = "Apache License, Version 2.0" diff --git a/exec_helpers/_log_templates.py b/exec_helpers/_log_templates.py index c0d9712..d2f551f 100644 --- a/exec_helpers/_log_templates.py +++ b/exec_helpers/_log_templates.py @@ -20,8 +20,8 @@ CMD_WAIT_ERROR = ( "Wait for {result.cmd!r} during {timeout!s}s: no return code!\n" - '\tSTDOUT:\n' - '{result.stdout_brief}\n' - '\tSTDERR"\n' - '{result.stderr_brief}' + "\tSTDOUT:\n" + "{result.stdout_brief}\n" + "\tSTDERR:\n" + "{result.stderr_brief}" ) diff --git a/exec_helpers/_ssh_client_base.py b/exec_helpers/_ssh_client_base.py index 80f05b1..084356f 100644 --- a/exec_helpers/_ssh_client_base.py +++ b/exec_helpers/_ssh_client_base.py @@ -16,10 +16,7 @@ """SSH client helper based on Paramiko. Base class.""" -__all__ = ( - 'SSHClientBase', - 'SshExecuteAsyncResult', -) +__all__ = ("SSHClientBase", "SshExecuteAsyncResult") import abc import base64 @@ -48,8 +45,8 @@ from exec_helpers import ssh_auth from exec_helpers import _log_templates -logging.getLogger('paramiko').setLevel(logging.WARNING) -logging.getLogger('iso8601').setLevel(logging.WARNING) +logging.getLogger("paramiko").setLevel(logging.WARNING) +logging.getLogger("iso8601").setLevel(logging.WARNING) class SshExecuteAsyncResult(api.ExecuteAsyncResult): @@ -76,7 +73,7 @@ def stdout(self) -> typing.Optional[paramiko.ChannelFile]: # type: ignore return super(SshExecuteAsyncResult, self).stdout -CPYTHON = 'CPython' == platform.python_implementation() +CPYTHON = "CPython" == platform.python_implementation() class _MemorizedSSH(abc.ABCMeta): @@ -112,10 +109,7 @@ class _MemorizedSSH(abc.ABCMeta): @classmethod def __prepare__( # pylint: disable=unused-argument - mcs: typing.Type['_MemorizedSSH'], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Any + mcs: typing.Type["_MemorizedSSH"], name: str, bases: typing.Iterable[typing.Type], **kwargs: typing.Any ) -> collections.OrderedDict: """Metaclass magic for object storage. @@ -124,7 +118,7 @@ def __prepare__( # pylint: disable=unused-argument return collections.OrderedDict() # pragma: no cover def __call__( # type: ignore - cls: '_MemorizedSSH', + cls: "_MemorizedSSH", host: str, port: int = 22, username: typing.Optional[str] = None, @@ -132,7 +126,7 @@ def __call__( # type: ignore private_keys: typing.Optional[typing.Iterable[paramiko.RSAKey]] = None, auth: typing.Optional[ssh_auth.SSHAuth] = None, verbose: bool = True, - ) -> 'SSHClientBase': + ) -> "SSHClientBase": """Main memorize method: check for cached instance and return it. API follows target __init__. :param host: remote hostname @@ -155,42 +149,37 @@ def __call__( # type: ignore if (host, port) in cls.__cache: key = host, port if auth is None: - auth = ssh_auth.SSHAuth( - username=username, - password=password, - keys=private_keys - ) + auth = ssh_auth.SSHAuth(username=username, password=password, keys=private_keys) if hash((cls, host, port, auth)) == hash(cls.__cache[key]): ssh = cls.__cache[key] # noinspection PyBroadException try: - ssh.execute('cd ~', timeout=5) + ssh.execute("cd ~", timeout=5) except BaseException: # Note: Do not change to lower level! - ssh.logger.debug('Reconnect') + ssh.logger.debug("Reconnect") ssh.reconnect() return ssh - if ( - CPYTHON and - sys.getrefcount(cls.__cache[key]) == 2 - ): # pragma: no cover + if CPYTHON and sys.getrefcount(cls.__cache[key]) == 2: # pragma: no cover # If we have only cache reference and temporary getrefcount # reference: close connection before deletion - cls.__cache[key].logger.debug('Closing as unused') + cls.__cache[key].logger.debug("Closing as unused") cls.__cache[key].close() # type: ignore del cls.__cache[key] # noinspection PyArgumentList - ssh = super( - _MemorizedSSH, - cls - ).__call__( - host=host, port=port, - username=username, password=password, private_keys=private_keys, - auth=auth, verbose=verbose) + ssh = super(_MemorizedSSH, cls).__call__( + host=host, + port=port, + username=username, + password=password, + private_keys=private_keys, + auth=auth, + verbose=verbose, + ) cls.__cache[(ssh.hostname, ssh.port)] = ssh return ssh @classmethod - def clear_cache(mcs: typing.Type['_MemorizedSSH']) -> None: + def clear_cache(mcs: typing.Type["_MemorizedSSH"]) -> None: """Clear cached connections for initialize new instance on next call. getrefcount is used to check for usage, so connections closed on CPYTHON only. @@ -199,16 +188,13 @@ def clear_cache(mcs: typing.Type['_MemorizedSSH']) -> None: # PY3: cache, ssh, temporary # PY4: cache, values mapping, ssh, temporary for ssh in mcs.__cache.values(): - if ( - CPYTHON and - sys.getrefcount(ssh) == n_count - ): # pragma: no cover - ssh.logger.debug('Closing as unused') + if CPYTHON and sys.getrefcount(ssh) == n_count: # pragma: no cover + ssh.logger.debug("Closing as unused") ssh.close() # type: ignore mcs.__cache = {} @classmethod - def close_connections(mcs: typing.Type['_MemorizedSSH']) -> None: + def close_connections(mcs: typing.Type["_MemorizedSSH"]) -> None: """Close connections for selected or all cached records.""" for ssh in mcs.__cache.values(): if ssh.is_alive: @@ -218,25 +204,14 @@ def close_connections(mcs: typing.Type['_MemorizedSSH']) -> None: class SSHClientBase(api.ExecHelper, metaclass=_MemorizedSSH): """SSH Client helper.""" - __slots__ = ( - '__hostname', '__port', '__auth', '__ssh', '__sftp', - '__sudo_mode', '__keepalive_mode', '__verbose', - ) + __slots__ = ("__hostname", "__port", "__auth", "__ssh", "__sftp", "__sudo_mode", "__keepalive_mode", "__verbose") class __get_sudo: """Context manager for call commands with sudo.""" - __slots__ = ( - '__ssh', - '__sudo_status', - '__enforce', - ) + __slots__ = ("__ssh", "__sudo_status", "__enforce") - def __init__( - self, - ssh: 'SSHClientBase', - enforce: typing.Optional[bool] = None - ) -> None: + def __init__(self, ssh: "SSHClientBase", enforce: typing.Optional[bool] = None) -> None: """Context manager for call commands with sudo. :param ssh: connection instance @@ -259,17 +234,9 @@ def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any class __get_keepalive: """Context manager for keepalive management.""" - __slots__ = ( - '__ssh', - '__keepalive_status', - '__enforce', - ) + __slots__ = ("__ssh", "__keepalive_status", "__enforce") - def __init__( - self, - ssh: 'SSHClientBase', - enforce: bool = True - ) -> None: + def __init__(self, ssh: "SSHClientBase", enforce: bool = True) -> None: """Context manager for keepalive management. :param ssh: connection instance @@ -294,11 +261,7 @@ def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any def __hash__(self) -> int: """Hash for usage as dict keys.""" - return hash(( - self.__class__, - self.hostname, - self.port, - self.auth)) + return hash((self.__class__, self.hostname, self.port, self.auth)) def __init__( self, @@ -330,11 +293,7 @@ def __init__( .. note:: auth has priority over username/password/private_keys """ super(SSHClientBase, self).__init__( - logger=logging.getLogger( - self.__class__.__name__ - ).getChild( - '{host}:{port}'.format(host=host, port=port) - ), + logger=logging.getLogger(self.__class__.__name__).getChild("{host}:{port}".format(host=host, port=port)) ) self.__hostname = host @@ -349,11 +308,7 @@ def __init__( self.__sftp = None if auth is None: - self.__auth = ssh_auth.SSHAuth( - username=username, - password=password, - keys=private_keys - ) + self.__auth = ssh_auth.SSHAuth(username=username, password=password, keys=private_keys) else: self.__auth = copy.copy(auth) @@ -398,16 +353,14 @@ def is_alive(self) -> bool: def __repr__(self) -> str: """Representation for debug purposes.""" - return '{cls}(host={host}, port={port}, auth={auth!r})'.format( - cls=self.__class__.__name__, host=self.hostname, port=self.port, - auth=self.auth + return "{cls}(host={self.hostname}, port={self.port}, auth={self.auth!r})".format( + cls=self.__class__.__name__, self=self ) def __str__(self) -> str: # pragma: no cover """Representation for debug purposes.""" - return '{cls}(host={host}, port={port}) for user {user}'.format( - cls=self.__class__.__name__, host=self.hostname, port=self.port, - user=self.auth.username + return "{cls}(host={self.hostname}, port={self.port}) for user {self.auth.username}".format( + cls=self.__class__.__name__, self=self ) @property @@ -430,10 +383,7 @@ def _ssh(self) -> paramiko.SSHClient: def __connect(self) -> None: """Main method for connection open.""" with self.lock: - self.auth.connect( - client=self.__ssh, - hostname=self.hostname, port=self.port, - log=self.__verbose) + self.auth.connect(client=self.__ssh, hostname=self.hostname, port=self.port, log=self.__verbose) def __connect_sftp(self) -> None: """SFTP connection opener.""" @@ -441,9 +391,7 @@ def __connect_sftp(self) -> None: try: self.__sftp = self.__ssh.open_sftp() except paramiko.SSHException: - self.logger.warning( - 'SFTP enable failed! SSH only is accessible.' - ) + self.logger.warning("SFTP enable failed! SSH only is accessible.") @property def _sftp(self) -> paramiko.sftp_client.SFTPClient: @@ -454,11 +402,11 @@ def _sftp(self) -> paramiko.sftp_client.SFTPClient: """ if self.__sftp is not None: return self.__sftp - self.logger.debug('SFTP is not connected, try to connect...') + self.logger.debug("SFTP is not connected, try to connect...") self.__connect_sftp() if self.__sftp is not None: return self.__sftp - raise paramiko.SSHException('SFTP connection failed') + raise paramiko.SSHException("SFTP connection failed") @advanced_descriptors.SeparateClassMethod def close(self) -> None: @@ -475,24 +423,19 @@ def close(self) -> None: try: self.__sftp.close() except Exception: - self.logger.exception( - "Could not close sftp connection" - ) + self.logger.exception("Could not close sftp connection") # noinspection PyMethodParameters @close.class_method # type: ignore - def close(cls: typing.Type['SSHClientBase']) -> None: # pylint: disable=no-self-argument + def close(cls: typing.Type["SSHClientBase"]) -> None: # pylint: disable=no-self-argument """Close all memorized SSH and SFTP sessions.""" # noinspection PyUnresolvedReferences cls.__class__.close_connections() @classmethod - def _clear_cache(cls: typing.Type['SSHClientBase']) -> None: + def _clear_cache(cls: typing.Type["SSHClientBase"]) -> None: """Enforce clear memorized records.""" - warnings.warn( - '_clear_cache() is dangerous and not recommended for normal use!', - Warning - ) + warnings.warn("_clear_cache() is dangerous and not recommended for normal use!", Warning) _MemorizedSSH.clear_cache() def __del__(self) -> None: @@ -504,12 +447,7 @@ def __del__(self) -> None: try: self.__ssh.close() except BaseException as e: # pragma: no cover - self.logger.debug( - 'Exception in {self!s} destructor call: {exc}'.format( - self=self, - exc=e - ) - ) + self.logger.debug("Exception in {self!s} destructor call: {exc}".format(self=self, exc=e)) self.__sftp = None def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any) -> None: @@ -566,10 +504,7 @@ def reconnect(self) -> None: self.__connect() - def sudo( - self, - enforce: typing.Optional[bool] = None - ) -> 'typing.ContextManager': + def sudo(self, enforce: typing.Optional[bool] = None) -> "typing.ContextManager": """Call contextmanager for sudo mode change. :param enforce: Enforce sudo enabled or disabled. By default: None @@ -579,10 +514,7 @@ def sudo( """ return self.__get_sudo(ssh=self, enforce=enforce) - def keepalive( - self, - enforce: bool = True - ) -> 'typing.ContextManager': + def keepalive(self, enforce: bool = True) -> "typing.ContextManager": """Call contextmanager with keepalive mode change. :param enforce: Enforce keepalive enabled or disabled. @@ -638,34 +570,32 @@ def execute_async( .. versionchanged:: 1.2.0 get_pty moved to `**kwargs` .. versionchanged:: 2.1.0 Use typed NamedTuple as result """ - cmd_for_log = self._mask_command( - cmd=command, - log_mask_re=log_mask_re - ) + cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) self.logger.log( # type: ignore - level=logging.INFO if verbose else logging.DEBUG, - msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) + level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) chan = self._ssh.get_transport().open_session() - if kwargs.get('get_pty', False): + if kwargs.get("get_pty", False): # Open PTY chan.get_pty( - term='vt100', - width=kwargs.get('width', 80), height=kwargs.get('height', 24), - width_pixels=0, height_pixels=0 + term="vt100", + width=kwargs.get("width", 80), + height=kwargs.get("height", 24), + width_pixels=0, + height_pixels=0, ) - _stdin = chan.makefile('wb') # type: paramiko.ChannelFile - stdout = chan.makefile('rb') if open_stdout else None - stderr = chan.makefile_stderr('rb') if open_stderr else None + _stdin = chan.makefile("wb") # type: paramiko.ChannelFile + stdout = chan.makefile("rb") if open_stdout else None + stderr = chan.makefile_stderr("rb") if open_stderr else None cmd = "{command}\n".format(command=command) if self.sudo_mode: - encoded_cmd = base64.b64encode(cmd.encode('utf-8')).decode('utf-8') - cmd = "sudo -S bash -c 'eval \"$(base64 -d <(echo \"{0}\"))\"'".format(encoded_cmd) + encoded_cmd = base64.b64encode(cmd.encode("utf-8")).decode("utf-8") + cmd = 'sudo -S bash -c \'eval "$(base64 -d <(echo "{0}"))"\''.format(encoded_cmd) chan.exec_command(cmd) # nosec # Sanitize on caller side if stdout.channel.closed is False: # noinspection PyTypeChecker @@ -676,10 +606,10 @@ def execute_async( if stdin is not None: if not _stdin.channel.closed: - _stdin.write('{stdin}\n'.format(stdin=stdin)) + _stdin.write("{stdin}\n".format(stdin=stdin)) _stdin.flush() else: - self.logger.warning('STDIN Send failed: closed channel') + self.logger.warning("STDIN Send failed: closed channel") return SshExecuteAsyncResult(chan, _stdin, stderr, stdout) @@ -719,20 +649,13 @@ def _exec_command( .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ + def poll_streams() -> None: """Poll FIFO buffers if data available.""" if stdout and interface.recv_ready(): - result.read_stdout( - src=stdout, - log=self.logger, - verbose=verbose - ) + result.read_stdout(src=stdout, log=self.logger, verbose=verbose) if stderr and interface.recv_stderr_ready(): - result.read_stderr( - src=stderr, - log=self.logger, - verbose=verbose - ) + result.read_stderr(src=stderr, log=self.logger, verbose=verbose) @threaded.threadpooled # type: ignore def poll_pipes(stop: threading.Event) -> None: @@ -746,24 +669,14 @@ def poll_pipes(stop: threading.Event) -> None: poll_streams() if interface.status_event.is_set(): - result.read_stdout( - src=stdout, - log=self.logger, - verbose=verbose) - result.read_stderr( - src=stderr, - log=self.logger, - verbose=verbose - ) + result.read_stdout(src=stdout, log=self.logger, verbose=verbose) + result.read_stderr(src=stderr, log=self.logger, verbose=verbose) result.exit_code = interface.exit_status stop.set() # channel.status_event.wait(timeout) - cmd_for_log = self._mask_command( - cmd=command, - log_mask_re=log_mask_re - ) + cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) # Store command with hidden data result = exec_result.ExecResult(cmd=cmd_for_log) @@ -786,10 +699,7 @@ def poll_pipes(stop: threading.Event) -> None: interface.close() future.cancel() - wait_err_msg = _log_templates.CMD_WAIT_ERROR.format( - result=result, - timeout=timeout - ) + wait_err_msg = _log_templates.CMD_WAIT_ERROR.format(result=result, timeout=timeout) self.logger.debug(wait_err_msg) raise exceptions.ExecHelperTimeoutError(result=result, timeout=timeout) # type: ignore @@ -829,22 +739,17 @@ def execute_through_host( .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ - cmd_for_log = self._mask_command( - cmd=command, - log_mask_re=kwargs.get('log_mask_re', None) - ) + cmd_for_log = self._mask_command(cmd=command, log_mask_re=kwargs.get("log_mask_re", None)) self.logger.log( # type: ignore - level=logging.INFO if verbose else logging.DEBUG, - msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) + level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) if auth is None: auth = self.auth intermediate_channel = self._ssh.get_transport().open_channel( - kind='direct-tcpip', - dest_addr=(hostname, target_port), - src_addr=(self.hostname, 0)) + kind="direct-tcpip", dest_addr=(hostname, target_port), src_addr=(self.hostname, 0) + ) transport = paramiko.Transport(sock=intermediate_channel) # start client and authenticate transport @@ -855,21 +760,22 @@ def execute_through_host( if get_pty: # Open PTY channel.get_pty( - term='vt100', - width=kwargs.get('width', 80), height=kwargs.get('height', 24), - width_pixels=0, height_pixels=0 + term="vt100", + width=kwargs.get("width", 80), + height=kwargs.get("height", 24), + width_pixels=0, + height_pixels=0, ) # Make proxy objects for read - stdout = channel.makefile('rb') - stderr = channel.makefile_stderr('rb') + stdout = channel.makefile("rb") + stderr = channel.makefile_stderr("rb") channel.exec_command(command) # nosec # Sanitize on caller side # noinspection PyDictCreation result = self._exec_command( - command, channel, stdout, stderr, timeout, verbose=verbose, - log_mask_re=kwargs.get('log_mask_re', None), + command, channel, stdout, stderr, timeout, verbose=verbose, log_mask_re=kwargs.get("log_mask_re", None) ) intermediate_channel.close() @@ -879,7 +785,7 @@ def execute_through_host( @classmethod def execute_together( cls, - remotes: typing.Iterable['SSHClientBase'], + remotes: typing.Iterable["SSHClientBase"], command: str, timeout: typing.Union[int, float, None] = constants.DEFAULT_TIMEOUT, expected: typing.Optional[typing.Iterable[int]] = None, @@ -908,8 +814,9 @@ def execute_together( .. versionchanged:: 1.2.0 default timeout 1 hour .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd """ + @threaded.threadpooled # type: ignore - def get_result(remote: 'SSHClientBase') -> exec_result.ExecResult: + def get_result(remote: "SSHClientBase") -> exec_result.ExecResult: """Get result from remote call.""" async_result = remote.execute_async(command, **kwargs) # type: SshExecuteAsyncResult @@ -917,10 +824,7 @@ def get_result(remote: 'SSHClientBase') -> exec_result.ExecResult: exit_code = async_result.interface.recv_exit_status() # pylint: disable=protected-access - cmd_for_log = remote._mask_command( - cmd=command, - log_mask_re=kwargs.get('log_mask_re', None) - ) + cmd_for_log = remote._mask_command(cmd=command, log_mask_re=kwargs.get("log_mask_re", None)) # pylint: enable=protected-access result = exec_result.ExecResult(cmd=cmd_for_log) @@ -939,21 +843,14 @@ def get_result(remote: 'SSHClientBase') -> exec_result.ExecResult: errors = {} raised_exceptions = {} - ( - _, - not_done, - ) = concurrent.futures.wait( - list(futures.values()), - timeout=timeout + (_, not_done) = concurrent.futures.wait( + list(futures.values()), timeout=timeout ) # type: typing.Set[concurrent.futures.Future], typing.Set[concurrent.futures.Future] - for future in not_done: # pragma: no cover - future.cancel() + for fut in not_done: # pragma: no cover + fut.cancel() - for ( - remote, - future, # type: ignore - ) in futures.items(): # type: SSHClientBase, concurrent.futures.Future + for (remote, future) in futures.items(): # type: SSHClientBase, concurrent.futures.Future try: result = future.result() results[(remote.hostname, remote.port)] = result @@ -963,20 +860,12 @@ def get_result(remote: 'SSHClientBase') -> exec_result.ExecResult: raised_exceptions[(remote.hostname, remote.port)] = e if raised_exceptions: # always raise - raise exceptions.ParallelCallExceptions( - command, - raised_exceptions, - errors, - results, - expected=expected - ) + raise exceptions.ParallelCallExceptions(command, raised_exceptions, errors, results, expected=expected) if errors and raise_on_err: - raise exceptions.ParallelCallProcessError( - command, errors, results, expected=expected - ) + raise exceptions.ParallelCallProcessError(command, errors, results, expected=expected) return results - def open(self, path: str, mode: str = 'r') -> paramiko.SFTPFile: + def open(self, path: str, mode: str = "r") -> paramiko.SFTPFile: """Open file on remote using SFTP session. :param path: filesystem object path @@ -1012,11 +901,7 @@ def stat(self, path: str) -> paramiko.sftp_attr.SFTPAttributes: """ return self._sftp.stat(path) # pragma: no cover - def utime( - self, - path: str, - times: typing.Optional[typing.Tuple[int, int]] = None - ) -> None: + def utime(self, path: str, times: typing.Optional[typing.Tuple[int, int]] = None) -> None: """Set atime, mtime. :param path: filesystem object path diff --git a/exec_helpers/api.py b/exec_helpers/api.py index 08dd89b..a1b0b68 100644 --- a/exec_helpers/api.py +++ b/exec_helpers/api.py @@ -19,10 +19,7 @@ .. versionchanged:: 1.3.5 make API public to use as interface """ -__all__ = ( - 'ExecHelper', - 'ExecuteAsyncResult', -) +__all__ = ("ExecHelper", "ExecuteAsyncResult") import abc import logging @@ -37,30 +34,22 @@ ExecuteAsyncResult = typing.NamedTuple( - 'ExecuteAsyncResult', + "ExecuteAsyncResult", [ - ('interface', typing.Any), - ('stdin', typing.Optional[typing.Any]), - ('stderr', typing.Optional[typing.Any]), - ('stdout', typing.Optional[typing.Any]), - ] + ("interface", typing.Any), + ("stdin", typing.Optional[typing.Any]), + ("stderr", typing.Optional[typing.Any]), + ("stdout", typing.Optional[typing.Any]), + ], ) class ExecHelper(metaclass=abc.ABCMeta): """ExecHelper global API.""" - __slots__ = ( - '__lock', - '__logger', - 'log_mask_re' - ) + __slots__ = ("__lock", "__logger", "log_mask_re") - def __init__( - self, - logger: logging.Logger, - log_mask_re: typing.Optional[str] = None, - ) -> None: + def __init__(self, logger: logging.Logger, log_mask_re: typing.Optional[str] = None) -> None: """Global ExecHelper API. :param logger: logger instance to use @@ -89,7 +78,7 @@ def lock(self) -> threading.RLock: """ return self.__lock - def __enter__(self) -> 'ExecHelper': + def __enter__(self) -> "ExecHelper": """Get context manager. .. versionchanged:: 1.1.0 lock on enter @@ -101,11 +90,7 @@ def __exit__(self, exc_type: typing.Any, exc_val: typing.Any, exc_tb: typing.Any """Context manager usage.""" self.lock.release() - def _mask_command( - self, - cmd: str, - log_mask_re: typing.Optional[str] = None, - ) -> str: + def _mask_command(self, cmd: str, log_mask_re: typing.Optional[str] = None) -> str: """Log command with masking and return parsed cmd. :param cmd: command @@ -118,6 +103,7 @@ def _mask_command( .. versionadded:: 1.2.0 """ + def mask(text: str, rules: str) -> str: """Mask part of text using rules.""" indexes = [0] # Start of the line @@ -134,9 +120,9 @@ def mask(text: str, rules: str) -> str: for idx in range(0, len(indexes) - 2, 2): start = indexes[idx] end = indexes[idx + 1] - masked += text[start: end] + '<*masked*>' + masked += text[start:end] + "<*masked*>" - masked += text[indexes[-2]: indexes[-1]] # final part + masked += text[indexes[-2] : indexes[-1]] # final part return masked cmd = cmd.rstrip() @@ -268,10 +254,7 @@ def execute( **kwargs ) message = "Command {result.cmd!r} exit code: {result.exit_code!s}".format(result=result) - self.logger.log( # type: ignore - level=logging.INFO if verbose else logging.DEBUG, - msg=message - ) + self.logger.log(level=logging.INFO if verbose else logging.DEBUG, msg=message) # type: ignore return result def check_call( @@ -309,20 +292,16 @@ def check_call( """ expected = proc_enums.exit_codes_to_enums(expected) ret = self.execute(command, verbose, timeout, **kwargs) - if ret['exit_code'] not in expected: + if ret["exit_code"] not in expected: message = ( "{append}Command {result.cmd!r} returned exit code " "{result.exit_code!s} while expected {expected!s}".format( - append=error_info + '\n' if error_info else '', - result=ret, - expected=expected - )) + append=error_info + "\n" if error_info else "", result=ret, expected=expected + ) + ) self.logger.error(message) if raise_on_err: - raise exceptions.CalledProcessError( - result=ret, - expected=expected, - ) + raise exceptions.CalledProcessError(result=ret, expected=expected) return ret def check_stderr( @@ -356,19 +335,14 @@ def check_stderr( .. versionchanged:: 1.2.0 default timeout 1 hour """ ret = self.check_call( - command, verbose, timeout=timeout, - error_info=error_info, raise_on_err=raise_on_err, **kwargs) - if ret['stderr']: + command, verbose, timeout=timeout, error_info=error_info, raise_on_err=raise_on_err, **kwargs + ) + if ret["stderr"]: message = ( "{append}Command {result.cmd!r} STDERR while not expected\n" - "\texit code: {result.exit_code!s}".format( - append=error_info + '\n' if error_info else '', - result=ret, - )) + "\texit code: {result.exit_code!s}".format(append=error_info + "\n" if error_info else "", result=ret) + ) self.logger.error(message) if raise_on_err: - raise exceptions.CalledProcessError( - result=ret, - expected=kwargs.get('expected'), - ) + raise exceptions.CalledProcessError(result=ret, expected=kwargs.get("expected")) return ret diff --git a/exec_helpers/exceptions.py b/exec_helpers/exceptions.py index 84f4848..1190744 100644 --- a/exec_helpers/exceptions.py +++ b/exec_helpers/exceptions.py @@ -23,12 +23,12 @@ from exec_helpers import exec_result # noqa: F401 # pylint: disable=cyclic-import __all__ = ( - 'ExecHelperError', - 'ExecHelperTimeoutError', - 'ExecCalledProcessError', - 'CalledProcessError', - 'ParallelCallProcessError', - 'ParallelCallExceptions', + "ExecHelperError", + "ExecHelperTimeoutError", + "ExecCalledProcessError", + "CalledProcessError", + "ParallelCallProcessError", + "ParallelCallExceptions", ) @@ -57,16 +57,9 @@ class ExecHelperTimeoutError(ExecCalledProcessError): .. versionchanged:: 1.3.0 subclass ExecCalledProcessError """ - __slots__ = ( - 'result', - 'timeout', - ) + __slots__ = ("result", "timeout") - def __init__( - self, - result: 'exec_result.ExecResult', - timeout: typing.Union[int, float], - ) -> None: + def __init__(self, result: "exec_result.ExecResult", timeout: typing.Union[int, float]) -> None: """Exception for error on process calls. :param result: execution result @@ -76,10 +69,7 @@ def __init__( """ self.result = result self.timeout = timeout - message = _log_templates.CMD_WAIT_ERROR.format( - result=result, - timeout=timeout - ) + message = _log_templates.CMD_WAIT_ERROR.format(result=result, timeout=timeout) super(ExecHelperTimeoutError, self).__init__(message) @property @@ -101,14 +91,11 @@ def stderr(self) -> str: class CalledProcessError(ExecCalledProcessError): """Exception for error on process calls.""" - __slots__ = ( - 'result', - 'expected', - ) + __slots__ = ("result", "expected") def __init__( self, - result: 'exec_result.ExecResult', + result: "exec_result.ExecResult", expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, ) -> None: """Exception for error on process calls. @@ -128,10 +115,7 @@ def __init__( "while expected {expected}\n" "\tSTDOUT:\n" "{result.stdout_brief}\n" - "\tSTDERR:\n{result.stderr_brief}".format( - result=self.result, - expected=self.expected - ) + "\tSTDERR:\n{result.stderr_brief}".format(result=self.result, expected=self.expected) ) super(CalledProcessError, self).__init__(message) @@ -159,20 +143,14 @@ def stderr(self) -> str: class ParallelCallExceptions(ExecCalledProcessError): """Exception raised during parallel call as result of exceptions.""" - __slots__ = ( - 'cmd', - 'exceptions', - 'errors', - 'results', - 'expected' - ) + __slots__ = ("cmd", "exceptions", "errors", "results", "expected") def __init__( self, command: str, exceptions: typing.Dict[typing.Tuple[str, int], Exception], - errors: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], - results: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + errors: typing.Dict[typing.Tuple[str, int], "exec_result.ExecResult"], + results: typing.Dict[typing.Tuple[str, int], "exec_result.ExecResult"], expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, ) -> None: """Exception raised during parallel call as result of exceptions. @@ -200,11 +178,9 @@ def __init__( "\t{exceptions}".format( self=self, exceptions="\n\t".join( - "{host}:{port} - {exc} ".format( - host=host, port=port, exc=exc - ) + "{host}:{port} - {exc} ".format(host=host, port=port, exc=exc) for (host, port), exc in exceptions.items() - ) + ), ) ) super(ParallelCallExceptions, self).__init__(message) @@ -213,18 +189,13 @@ def __init__( class ParallelCallProcessError(ExecCalledProcessError): """Exception during parallel execution.""" - __slots__ = ( - 'cmd', - 'errors', - 'results', - 'expected' - ) + __slots__ = ("cmd", "errors", "results", "expected") def __init__( self, command: str, - errors: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], - results: typing.Dict[typing.Tuple[str, int], 'exec_result.ExecResult'], + errors: typing.Dict[typing.Tuple[str, int], "exec_result.ExecResult"], + results: typing.Dict[typing.Tuple[str, int], "exec_result.ExecResult"], expected: typing.Optional[typing.List[typing.Union[int, proc_enums.ExitCodes]]] = None, ) -> None: """Exception during parallel execution. @@ -251,11 +222,9 @@ def __init__( "\t{errors}".format( self=self, errors="\n\t".join( - "{host}:{port} - {code} ".format( - host=host, port=port, code=result.exit_code - ) + "{host}:{port} - {code} ".format(host=host, port=port, code=result.exit_code) for (host, port), result in errors.items() - ) + ), ) ) super(ParallelCallProcessError, self).__init__(message) diff --git a/exec_helpers/exec_result.py b/exec_helpers/exec_result.py index 87dac19..5a9829e 100644 --- a/exec_helpers/exec_result.py +++ b/exec_helpers/exec_result.py @@ -27,7 +27,7 @@ from exec_helpers import exceptions # pylint: disable=cyclic-import from exec_helpers import proc_enums -__all__ = ('ExecResult', ) +__all__ = ("ExecResult",) logger = logging.getLogger(__name__) @@ -36,10 +36,17 @@ class ExecResult: """Execution result.""" __slots__ = [ - '__cmd', '__stdin', '__stdout', '__stderr', '__exit_code', - '__timestamp', - '__stdout_str', '__stderr_str', '__stdout_brief', '__stderr_brief', - '__lock' + "__cmd", + "__stdin", + "__stdout", + "__stderr", + "__exit_code", + "__timestamp", + "__stdout_str", + "__stderr_str", + "__stdout_brief", + "__stderr_brief", + "__lock", ] def __init__( @@ -48,7 +55,7 @@ def __init__( stdin: typing.Union[bytes, str, bytearray, None] = None, stdout: typing.Optional[typing.Iterable[bytes]] = None, stderr: typing.Optional[typing.Iterable[bytes]] = None, - exit_code: typing.Union[int, proc_enums.ExitCodes] = proc_enums.ExitCodes.EX_INVALID + exit_code: typing.Union[int, proc_enums.ExitCodes] = proc_enums.ExitCodes.EX_INVALID, ) -> None: """Command execution result. @@ -119,7 +126,7 @@ def _get_bytearray_from_array(src: typing.Iterable[bytes]) -> bytearray: :return: bytearray :rtype: bytearray """ - return bytearray(b''.join(src)) + return bytearray(b"".join(src)) @staticmethod def _get_str_from_bin(src: bytearray) -> str: @@ -130,10 +137,7 @@ def _get_str_from_bin(src: bytearray) -> str: :return: decoded string :rtype: str """ - return src.strip().decode( - encoding='utf-8', - errors='backslashreplace' - ) + return src.strip().decode(encoding="utf-8", errors="backslashreplace") @classmethod def _get_brief(cls, data: typing.Tuple[bytes, ...]) -> str: @@ -147,10 +151,8 @@ def _get_brief(cls, data: typing.Tuple[bytes, ...]) -> str: if len(data) <= 7: src = data # type: typing.Tuple[bytes, ...] else: - src = data[:3] + (b'...\n',) + data[-3:] - return cls._get_str_from_bin( - cls._get_bytearray_from_array(src) - ) + src = data[:3] + (b"...\n",) + data[-3:] + return cls._get_str_from_bin(cls._get_bytearray_from_array(src)) @property def cmd(self) -> str: @@ -186,9 +188,7 @@ def stderr(self) -> typing.Tuple[bytes, ...]: @staticmethod def __poll_stream( - src: typing.Iterable[bytes], - log: typing.Optional[logging.Logger] = None, - verbose: bool = False + src: typing.Iterable[bytes], log: typing.Optional[logging.Logger] = None, verbose: bool = False ) -> typing.List[bytes]: dst = [] try: @@ -197,7 +197,7 @@ def __poll_stream( if log: log.log( # type: ignore level=logging.INFO if verbose else logging.DEBUG, - msg=line.decode('utf-8', errors='backslashreplace').rstrip() + msg=line.decode("utf-8", errors="backslashreplace").rstrip(), ) except IOError: pass @@ -207,7 +207,7 @@ def read_stdout( self, src: typing.Optional[typing.Iterable] = None, log: typing.Optional[logging.Logger] = None, - verbose: bool = False + verbose: bool = False, ) -> None: """Read stdout file-like object to stdout. @@ -224,7 +224,7 @@ def read_stdout( if not src: return if self.timestamp: - raise RuntimeError('Final exit code received.') + raise RuntimeError("Final exit code received.") with self.lock: self.__stdout_str = self.__stdout_brief = None @@ -234,7 +234,7 @@ def read_stderr( self, src: typing.Optional[typing.Iterable] = None, log: typing.Optional[logging.Logger] = None, - verbose: bool = False + verbose: bool = False, ) -> None: """Read stderr file-like object to stdout. @@ -251,7 +251,7 @@ def read_stderr( if not src: return if self.timestamp: - raise RuntimeError('Final exit code received.') + raise RuntimeError("Final exit code received.") with self.lock: self.__stderr_str = self.__stderr_brief = None @@ -342,9 +342,9 @@ def exit_code(self, new_val: typing.Union[int, proc_enums.ExitCodes]) -> None: If valid exit code is set - object became read-only. """ if self.timestamp: - raise RuntimeError('Exit code is already received.') + raise RuntimeError("Exit code is already received.") if not isinstance(new_val, int): - raise TypeError('Exit code is strictly int, received: {code!r}'.format(code=new_val)) + raise TypeError("Exit code is strictly int, received: {code!r}".format(code=new_val)) with self.lock: self.__exit_code = proc_enums.exit_code_to_enum(new_val) if self.__exit_code != proc_enums.ExitCodes.EX_INVALID: @@ -361,20 +361,15 @@ def __deserialize(self, fmt: str) -> typing.Any: :raises DeserializeValueError: Not valid source format """ try: - if fmt == 'json': # pylint: disable=no-else-return - return json.loads(self.stdout_str, encoding='utf-8') - elif fmt == 'yaml': + if fmt == "json": # pylint: disable=no-else-return + return json.loads(self.stdout_str, encoding="utf-8") + elif fmt == "yaml": return yaml.safe_load(self.stdout_str) except Exception: - tmpl = ( - " stdout is not valid {fmt}:\n" - '{{stdout!r}}\n'.format( - fmt=fmt)) + tmpl = " stdout is not valid {fmt}:\n" "{{stdout!r}}\n".format(fmt=fmt) logger.exception(self.cmd + tmpl.format(stdout=self.stdout_str)) # pylint: disable=logging-not-lazy - raise exceptions.DeserializeValueError( - self.cmd + tmpl.format(stdout=self.stdout_brief) - ) - msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt) + raise exceptions.DeserializeValueError(self.cmd + tmpl.format(stdout=self.stdout_brief)) + msg = "{fmt} deserialize target is not implemented".format(fmt=fmt) logger.error(msg) raise NotImplementedError(msg) @@ -385,7 +380,7 @@ def stdout_json(self) -> typing.Any: :rtype: typing.Any """ with self.lock: - return self.__deserialize(fmt='json') + return self.__deserialize(fmt="json") @property def stdout_yaml(self) -> typing.Any: @@ -394,16 +389,24 @@ def stdout_yaml(self) -> typing.Any: :rtype: typing.Any """ with self.lock: - return self.__deserialize(fmt='yaml') + return self.__deserialize(fmt="yaml") def __dir__(self) -> typing.List[str]: """Override dir for IDE and as source for getitem checks.""" return [ - 'cmd', 'stdout', 'stderr', 'exit_code', - 'stdout_bin', 'stderr_bin', - 'stdout_str', 'stderr_str', 'stdout_brief', 'stderr_brief', - 'stdout_json', 'stdout_yaml', - 'lock' + "cmd", + "stdout", + "stderr", + "exit_code", + "stdout_bin", + "stderr_bin", + "stdout_str", + "stderr_str", + "stdout_brief", + "stderr_brief", + "stdout_json", + "stdout_yaml", + "lock", ] def __getitem__(self, item: str) -> typing.Any: @@ -417,23 +420,14 @@ def __getitem__(self, item: str) -> typing.Any: """ if item in dir(self): return getattr(self, item) - raise IndexError( - '"{item}" not found in {dir}'.format( - item=item, dir=dir(self) - ) - ) + raise IndexError('"{item}" not found in {dir}'.format(item=item, dir=dir(self))) def __repr__(self) -> str: """Representation for debugging.""" return ( - '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, ' - 'exit_code={exit_code!s})'.format( - cls=self.__class__.__name__, - cmd=self.cmd, - stdout=self.stdout, - stderr=self.stderr, - exit_code=self.exit_code - )) + "{cls}(cmd={self.cmd!r}, stdout={self.stdout}, stderr={self.stderr}, " + "exit_code={self.exit_code!s})".format(cls=self.__class__.__name__, self=self) + ) def __str__(self) -> str: """Representation for logging.""" @@ -441,12 +435,12 @@ def __str__(self) -> str: "{cls}(\n\tcmd={cmd!r}," "\n\t stdout=\n'{stdout_brief}'," "\n\tstderr=\n'{stderr_brief}', " - '\n\texit_code={exit_code!s}\n)'.format( + "\n\texit_code={exit_code!s}\n)".format( cls=self.__class__.__name__, cmd=self.cmd, stdout_brief=self.stdout_brief, stderr_brief=self.stderr_brief, - exit_code=self.exit_code + exit_code=self.exit_code, ) ) @@ -460,8 +454,4 @@ def __ne__(self, other: typing.Any) -> bool: def __hash__(self) -> int: """Hash for usage as dict key and in sets.""" - return hash( - ( - self.__class__, self.cmd, self.stdin, self.stdout, self.stderr, - self.exit_code - )) + return hash((self.__class__, self.cmd, self.stdin, self.stdout, self.stderr, self.exit_code)) diff --git a/exec_helpers/proc_enums.py b/exec_helpers/proc_enums.py index dd72f21..27659d0 100644 --- a/exec_helpers/proc_enums.py +++ b/exec_helpers/proc_enums.py @@ -22,12 +22,7 @@ import enum import typing -__all__ = ( - 'SigNum', - 'ExitCodes', - 'exit_code_to_enum', - 'exit_codes_to_enums', -) +__all__ = ("SigNum", "ExitCodes", "exit_code_to_enum", "exit_codes_to_enums") @enum.unique @@ -68,10 +63,7 @@ class SigNum(enum.IntEnum): def __str__(self) -> str: """Representation for logs.""" - return "{name}<{value:d}(0x{value:02X})>".format( # pragma: no cover - name=self.name, - value=self.value - ) + return "{self.name}<{self.value:d}(0x{self.value:02X})>".format(self=self) # pragma: no cover @enum.unique @@ -139,10 +131,7 @@ class ExitCodes(int, enum.Enum): def __str__(self) -> str: """Representation for logs.""" - return "{name}<{value:d}(0x{value:02X})>".format( - name=self.name, - value=self.value - ) + return "{self.name}<{self.value:d}(0x{self.value:02X})>".format(self=self) def exit_code_to_enum(code: typing.Union[int, ExitCodes]) -> typing.Union[int, ExitCodes]: diff --git a/exec_helpers/ssh_auth.py b/exec_helpers/ssh_auth.py index 95e97ee..f502f05 100644 --- a/exec_helpers/ssh_auth.py +++ b/exec_helpers/ssh_auth.py @@ -22,20 +22,17 @@ import paramiko # type: ignore -__all__ = ('SSHAuth', ) +__all__ = ("SSHAuth",) logger = logging.getLogger(__name__) -logging.getLogger('paramiko').setLevel(logging.WARNING) -logging.getLogger('iso8601').setLevel(logging.WARNING) +logging.getLogger("paramiko").setLevel(logging.WARNING) +logging.getLogger("iso8601").setLevel(logging.WARNING) class SSHAuth: """SSH Authorization object.""" - __slots__ = ( - '__username', '__password', '__key', '__keys', - '__key_filename', '__passphrase' - ) + __slots__ = ("__username", "__password", "__key", "__keys", "__key_filename", "__passphrase") def __init__( self, @@ -91,14 +88,14 @@ def username(self) -> typing.Optional[str]: return self.__username @staticmethod - def __get_public_key(key: typing.Union[paramiko.RSAKey, None])-> typing.Optional[str]: + def __get_public_key(key: typing.Union[paramiko.RSAKey, None]) -> typing.Optional[str]: """Internal method for get public key from private. :type key: paramiko.RSAKey """ if key is None: return None - return '{0} {1}'.format(key.get_name(), key.get_base64()) + return "{0} {1}".format(key.get_name(), key.get_base64()) @property def public_key(self) -> typing.Optional[str]: @@ -125,7 +122,7 @@ def enter_password(self, tgt: typing.IO) -> None: :type tgt: file """ # noinspection PyTypeChecker - tgt.write('{}\n'.format(self.__password)) + tgt.write("{}\n".format(self.__password)) def connect( self, @@ -147,64 +144,56 @@ def connect( :raises PasswordRequiredException: No password has been set, but required. :raises AuthenticationException: Authentication failed. """ - kwargs = { - 'username': self.username, - 'password': self.__password, - } # type: typing.Dict[str, typing.Any] + kwargs = {"username": self.username, "password": self.__password} # type: typing.Dict[str, typing.Any] if hostname is not None: - kwargs['hostname'] = hostname - kwargs['port'] = port + kwargs["hostname"] = hostname + kwargs["port"] = port if isinstance(client, paramiko.client.SSHClient): # pragma: no cover # paramiko.transport.Transport still do not allow passphrase and key filename if self.key_filename is not None: - kwargs['key_filename'] = self.key_filename + kwargs["key_filename"] = self.key_filename if self.__passphrase is not None: - kwargs['passphrase'] = self.__passphrase + kwargs["passphrase"] = self.__passphrase keys = [self.__key] keys.extend([k for k in self.__keys if k != self.__key]) for key in keys: - kwargs['pkey'] = key + kwargs["pkey"] = key try: client.connect(**kwargs) if self.__key != key: self.__key = key - logger.debug( - 'Main key has been updated, public key is: \n' - '{}'.format(self.public_key)) + logger.debug("Main key has been updated, public key is: \n{}".format(self.public_key)) return except paramiko.PasswordRequiredException: if self.__password is None: - logger.exception('No password has been set!') + logger.exception("No password has been set!") raise else: - logger.critical('Unexpected PasswordRequiredException, when password is set!') + logger.critical("Unexpected PasswordRequiredException, when password is set!") raise - except (paramiko.AuthenticationException, - paramiko.BadHostKeyException): + except (paramiko.AuthenticationException, paramiko.BadHostKeyException): continue - msg = 'Connection using stored authentication info failed!' + msg = "Connection using stored authentication info failed!" if log: logger.exception(msg) raise paramiko.AuthenticationException(msg) def __hash__(self) -> int: """Hash for usage as dict keys and comparison.""" - return hash(( - self.__class__, - self.username, - self.__password, - tuple(self.__keys), + return hash( ( - tuple(self.key_filename) - if isinstance(self.key_filename, list) - else self.key_filename - ), - self.__passphrase - )) + self.__class__, + self.username, + self.__password, + tuple(self.__keys), + (tuple(self.key_filename) if isinstance(self.key_filename, list) else self.key_filename), + self.__passphrase, + ) + ) def __eq__(self, other: typing.Any) -> bool: """Comparison helper.""" @@ -214,59 +203,39 @@ def __ne__(self, other: typing.Any) -> bool: """Comparison helper.""" return not self.__eq__(other) - def __deepcopy__(self, memo: typing.Any) -> 'SSHAuth': + def __deepcopy__(self, memo: typing.Any) -> "SSHAuth": """Helper for copy.deepcopy.""" return self.__class__( # type: ignore - username=self.username, - password=self.__password, - key=self.__key, - keys=copy.deepcopy(self.__keys) + username=self.username, password=self.__password, key=self.__key, keys=copy.deepcopy(self.__keys) ) - def __copy__(self) -> 'SSHAuth': + def __copy__(self) -> "SSHAuth": """Copy self.""" return self.__class__( # type: ignore - username=self.username, - password=self.__password, - key=self.__key, - keys=self.__keys + username=self.username, password=self.__password, key=self.__key, keys=self.__keys ) def __repr__(self) -> str: """Representation for debug purposes.""" - _key = ( - None if self.__key is None else - ''.format(self.public_key) - ) + _key = None if self.__key is None else "".format(self.public_key) _keys = [] # type: typing.List[typing.Union[str, None]] for k in self.__keys: if k == self.__key: continue # noinspection PyTypeChecker - _keys.append( - ''.format( - self.__get_public_key(key=k)) if k is not None else None) + _keys.append("".format(self.__get_public_key(key=k)) if k is not None else None) return ( - '{cls}(' - 'username={self.username!r}, ' - 'password=<*masked*>, ' - 'key={key}, ' - 'keys={keys}, ' - 'key_filename={self.key_filename!r}, ' - 'passphrase=<*masked*>,' - ')'.format( - cls=self.__class__.__name__, - self=self, - key=_key, - keys=_keys) + "{cls}(" + "username={self.username!r}, " + "password=<*masked*>, " + "key={key}, " + "keys={keys}, " + "key_filename={self.key_filename!r}, " + "passphrase=<*masked*>," + ")".format(cls=self.__class__.__name__, self=self, key=_key, keys=_keys) ) def __str__(self) -> str: """Representation for debug purposes.""" - return ( - '{cls} for {username}'.format( - cls=self.__class__.__name__, - username=self.username, - ) - ) + return "{cls} for {self.username}".format(cls=self.__class__.__name__, self=self) diff --git a/exec_helpers/ssh_client.py b/exec_helpers/ssh_client.py index 2575b6e..8a219bf 100644 --- a/exec_helpers/ssh_client.py +++ b/exec_helpers/ssh_client.py @@ -23,10 +23,10 @@ from ._ssh_client_base import SSHClientBase -__all__ = ('SSHClient', ) +__all__ = ("SSHClient",) logger = logging.getLogger(__name__) -logging.getLogger('paramiko').setLevel(logging.WARNING) +logging.getLogger("paramiko").setLevel(logging.WARNING) class SSHClient(SSHClientBase): @@ -37,7 +37,7 @@ class SSHClient(SSHClientBase): @staticmethod def _path_esc(path: str) -> str: """Escape space character in the path.""" - return path.replace(' ', '\ ') + return path.replace(" ", "\ ") def mkdir(self, path: str) -> None: """Run 'mkdir -p path' on remote. @@ -74,10 +74,7 @@ def upload(self, source: str, target: str) -> None: return for rootdir, _, files in os.walk(source): - targetdir = os.path.normpath( - os.path.join( - target, - os.path.relpath(rootdir, source))).replace("\\", "/") + targetdir = os.path.normpath(os.path.join(target, os.path.relpath(rootdir, source))).replace("\\", "/") self.mkdir(targetdir) @@ -98,10 +95,7 @@ def download(self, destination: str, target: str) -> bool: :return: downloaded file present on local filesystem :rtype: bool """ - self.logger.debug( - "Copying '%s' -> '%s' from remote to local host", - destination, target - ) + self.logger.debug("Copying '%s' -> '%s' from remote to local host", destination, target) if os.path.isdir(target): target = posixpath.join(target, os.path.basename(destination)) diff --git a/exec_helpers/subprocess_runner.py b/exec_helpers/subprocess_runner.py index f8cda74..16c6ca8 100644 --- a/exec_helpers/subprocess_runner.py +++ b/exec_helpers/subprocess_runner.py @@ -16,10 +16,7 @@ """Python subprocess.Popen wrapper.""" -__all__ = ( - 'Subprocess', - 'SubprocessExecuteAsyncResult' -) +__all__ = ("Subprocess", "SubprocessExecuteAsyncResult") import abc import collections @@ -59,7 +56,7 @@ def stderr(self) -> typing.Optional[typing.IO]: # type: ignore return super(SubprocessExecuteAsyncResult, self).stderr @property - def stdout(self) -> typing.Optional[typing.IO]: # type: ignore + def stdout(self) -> typing.Optional[typing.IO]: # type: ignore """Override original NamedTuple with proper typing.""" return super(SubprocessExecuteAsyncResult, self).stdout @@ -73,7 +70,7 @@ class SingletonMeta(abc.ABCMeta): _instances = {} # type: typing.Dict[typing.Type, typing.Any] _lock = threading.RLock() # type: threading.RLock - def __call__(cls: 'SingletonMeta', *args: typing.Any, **kwargs: typing.Any) -> typing.Any: + def __call__(cls: "SingletonMeta", *args: typing.Any, **kwargs: typing.Any) -> typing.Any: """Singleton.""" with cls._lock: if cls not in cls._instances: @@ -83,10 +80,7 @@ def __call__(cls: 'SingletonMeta', *args: typing.Any, **kwargs: typing.Any) -> t @classmethod def __prepare__( # pylint: disable=unused-argument - mcs: typing.Type['SingletonMeta'], - name: str, - bases: typing.Iterable[typing.Type], - **kwargs: typing.Any + mcs: typing.Type["SingletonMeta"], name: str, bases: typing.Iterable[typing.Type], **kwargs: typing.Any ) -> collections.OrderedDict: """Metaclass magic for object storage. @@ -149,23 +143,16 @@ def _exec_command( .. versionadded:: 1.2.0 """ + @threaded.threadpooled # type: ignore def poll_stdout() -> None: """Sync stdout poll.""" - result.read_stdout( - src=stdout, - log=logger, - verbose=verbose - ) + result.read_stdout(src=stdout, log=logger, verbose=verbose) @threaded.threadpooled # type: ignore def poll_stderr() -> None: """Sync stderr poll.""" - result.read_stderr( - src=stderr, - log=logger, - verbose=verbose - ) + result.read_stderr(src=stderr, log=logger, verbose=verbose) # Store command with hidden data cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) @@ -254,8 +241,7 @@ def execute_async( cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re) self.logger.log( # type: ignore - level=logging.INFO if verbose else logging.DEBUG, - msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) + level=logging.INFO if verbose else logging.DEBUG, msg=_log_templates.CMD_EXEC.format(cmd=cmd_for_log) ) process = subprocess.Popen( @@ -264,8 +250,8 @@ def execute_async( stderr=subprocess.PIPE if open_stderr else subprocess.DEVNULL, stdin=subprocess.PIPE, shell=True, - cwd=kwargs.get('cwd', None), - env=kwargs.get('env', None), + cwd=kwargs.get("cwd", None), + env=kwargs.get("env", None), universal_newlines=False, ) @@ -273,7 +259,7 @@ def execute_async( process_stdin = process.stdin else: if isinstance(stdin, str): - stdin = stdin.encode(encoding='utf-8') + stdin = stdin.encode(encoding="utf-8") elif isinstance(stdin, bytearray): stdin = bytes(stdin) try: @@ -283,9 +269,9 @@ def execute_async( # bpo-19612, bpo-30418: On Windows, stdin.write() fails # with EINVAL if the child process exited or if the child # process is still running but closed the pipe. - self.logger.warning('STDIN Send failed: closed PIPE') + self.logger.warning("STDIN Send failed: closed PIPE") elif exc.errno in (errno.EPIPE, errno.ESHUTDOWN): # pragma: no cover - self.logger.warning('STDIN Send failed: broken PIPE') + self.logger.warning("STDIN Send failed: broken PIPE") else: process.kill() raise diff --git a/pyproject.toml b/pyproject.toml index 1c48e2c..708fb53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,7 @@ requires = [ "setuptools >= 21.0.0,!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0", # PSF/ZPL "wheel", ] + +[tool.black] +line-length = 120 +safe = true diff --git a/setup.cfg b/setup.cfg index c678ea4..844ab05 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,13 +39,23 @@ exclude = __init__.py, docs ignore = + E203 +# whitespace before ':' show-pep8 = True show-source = True count = True max-line-length = 120 [pydocstyle] -ignore = D401, D203, D213 +ignore = + D401, + D202, + D203, + D213 +# First line should be in imperative mood; try rephrasing +# No blank lines allowed after function docstring +# 1 blank line required before class docstring +# Multi-line docstring summary should start at the second line [aliases] test=pytest diff --git a/tox.ini b/tox.ini index 8f6ad67..9d58963 100644 --- a/tox.ini +++ b/tox.ini @@ -103,13 +103,23 @@ exclude = __init__.py, docs ignore = + E203 +# whitespace before ':' show-pep8 = True show-source = True count = True max-line-length = 120 [pydocstyle] -ignore = D401, D203, D213 +ignore = + D401, + D202, + D203, + D213 +# First line should be in imperative mood; try rephrasing +# No blank lines allowed after function docstring +# 1 blank line required before class docstring +# Multi-line docstring summary should start at the second line [testenv:docs] deps = @@ -120,6 +130,13 @@ commands = python setup.py build_sphinx deps = bandit commands = bandit -r exec_helpers +[testenv:black] +deps = + black +usedevelop = False +commands = + black exec_helpers + [testenv:dep-graph] deps = . @@ -128,6 +145,6 @@ commands = pipdeptree [testenv:mypy] deps = - mypy>=0.620 + mypy>=0.630 -r{toxinidir}/CI_REQUIREMENTS.txt commands = mypy --strict exec_helpers