Rework exceptions

1. Allow substitute `CalledProcessError`
2. Modify exceptions class hierarchy
3. Expose **kwargs in stable places
4. Temporary disable black formatter: due to bug it's destroy py35

Signed-off-by: Alexey Stepanov <penguinolog@gmail.com>

.pylintrc

@@ -137,7 +137,6 @@ enable=old-style-class,
        old-ne-operator,
        old-octal-literal,
        import-star-module-level,
-       c-extension-no-member,
        lowercase-l-suffix,
        deprecated-module,
        invalid-encoded-data,

.travis.yml

@@ -99,13 +99,13 @@ jobs:
     - pip install --upgrade pydocstyle
     script:
     - pydocstyle exec_helpers
-  - <<: *code_style_check
-    name: "Black formatting"
-    install:
-    - *upgrade_python_toolset
-    - pip install --upgrade black
-    script:
-    - black --check exec_helpers
+#  - <<: *code_style_check
+#    name: "Black formatting"
+#    install:
+#    - *upgrade_python_toolset
+#    - pip install --upgrade black
+#    script:
+#    - black --check exec_helpers
 
   - stage: test
     <<: *python35

MANIFEST.in

@@ -2,9 +2,10 @@ include *.rst LICENSE requirements.txt
 global-exclude *.c
 exclude Makefile
 prune tools
-exclude .travis.yml appveyor.yml
+exclude .travis.yml appveyor.yml azure-pipelines.yml
 exclude tox.ini pytest.ini .coveragerc
 prune test
 prune .github
+prune .azure_pipelines
 prune docs
 exclude CODEOWNERS CODE_OF_CONDUCT.md _config.yml

README.rst

@@ -149,6 +149,8 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif
         error_info=None,  # type: typing.Optional[str]
         expected=None,  # type: typing.Optional[typing.Iterable[int]]
         raise_on_err=True,  # type: bool
+        # Keyword only:
+        exception_class=CalledProcessError,  # typing.Type[CalledProcessError]
         **kwargs
     )
 
@@ -160,6 +162,9 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
         error_info=None,  # type: typing.Optional[str]
         raise_on_err=True,  # type: bool
+        # Keyword only:
+        expected=None,  # typing.Optional[typing.Iterable[typing.Union[int, ExitCodes]]]
+        exception_class=CalledProcessError,  # typing.Type[CalledProcessError]
     )
 
 If no STDOUT or STDERR required, it is possible to disable this FIFO pipes via `**kwargs` with flags `open_stdout=False` and `open_stderr=False`.
@@ -208,7 +213,7 @@ SSHClient specific
 ------------------
 
 SSHClient commands support get_pty flag, which enables PTY open on remote side.
-PTY width and height can be set via kwargs, dimensions in pixels are always 0x0.
+PTY width and height can be set via keyword arguments, dimensions in pixels are always 0x0.
 
 Possible to call commands in parallel on multiple hosts if it's not produce huge output:
 
@@ -219,7 +224,9 @@ Possible to call commands in parallel on multiple hosts if it's not produce huge
         command,  # type: str
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
         expected=None,  # type: typing.Optional[typing.Iterable[int]]
-        raise_on_err=True  # type: bool
+        raise_on_err=True,  # type: bool
+        # Keyword only:
+        exception_class=ParallelCallProcessError  # typing.Type[ParallelCallProcessError]
     )
     results  # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult]
 
@@ -237,7 +244,10 @@ For execute through SSH host can be used `execute_through_host` method:
         target_port=22,  # type: int
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
         verbose=False,  # type: bool
+        # Keyword only:
         get_pty=False,  # type: bool
+        width=80,  # type: int
+        height=24  # type: int
     )
 
 Where hostname is a target hostname, auth is an alternate credentials for target host.
@@ -312,7 +322,7 @@ Additional (non-standard) helpers:
 
 Subprocess specific
 -------------------
-Kwargs set properties:
+Keyword arguments:
 
 - cwd - working directory.
 - env - environment variables dict.

doc/source/SSHClient.rst

@@ -121,7 +121,7 @@ API: SSHClient and SSHAuth.
         .. Note:: Enter and exit ssh context manager is produced as well.
         .. versionadded:: 1.2.1
 
-    .. py:method:: execute_async(command, stdin=None, open_stdout=True, open_stderr=True, verbose=False, log_mask_re=None, **kwargs)
+    .. py:method:: execute_async(command, stdin=None, open_stdout=True, open_stderr=True, verbose=False, log_mask_re=None, *, get_pty=False, width=80, height=24, **kwargs)
 
         Execute command in async mode and return channel with IO objects.
 
@@ -138,12 +138,19 @@ API: SSHClient and SSHAuth.
         :param log_mask_re: regex lookup rule to mask command for logger.
                             all MATCHED groups will be replaced by '<*masked*>'
         :type log_mask_re: typing.Optional[str]
+        :param get_pty: Get PTY for connection
+        :type get_pty: bool
+        :param width: PTY width
+        :type width: int
+        :param height: PTY height
+        :type height: int
         :rtype: SshExecuteAsyncResult
 
         .. versionchanged:: 1.2.0 open_stdout and open_stderr flags
         .. versionchanged:: 1.2.0 stdin data
         .. versionchanged:: 1.2.0 get_pty moved to `**kwargs`
         .. versionchanged:: 2.1.0 Use typed NamedTuple as result
+        .. versionchanged:: 3.2.0 Expose pty options as optional keyword-only arguments
 
     .. py:method:: execute(command, verbose=False, timeout=1*60*60, **kwargs)
 
@@ -160,7 +167,7 @@ API: SSHClient and SSHAuth.
 
         .. versionchanged:: 1.2.0 default timeout 1 hour
 
-    .. py:method:: check_call(command, verbose=False, timeout=1*60*60, error_info=None, expected=None, raise_on_err=True, **kwargs)
+    .. py:method:: check_call(command, verbose=False, timeout=1*60*60, error_info=None, expected=None, raise_on_err=True, *, exception_class=CalledProcessError, **kwargs)
 
         Execute command and check for return code.
 
@@ -176,13 +183,16 @@ API: SSHClient and SSHAuth.
         :type expected: ``typing.Optional[typing.Iterable[int]]``
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param exception_class: Exception class for errors. Subclass of CalledProcessError is mandatory.
+        :type exception_class: typing.Type[CalledProcessError]
         :rtype: ExecResult
         :raises ExecHelperTimeoutError: Timeout exceeded
         :raises CalledProcessError: Unexpected exit code
 
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
-    .. py:method:: check_stderr(command, verbose=False, timeout=1*60*60, error_info=None, raise_on_err=True, **kwargs)
+    .. py:method:: check_stderr(command, verbose=False, timeout=1*60*60, error_info=None, raise_on_err=True, *, expected=None, exception_class=CalledProcessError, **kwargs)
 
         Execute command expecting return code 0 and empty STDERR.
 
@@ -196,14 +206,18 @@ API: SSHClient and SSHAuth.
         :type error_info: ``typing.Optional[str]``
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param expected: expected return codes (0 by default)
+        :type expected: typing.Optional[typing.Iterable[typing.Union[int, ExitCodes]]]
+        :param exception_class: Exception class for errors. Subclass of CalledProcessError is mandatory.
+        :type exception_class: typing.Type[CalledProcessError]
         :rtype: ExecResult
         :raises ExecHelperTimeoutError: Timeout exceeded
         :raises CalledProcessError: Unexpected exit code or stderr presents
 
-        .. note:: expected return codes can be overridden via kwargs.
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
-    .. py:method:: execute_through_host(hostname, command, auth=None, target_port=22, verbose=False, timeout=1*60*60, get_pty=False, **kwargs)
+    .. py:method:: execute_through_host(hostname, command, auth=None, target_port=22, verbose=False, timeout=1*60*60, *, get_pty=False, width=80, height=24, **kwargs)
 
         Execute command on remote host through currently connected host.
 
@@ -221,12 +235,18 @@ API: SSHClient and SSHAuth.
         :type timeout: ``typing.Union[int, float, None]``
         :param get_pty: open PTY on target machine
         :type get_pty: ``bool``
+        :param width: PTY width
+        :type width: int
+        :param height: PTY height
+        :type height: int
         :rtype: ExecResult
         :raises ExecHelperTimeoutError: Timeout exceeded
 
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Expose pty options as optional keyword-only arguments
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
-    .. py:classmethod:: execute_together(remotes, command, timeout=1*60*60, expected=None, raise_on_err=True, **kwargs)
+    .. py:classmethod:: execute_together(remotes, command, timeout=1*60*60, expected=None, raise_on_err=True, *, exception_class=ParallelCallProcessError, **kwargs)
 
         Execute command on multiple remotes in async mode.
 
@@ -240,12 +260,15 @@ API: SSHClient and SSHAuth.
         :type expected: ``typing.Optional[typing.Iterable[]]``
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param exception_class: Exception to raise on error. Mandatory subclass of ParallelCallProcessError
+        :type exception_class: typing.Type[ParallelCallProcessError]
         :return: dictionary {(hostname, port): result}
         :rtype: typing.Dict[typing.Tuple[str, int], ExecResult]
         :raises ParallelCallProcessError: Unexpected any code at lest on one target
         :raises ParallelCallExceptions: At lest one exception raised during execution (including timeout)
 
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
     .. py:method:: open(path, mode='r')
 

doc/source/Subprocess.rst

@@ -8,15 +8,18 @@ API: Subprocess
 
 .. py:class:: Subprocess()
 
-    .. py:method:: __init__(logger, log_mask_re=None)
+    .. py:method:: __init__(logger, log_mask_re=None, *, logger=logging.getLogger("exec_helpers.subprocess_runner"))
 
         ExecHelper global API.
 
         :param log_mask_re: regex lookup rule to mask command for logger. all MATCHED groups will be replaced by '<*masked*>'
         :type log_mask_re: typing.Optional[str]
+        :param logger: logger instance to use
+        :type logger: logging.Logger
 
         .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd
         .. versionchanged:: 3.1.0 Not singleton anymore. Only lock is shared between all instances.
+        .. versionchanged:: 3.2.0 Logger can be enforced.
 
     .. py:attribute:: log_mask_re
 
@@ -40,7 +43,7 @@ API: Subprocess
 
         .. versionchanged:: 1.1.0 release lock on exit
 
-    .. py:method:: execute_async(command, stdin=None, open_stdout=True, open_stderr=True, verbose=False, log_mask_re=None, **kwargs)
+    .. py:method:: execute_async(command, stdin=None, open_stdout=True, open_stderr=True, verbose=False, log_mask_re=None, *, cwd=None, env=None, **kwargs)
 
         Execute command in async mode and return Popen with IO objects.
 
@@ -57,11 +60,16 @@ API: Subprocess
         :param log_mask_re: regex lookup rule to mask command for logger.
                             all MATCHED groups will be replaced by '<*masked*>'
         :type log_mask_re: ``typing.Optional[str]``
+        :param cwd: Sets the current directory before the child is executed.
+        :type cwd: typing.Optional[typing.Union[str, bytes]]
+        :param env: Defines the environment variables for the new process.
+        :type env: typing.Optional[typing.Mapping[typing.Union[str, bytes], typing.Union[str, bytes]]]
         :rtype: SubprocessExecuteAsyncResult
         :raises OSError: impossible to process STDIN
 
         .. versionadded:: 1.2.0
         .. versionchanged:: 2.1.0 Use typed NamedTuple as result
+        .. versionchanged:: 3.2.0 Expose cwd and env as optional keyword-only arguments
 
     .. py:method:: execute(command, verbose=False, timeout=1*60*60, **kwargs)
 
@@ -82,7 +90,7 @@ API: Subprocess
         .. versionchanged:: 1.2.0 default timeout 1 hour
         .. versionchanged:: 1.2.0 stdin data
 
-    .. py:method:: check_call(command, verbose=False, timeout=1*60*60, error_info=None, expected=None, raise_on_err=True, **kwargs)
+    .. py:method:: check_call(command, verbose=False, timeout=1*60*60, error_info=None, expected=None, raise_on_err=True, *, exception_class=CalledProcessError, **kwargs)
 
         Execute command and check for return code.
 
@@ -98,14 +106,17 @@ API: Subprocess
         :type expected: ``typing.Optional[typing.Iterable[int]]``
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param exception_class: Exception class for errors. Subclass of CalledProcessError is mandatory.
+        :type exception_class: typing.Type[CalledProcessError]
         :rtype: ExecResult
         :raises ExecHelperTimeoutError: Timeout exceeded
         :raises CalledProcessError: Unexpected exit code
 
         .. versionchanged:: 1.1.0 make method
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
-    .. py:method:: check_stderr(command, verbose=False, timeout=1*60*60, error_info=None, raise_on_err=True, **kwargs)
+    .. py:method:: check_stderr(command, verbose=False, timeout=1*60*60, error_info=None, raise_on_err=True, *, expected=None, exception_class=CalledProcessError, **kwargs)
 
         Execute command expecting return code 0 and empty STDERR.
 
@@ -119,14 +130,17 @@ API: Subprocess
         :type error_info: ``typing.Optional[str]``
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param expected: expected return codes (0 by default)
+        :type expected: typing.Optional[typing.Iterable[typing.Union[int, ExitCodes]]]
+        :param exception_class: Exception class for errors. Subclass of CalledProcessError is mandatory.
+        :type exception_class: typing.Type[CalledProcessError]
         :rtype: ExecResult
         :raises ExecHelperTimeoutError: Timeout exceeded
         :raises CalledProcessError: Unexpected exit code or stderr presents
 
-        .. note:: expected return codes can be overridden via kwargs.
-
         .. versionchanged:: 1.1.0 make method
         .. versionchanged:: 1.2.0 default timeout 1 hour
+        .. versionchanged:: 3.2.0 Exception class can be substituted
 
 
 .. py:class:: SubprocessExecuteAsyncResult

doc/source/exceptions.rst

@@ -100,16 +100,14 @@ API: exceptions
         ``str``
         stdout string or brief string
 
-.. py:exception:: ParallelCallExceptions(ExecCalledProcessError)
+.. py:exception:: ParallelCallProcessError(ExecCalledProcessError)
 
-    Exception raised during parallel call as result of exceptions.
+    Exception during parallel execution.
 
-    .. py:method:: __init__(command, exceptions, errors, results, expected=None, )
+    .. py:method:: __init__(command, errors, results, expected=None, )
 
         :param command: command
         :type command: ``str``
-        :param exceptions: Exception on connections
-        :type exceptions: ``typing.Dict[typing.Tuple[str, int], Exception]``
         :param errors: results with errors
         :type errors: typing.Dict[typing.Tuple[str, int], ExecResult]
         :param results: all results
@@ -124,11 +122,6 @@ API: exceptions
         ``str``
         command
 
-    .. py:attribute:: exceptions
-
-        ``typing.Dict[typing.Tuple[str, int], Exception]``
-        Exception on connections
-
     .. py:attribute:: errors
 
         results with errors
@@ -147,14 +140,16 @@ API: exceptions
 
         :rtype: typing.List[typing.Union[int, ExitCodes]]
 
-.. py:exception:: ParallelCallProcessError(ExecCalledProcessError)
+.. py:exception:: ParallelCallExceptions(ParallelCallProcessError)
 
-    Exception during parallel execution.
+    Exception raised during parallel call as result of exceptions.
 
-    .. py:method:: __init__(command, errors, results, expected=None, )
+    .. py:method:: __init__(command, exceptions, errors, results, expected=None, )
 
         :param command: command
         :type command: ``str``
+        :param exceptions: Exception on connections
+        :type exceptions: ``typing.Dict[typing.Tuple[str, int], Exception]``
         :param errors: results with errors
         :type errors: typing.Dict[typing.Tuple[str, int], ExecResult]
         :param results: all results
@@ -169,6 +164,11 @@ API: exceptions
         ``str``
         command
 
+    .. py:attribute:: exceptions
+
+        ``typing.Dict[typing.Tuple[str, int], Exception]``
+        Exception on connections
+
     .. py:attribute:: errors
 
         results with errors

exec_helpers/__init__.py

@@ -58,6 +58,7 @@
 except pkg_resources.DistributionNotFound:
     # package is not installed, try to get from SCM
     try:
+        # noinspection PyPackageRequirements,PyUnresolvedReferences
         import setuptools_scm  # type: ignore
 
         __version__ = setuptools_scm.get_version()