Move forward: split python 3.5 and actual pythons

Early split python 3.5 due to useful features in python 3.6 (typing) * f strings (faster) * type annotations for calls (already helped to find 1 issue)
python-useful-helpers · Dec 13, 2018 · c43f92f · c43f92f
1 parent b3dc162
commit c43f92f
Show file tree

Hide file tree

Showing 37 changed files with 457 additions and 499 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,20 +8,14 @@ install:
 - pip install --upgrade pytest-cov coveralls
 
 _python:
-- &python35
-  name: "Python 3.5"
-  python: 3.5
 - &python36
   name: "Python 3.6"
-  python: 3.6
+  python: "3.6"
 - &python37
   name: "Python 3.7"
-  python: 3.7
+  python: "3.7"
   dist: xenial
   sudo: true
-- &pypy3
-  name: "PyPy3"
-  python: pypy3.5
 
 _helpers:
 - &install_cython pip install --upgrade Cython
@@ -99,25 +93,19 @@ jobs:
 #    - pip install --upgrade pydocstyle
 #    script:
 #    - pydocstyle -v exec_helpers
-#  - <<: *code_style_check
-#    name: "Black formatting"
-#    install:
-#    - *upgrade_python_toolset
-#    - pip install --upgrade black
-#    script:
-#    - black --check exec_helpers
+  - <<: *code_style_check
+    name: "Black formatting"
+    install:
+    - *upgrade_python_toolset
+    - pip install --upgrade black
+    script:
+    - black --check exec_helpers
 
-  - stage: test
-    <<: *python35
   - stage: test
     <<: *python36
   - stage: test
     <<: *python37
-  - stage: test
-    <<: *pypy3
 
-  - <<: *test_cythonized
-    <<: *python35
   - <<: *test_cythonized
     <<: *python36
   - <<: *test_cythonized
@@ -127,16 +115,16 @@ jobs:
     # This prevents job from appearing in test plan unless commit is tagged:
     if: tag IS present
     # Run on pypy to build not cythonized wheel
-    <<: *pypy3
+    <<: *python37
     name: Build universal and cythonized bdist_wheel. Deploy bdist and sdist.
     services:
     - docker
     install:
     - *upgrade_python_toolset
+    - *install_deps
     script:
     - ./tools/run_docker.sh "exec_helpers"
     before_deploy:
-    - pip install -r build_requirements.txt
     - *build_package
     deploy:
     - provider: pypi

diff --git a/README.rst b/README.rst
@@ -46,12 +46,10 @@ Pros:
 
 ::
 
-    Python 3.5
     Python 3.6
     Python 3.7
-    PyPy3 3.5+
 
-.. note:: For Python 2.7 and PyPy please use versions 1.x.x. For python 3.4 use versions 2.x.x
+.. note:: Old pythons: For Python 2.7 and PyPy use versions 1.x.x, python 3.4 use versions 2.x.x, python 3.5 and PyPy 3.5 use versions 3.x.x
 
 This package includes:
 
@@ -125,13 +123,13 @@ Base methods
 ------------
 Main methods are `execute`, `check_call` and `check_stderr` for simple executing, executing and checking return code
 and executing, checking return code and checking for empty stderr output.
-This methods are almost the same for `SSHCleint` and `Subprocess`, except specific flags.
+This methods are almost the same for `SSHClient` and `Subprocess`, except specific flags.
 
 .. note:: By default ALL methods have timeout 1 hour, infinite waiting can be enabled, but it's special case.
 
 .. code-block:: python
 
-    result = helper.execute(
+    result: ExecResult = helper.execute(
         command,  # type: str
         verbose=False,  # type: bool
         timeout=1 * 60 * 60,  # type: typing.Union[int, float, None]
@@ -141,7 +139,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif
 
 .. code-block:: python
 
-    result = helper.check_call(
+    result: ExecResult = helper.check_call(
         command,  # type: str
         verbose=False,  # type: bool
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
@@ -155,7 +153,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif
 
 .. code-block:: python
 
-    result = helper.check_stderr(
+    result: ExecResult = helper.check_stderr(
         command,  # type: str
         verbose=False,  # type: bool
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
@@ -168,7 +166,7 @@ This methods are almost the same for `SSHCleint` and `Subprocess`, except specif
 
 .. code-block:: python
 
-    result = helper(  # Lazy way: instances are callable and uses `execute`.
+    result: ExecResult = helper(  # Lazy way: instances are callable and uses `execute`.
         command,  # type: str
         verbose=False,  # type: bool
         timeout=1 * 60 * 60,  # type: typing.Union[int, float, None]
@@ -185,7 +183,7 @@ All regex matched groups will be replaced by `'<*masked*>'`.
 
 .. code-block:: python
 
-    result = helper.execute(
+    result: ExecResult = helper.execute(
         command="AUTH='top_secret_key'; run command",  # type: str
         verbose=False,  # type: bool
         timeout=1 * 60 * 60,  # type: typing.Optional[int]
@@ -227,13 +225,15 @@ Possible to call commands in parallel on multiple hosts if it's not produce huge
 
 .. code-block:: python
 
-    results = SSHClient.execute_together(
+    results: Dict[Tuple[str, int], ExecResult] = SSHClient.execute_together(
         remotes,  # type: typing.Iterable[SSHClient]
         command,  # type: str
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
         expected=(0,),  # type: typing.Iterable[typing.Union[int, ExitCodes]]
         raise_on_err=True,  # type: bool
         # Keyword only:
+        stdin=None,  # type: typing.Union[bytes, str, bytearray, None]
+        log_mask_re=None,  # type: typing.Optional[str]
         exception_class=ParallelCallProcessError  # typing.Type[ParallelCallProcessError]
     )
     results  # type: typing.Dict[typing.Tuple[str, int], exec_result.ExecResult]
@@ -245,14 +245,16 @@ For execute through SSH host can be used `execute_through_host` method:
 
 .. code-block:: python
 
-    result = client.execute_through_host(
+    result: ExecResult = client.execute_through_host(
         hostname,  # type: str
         command,  # type: str
         auth=None,  # type: typing.Optional[SSHAuth]
         target_port=22,  # type: int
         timeout=1 * 60 * 60,  # type: type: typing.Union[int, float, None]
         verbose=False,  # type: bool
         # Keyword only:
+        stdin=None,  # type: typing.Union[bytes, str, bytearray, None]
+        log_mask_re=None,  # type: typing.Optional[str]
         get_pty=False,  # type: bool
         width=80,  # type: int
         height=24  # type: int
@@ -347,7 +349,7 @@ Example:
 .. code-block:: python
 
     async with helper:
-      result = await helper.execute(
+      result: ExecResult = await helper.execute(
           command,  # type: str
           verbose=False,  # type: bool
           timeout=1 * 60 * 60,  # type: typing.Union[int, float, None]

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -2,6 +2,12 @@
 # Create and test a Python package on multiple Python versions.
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 # https://docs.microsoft.com/azure/devops/pipelines/languages/python
+trigger:
+- master
+- py35
+pr:
+- master
+- py35
 
 jobs:
 - job: 'PyLint'
@@ -58,17 +64,11 @@ jobs:
       testResultsFiles: ${{ format('$(System.DefaultWorkingDirectory)/mypy_result.xml') }}
       testRunTitle: 'MyPy'
 
-- template: .azure_pipelines/run_tests.yml
-  parameters: {name: 'Python_35', python: '3.5', architecture: 'x64', kind: 'native'}
 - template: .azure_pipelines/run_tests.yml
   parameters: {name: 'Python_36', python: '3.6', architecture: 'x64', kind: 'native'}
 - template: .azure_pipelines/run_tests.yml
   parameters: {name: 'Python_37', python: '3.7', architecture: 'x64', kind: 'native'}
 
-- template: .azure_pipelines/run_tests.yml
-  parameters: {name: 'Python_35', python: '3.5', architecture: 'x64', kind: 'cython'}
-- template: .azure_pipelines/run_tests.yml
-  parameters: {name: 'Python_35', python: '3.5', architecture: 'x86', kind: 'cython'}
 - template: .azure_pipelines/run_tests.yml
   parameters: {name: 'Python_36', python: '3.6', architecture: 'x64', kind: 'cython'}
 - template: .azure_pipelines/run_tests.yml
@@ -80,12 +80,9 @@ jobs:
 
 - job: 'Build_and_deploy'
   dependsOn:
-  - Python_35_x64_native
   - Python_36_x64_native
   - Python_37_x64_native
 
-  - Python_35_x64_cython
-  - Python_35_x86_cython
   - Python_36_x64_cython
   - Python_36_x86_cython
   - Python_37_x64_cython
@@ -96,12 +93,6 @@ jobs:
   strategy:
     maxParallel: 6
     matrix:
-      Python35_x64:
-        python.version: '3.5'
-        python.architecture: 'x64'
-      Python35_x86:
-        python.version: '3.5'
-        python.architecture: 'x86'
       Python36_x64:
         python.version: '3.6'
         python.architecture: 'x64'

diff --git a/doc/source/SSHClient.rst b/doc/source/SSHClient.rst
@@ -233,7 +233,7 @@ API: SSHClient and SSHAuth.
         .. versionchanged:: 1.2.0 default timeout 1 hour
         .. versionchanged:: 3.2.0 Exception class can be substituted
 
-    .. py:method:: execute_through_host(hostname, command, auth=None, target_port=22, verbose=False, timeout=1*60*60, *, get_pty=False, width=80, height=24, **kwargs)
+    .. py:method:: execute_through_host(hostname, command, auth=None, target_port=22, verbose=False, timeout=1*60*60, *, stdin=None, log_mask_re="", get_pty=False, width=80, height=24, **kwargs)
 
         Execute command on remote host through currently connected host.
 
@@ -249,6 +249,11 @@ API: SSHClient and SSHAuth.
         :type verbose: ``bool``
         :param timeout: Timeout for command execution.
         :type timeout: ``typing.Union[int, float, None]``
+        :param stdin: pass STDIN text to the process
+        :type stdin: typing.Union[bytes, str, bytearray, None]
+        :param log_mask_re: regex lookup rule to mask command for logger.
+                            all MATCHED groups will be replaced by '<*masked*>'
+        :type log_mask_re: typing.Optional[str]
         :param get_pty: open PTY on target machine
         :type get_pty: ``bool``
         :param width: PTY width
@@ -261,9 +266,9 @@ API: SSHClient and SSHAuth.
         .. versionchanged:: 1.2.0 default timeout 1 hour
         .. versionchanged:: 3.2.0 Expose pty options as optional keyword-only arguments
         .. versionchanged:: 3.2.0 Exception class can be substituted
-        .. versionchanged:: 3.4.0 Expected is not optional, defaults os dependent
+        .. versionchanged:: 4.0.0 Expose stdin and log_mask_re as optional keyword-only arguments
 
-    .. py:classmethod:: execute_together(remotes, command, timeout=1*60*60, expected=(0,), raise_on_err=True, *, exception_class=ParallelCallProcessError, **kwargs)
+    .. py:classmethod:: execute_together(remotes, command, timeout=1*60*60, expected=(0,), raise_on_err=True, *, stdin=None, log_mask_re="", exception_class=ParallelCallProcessError, **kwargs)
 
         Execute command on multiple remotes in async mode.
 
@@ -277,6 +282,11 @@ API: SSHClient and SSHAuth.
         :type expected: typing.Iterable[typing.Union[int, ExitCodes]]
         :param raise_on_err: Raise exception on unexpected return code
         :type raise_on_err: ``bool``
+        :param stdin: pass STDIN text to the process
+        :type stdin: typing.Union[bytes, str, bytearray, None]
+        :param log_mask_re: regex lookup rule to mask command for logger.
+                            all MATCHED groups will be replaced by '<*masked*>'
+        :type log_mask_re: typing.Optional[str]
         :param exception_class: Exception to raise on error. Mandatory subclass of ParallelCallProcessError
         :type exception_class: typing.Type[ParallelCallProcessError]
         :return: dictionary {(hostname, port): result}
@@ -287,6 +297,7 @@ API: SSHClient and SSHAuth.
         .. versionchanged:: 1.2.0 default timeout 1 hour
         .. versionchanged:: 3.2.0 Exception class can be substituted
         .. versionchanged:: 3.4.0 Expected is not optional, defaults os dependent
+        .. versionchanged:: 4.0.0 Expose stdin and log_mask_re as optional keyword-only arguments
 
     .. py:method:: open(path, mode='r')
 

diff --git a/exec_helpers/__init__.py b/exec_helpers/__init__.py
@@ -14,7 +14,24 @@
 
 """Execution helpers for simplified usage of subprocess and ssh."""
 
-import typing
+__all__ = (
+    "ExecHelperError",
+    "ExecCalledProcessError",
+    "CalledProcessError",
+    "ParallelCallExceptions",
+    "ParallelCallProcessError",
+    "ExecHelperNoKillError",
+    "ExecHelperTimeoutError",
+    "ExecHelper",
+    "SSHClient",
+    "SshExecuteAsyncResult",
+    "SSHAuth",
+    "Subprocess",
+    "SubprocessExecuteAsyncResult",
+    "ExitCodes",
+    "ExecResult",
+    "async_api",
+)
 
 import pkg_resources
 
@@ -38,27 +55,8 @@
 from .subprocess_runner import Subprocess, SubprocessExecuteAsyncResult  # nosec  # Expected
 from . import async_api
 
-__all__ = (
-    "ExecHelperError",
-    "ExecCalledProcessError",
-    "CalledProcessError",
-    "ParallelCallExceptions",
-    "ParallelCallProcessError",
-    "ExecHelperNoKillError",
-    "ExecHelperTimeoutError",
-    "ExecHelper",
-    "SSHClient",
-    "SshExecuteAsyncResult",
-    "SSHAuth",
-    "Subprocess",
-    "SubprocessExecuteAsyncResult",
-    "ExitCodes",
-    "ExecResult",
-    "async_api",
-)  # type: typing.Tuple[str, ...]
-
 try:  # pragma: no cover
-    __version__ = pkg_resources.get_distribution(__name__).version
+    __version__: str = pkg_resources.get_distribution(__name__).version
 except pkg_resources.DistributionNotFound:  # pragma: no cover
     # package is not installed, try to get from SCM
     try:

diff --git a/exec_helpers/_log_templates.py b/exec_helpers/_log_templates.py
@@ -16,17 +16,9 @@
 
 """Text templates for logging."""
 
-CMD_EXEC = "Executing command:\n{cmd!r}\n"
+CMD_EXEC: str = "Executing command:\n{cmd!r}\n"
 
-CMD_KILL_ERROR = (
-    "Wait for {result.cmd!r} during {timeout!s}s: no return code and no response on SIGTERM + SIGKILL signals!\n"
-    "\tSTDOUT:\n"
-    "{result.stdout_brief}\n"
-    "\tSTDERR:\n"
-    "{result.stderr_brief}"
-)
-
-CMD_WAIT_ERROR = (
+CMD_WAIT_ERROR: str = (
     "Wait for {result.cmd!r} during {timeout!s}s: no return code!\n"
     "\tSTDOUT:\n"
     "{result.stdout_brief}\n"