diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..f253fa32 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,141 @@ +version: 2.1 + +orbs: + python: circleci/python@0.3.2 + +jobs: + python_test: + parameters: + python_ver: + type: string + default: "3.6" + docker: + - image: circleci/python:<< parameters.python_ver >> + steps: + - checkout + - python/load-cache: + dependency-file: requirements_dev.txt + key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >> + - run: + name: Deps + command: | + sudo apt-get install openssh-server + - python/save-cache: + dependency-file: requirements_dev.txt + key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >> + - run: + command: | + pip install -U -r requirements_dev.txt + name: Build + - run: + command: | + eval "$(ssh-agent -s)" + pytest --cov-append --cov=pssh tests/test_imports.py tests/test_output.py tests/test_utils.py + pytest --reruns 5 --cov-append --cov=pssh tests/miko + pytest --reruns 10 --cov-append --cov=pssh tests/native/test_tunnel.py tests/native/test_agent.py + pytest --reruns 5 --cov-append --cov=pssh tests/native/test_*_client.py + pytest --reruns 5 --cov-append --cov=pssh tests/ssh + flake8 pssh + cd doc; make html; cd .. + # Test building from source distribution + python setup.py sdist + cd dist; pip install *; cd .. + python setup.py check --restructuredtext + name: Test + - run: + command: codecov + name: Coverage + + osx: + parameters: + xcode_ver: + type: string + default: "11.6.0" + macos: + xcode: << parameters.xcode_ver >> + environment: + HOMEBREW_NO_AUTO_UPDATE: 1 + steps: + - checkout + - run: + name: deps + command: | + pip3 install twine + which twine + - run: + name: Build Wheel + command: | + ./ci/osx-wheel.sh + - store_artifacts: + path: wheels + - run: + name: Upload Wheel + command: | + twine upload --skip-existing -u $PYPI_U -p $PYPI_P wheels/* + + manylinux: + machine: + image: ubuntu-1604:201903-01 + steps: + - checkout + - run: + name: sdist + command: python setup.py sdist + - python/load-cache: + key: manylinuxdepsv6-{{ .Branch }}.{{ arch }} + dependency-file: requirements.txt + - run: + name: Deps + command: | + sudo apt-get install python-pip + pip install -U pip + pip install twine + which twine + - python/save-cache: + key: manylinuxdepsv6-{{ .Branch }}.{{ arch }} + dependency-file: requirements.txt + - run: + name: Build Wheels + command: | + if [[ -z "${CIRCLE_PULL_REQUEST}" ]]; then + echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin; + fi + ./ci/travis/build-manylinux.sh + - run: + name: PyPi Upload + command: | + twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASSWORD dist/* wheelhouse/* + +workflows: + version: 2.1 + main: + jobs: + - python_test: + matrix: + parameters: + python_ver: + - "3.6" + - "3.7" + - "3.8" + filters: + tags: + ignore: /.*/ + - manylinux: + context: Docker + filters: + tags: + only: /.*/ + branches: + ignore: /.*/ + - osx: + matrix: + parameters: + xcode_ver: + - "11.6.0" + - "11.1.0" + context: Docker + filters: + tags: + only: /.*/ + branches: + ignore: /.*/ diff --git a/Changelog.rst b/Changelog.rst index 170001ec..8fa31309 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,6 +1,15 @@ Change Log ============ +1.12.2 +++++++ + +Fixes +------ + +* `ParallelSSHClient.copy_file` with recurse enabled and absolute destination path would create empty directory in home directory of user - #197. +* `ParallelSSHClient.copy_file` and `scp_recv` with recurse enabled would not create remote directories when copying empty local directories. + 1.12.1 ++++++ diff --git a/pssh/clients/base/single.py b/pssh/clients/base/single.py index 602d2d46..efe9514e 100644 --- a/pssh/clients/base/single.py +++ b/pssh/clients/base/single.py @@ -294,6 +294,7 @@ def mkdir(self, sftp, directory, _parent_path=None): def _copy_dir(self, local_dir, remote_dir, sftp): """Call copy_file on every file in the specified directory, copying them to the specified remote directory.""" + self.mkdir(sftp, remote_dir) file_list = os.listdir(local_dir) for file_name in file_list: local_path = os.path.join(local_dir, file_name) diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index 32ee778a..227de0b3 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -17,6 +17,7 @@ import logging import os +from collections import deque from warnings import warn from gevent import sleep, spawn @@ -325,8 +326,7 @@ def _mkdir(self, sftp, directory): raise SFTPIOError(msg, directory, self.host, error) logger.debug("Created remote directory %s", directory) - def copy_file(self, local_file, remote_file, recurse=False, - sftp=None, _dir=None): + def copy_file(self, local_file, remote_file, recurse=False, sftp=None): """Copy local file to host via SFTP. :param local_file: Local filepath to copy to remote host @@ -383,7 +383,7 @@ def sftp_put(self, sftp, local_file, remote_file): logger.error(msg, remote_file, ex) raise SFTPIOError(msg, remote_file, ex) - def mkdir(self, sftp, directory, _parent_path=None): + def mkdir(self, sftp, directory): """Make directory via SFTP channel. Parent paths in the directory are created if they do not exist. @@ -395,27 +395,20 @@ def mkdir(self, sftp, directory, _parent_path=None): Catches and logs at error level remote IOErrors on creating directory. """ - try: - _dir, sub_dirs = directory.split('/', 1) - except ValueError: - _dir = directory.split('/', 1)[0] - sub_dirs = None - if not _dir and directory.startswith('/'): + _paths_to_create = deque() + for d in directory.split('/'): + if not d: + continue + _paths_to_create.append(d) + cwd = '' if directory.startswith('/') else '.' + while _paths_to_create: + cur_dir = _paths_to_create.popleft() + cwd = '/'.join([cwd, cur_dir]) try: - _dir, sub_dirs = sub_dirs.split(os.path.sep, 1) - except ValueError: - return True - if _parent_path is not None: - _dir = '/'.join((_parent_path, _dir)) - try: - self._eagain(sftp.stat, _dir) - except (SFTPHandleError, SFTPProtocolError) as ex: - logger.debug("Stat for %s failed with %s", _dir, ex) - self._mkdir(sftp, _dir) - if sub_dirs is not None: - if directory.startswith('/'): - _dir = ''.join(('/', _dir)) - return self.mkdir(sftp, sub_dirs, _parent_path=_dir) + self._eagain(sftp.stat, cwd) + except (SFTPHandleError, SFTPProtocolError) as ex: + logger.debug("Stat for %s failed with %s", cwd, ex) + self._mkdir(sftp, cwd) def copy_remote_file(self, remote_file, local_file, recurse=False, sftp=None, encoding='utf-8'): @@ -470,7 +463,7 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None, encoding='utf-8'): """Copy remote file to local host via SCP. - Note - Remote directory listings are gather via SFTP when + Note - Remote directory listings are gathered via SFTP when ``recurse`` is enabled - SCP lacks directory list support. Enabling recursion therefore involves creating an extra SFTP channel and requires SFTP support on the server. @@ -504,6 +497,10 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None, except SFTPError: pass else: + try: + os.makedirs(local_file) + except OSError: + pass file_list = self._sftp_readdir(dir_h) return self._scp_recv_dir(file_list, remote_file, local_file, sftp, diff --git a/tests/native/test_parallel_client.py b/tests/native/test_parallel_client.py index e0f2ad24..3a3ed1db 100644 --- a/tests/native/test_parallel_client.py +++ b/tests/native/test_parallel_client.py @@ -384,8 +384,14 @@ def test_pssh_copy_file(self): except Exception: raise finally: - os.unlink(remote_file_abspath) - shutil.rmtree(remote_test_dir_abspath) + try: + os.unlink(remote_file_abspath) + except OSError: + pass + try: + shutil.rmtree(remote_test_dir_abspath) + except Exception: + pass # No directory remote_file_abspath = os.path.expanduser('~/' + remote_filepath) cmds = client.copy_file(local_filename, remote_filepath) @@ -494,11 +500,11 @@ def test_pssh_client_directory_relative_path(self): for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) finally: - try: - shutil.rmtree(local_test_path) - shutil.rmtree(remote_test_path_abs) - except Exception: - pass + for _path in (local_test_path, remote_test_path_abs): + try: + shutil.rmtree(_path) + except Exception: + pass def test_pssh_client_directory_abs_path(self): client = ParallelSSHClient([self.host], port=self.port, @@ -533,8 +539,11 @@ def test_pssh_client_directory_abs_path(self): for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) finally: - shutil.rmtree(local_test_path) - shutil.rmtree(remote_test_path_abs) + for _path in (local_test_path, remote_test_path_abs): + try: + shutil.rmtree(_path) + except Exception: + pass def test_pssh_client_copy_file_failure(self): """Test failure scenarios of file copy""" @@ -588,7 +597,10 @@ def test_pssh_client_copy_file_failure(self): mask = 0o600 os.chmod(remote_test_path_abs, mask) for path in [local_test_path, remote_test_path_abs]: - shutil.rmtree(path) + try: + shutil.rmtree(path) + except Exception: + pass def test_pssh_copy_remote_file_failure(self): cmds = self.client.copy_remote_file( @@ -646,10 +658,16 @@ def test_pssh_copy_remote_file(self): for path in local_file_paths: self.assertTrue(os.path.isfile(path)) except Exception: - shutil.rmtree(remote_test_path_abs) + try: + shutil.rmtree(remote_test_path_abs) + except Exception: + pass raise finally: - shutil.rmtree(local_copied_dir) + try: + shutil.rmtree(local_copied_dir) + except Exception: + pass # Relative path cmds = self.client.copy_remote_file(remote_test_path_rel, local_test_path, @@ -660,7 +678,10 @@ def test_pssh_copy_remote_file(self): for path in local_file_paths: self.assertTrue(os.path.isfile(path)) finally: - shutil.rmtree(local_copied_dir) + try: + shutil.rmtree(local_copied_dir) + except Exception: + pass # Different suffix cmds = self.client.copy_remote_file(remote_test_path_abs, local_test_path, @@ -672,8 +693,11 @@ def test_pssh_copy_remote_file(self): path = path.replace(local_copied_dir, new_local_copied_dir) self.assertTrue(os.path.isfile(path)) finally: - shutil.rmtree(new_local_copied_dir) - shutil.rmtree(remote_test_path_abs) + for _path in (new_local_copied_dir, remote_test_path_abs): + try: + shutil.rmtree(_path) + except Exception: + pass def test_pssh_copy_remote_file_per_host_args(self): """Test parallel remote copy file with per-host arguments""" @@ -1316,6 +1340,9 @@ def test_scp_send_dir(self): finally: try: os.unlink(local_filename) + except OSError: + pass + try: shutil.rmtree(remote_test_dir_abspath) except OSError: pass @@ -1399,10 +1426,16 @@ def test_scp_recv(self): for path in local_file_paths: self.assertTrue(os.path.isfile(path)) except Exception: - shutil.rmtree(remote_test_path_abs) + try: + shutil.rmtree(remote_test_path_abs) + except Exception: + pass raise finally: - shutil.rmtree(local_copied_dir) + try: + shutil.rmtree(local_copied_dir) + except Exception: + pass # Relative path cmds = self.client.scp_recv(remote_test_path_rel, local_test_path, @@ -1413,8 +1446,11 @@ def test_scp_recv(self): for path in local_file_paths: self.assertTrue(os.path.isfile(path)) finally: - shutil.rmtree(remote_test_path_abs) - shutil.rmtree(local_copied_dir) + for _path in (remote_test_path_abs, local_copied_dir): + try: + shutil.rmtree(_path) + except Exception: + pass def test_bad_hosts_value(self): self.assertRaises(TypeError, ParallelSSHClient, 'a host') diff --git a/tests/native/test_single_client.py b/tests/native/test_single_client.py index 869478c8..dcbf8084 100644 --- a/tests/native/test_single_client.py +++ b/tests/native/test_single_client.py @@ -20,6 +20,7 @@ import logging import time import subprocess +import shutil from gevent import socket, sleep, spawn @@ -224,3 +225,131 @@ def test_finished(self): stdout = list(self.client.read_output(channel)) self.assertTrue(self.client.finished(channel)) self.assertListEqual(stdout, [b'me']) + + def test_scp_abspath_recursion(self): + cur_dir = os.path.dirname(__file__) + dir_name_to_copy = 'a_dir' + files = ['file1', 'file2'] + dir_paths = [cur_dir, dir_name_to_copy] + to_copy_dir_path = os.path.abspath(os.path.sep.join(dir_paths)) + # Dir to copy to + copy_to_path = '/tmp/copied_dir' + try: + shutil.rmtree(copy_to_path) + except Exception: + pass + try: + try: + os.makedirs(to_copy_dir_path) + except OSError: + pass + # Copy for empty remote dir should create local dir + self.client.scp_recv(to_copy_dir_path, copy_to_path, recurse=True) + self.assertTrue(os.path.isdir(copy_to_path)) + for _file in files: + _filepath = os.path.sep.join([to_copy_dir_path, _file]) + with open(_filepath, 'w') as fh: + fh.writelines(['asdf']) + self.client.scp_recv(to_copy_dir_path, copy_to_path, recurse=True) + for _file in files: + local_file_path = os.path.sep.join([copy_to_path, _file]) + self.assertTrue(os.path.isfile(local_file_path)) + finally: + for _path in (to_copy_dir_path, copy_to_path): + try: + shutil.rmtree(_path) + except Exception: + pass + + def test_copy_file_abspath_recurse(self): + cur_dir = os.path.dirname(__file__) + dir_name_to_copy = 'a_dir' + files = ['file1', 'file2'] + dir_paths = [cur_dir, dir_name_to_copy] + to_copy_dir_path = os.path.abspath(os.path.sep.join(dir_paths)) + copy_to_path = '/tmp/dest_path//' + for _path in (copy_to_path, to_copy_dir_path): + try: + shutil.rmtree(_path) + except Exception: + pass + try: + try: + os.makedirs(to_copy_dir_path) + except OSError: + pass + self.client.copy_file(to_copy_dir_path, copy_to_path, recurse=True) + self.assertTrue(os.path.isdir(copy_to_path)) + for _file in files: + _filepath = os.path.sep.join([to_copy_dir_path, _file]) + with open(_filepath, 'w') as fh: + fh.writelines(['asdf']) + self.client.copy_file(to_copy_dir_path, copy_to_path, recurse=True) + self.assertFalse(os.path.exists(os.path.expanduser('~/tmp'))) + for _file in files: + local_file_path = os.path.sep.join([copy_to_path, _file]) + self.assertTrue(os.path.isfile(local_file_path)) + finally: + for _path in (copy_to_path, to_copy_dir_path): + try: + shutil.rmtree(_path) + except Exception: + pass + + def test_copy_file_remote_dir_relpath(self): + cur_dir = os.path.dirname(__file__) + dir_base_dir = 'a_dir' + dir_name_to_copy = '//'.join([dir_base_dir, 'dir1', 'dir2']) + file_to_copy = 'file_to_copy' + dir_path = [cur_dir, file_to_copy] + copy_from_file_path = os.path.abspath(os.path.sep.join(dir_path)) + copy_to_file_path = '///'.join([dir_name_to_copy, file_to_copy]) + copy_to_abs_path = os.path.abspath(os.path.expanduser('~/' + copy_to_file_path)) + copy_to_abs_dir = os.path.abspath(os.path.expanduser('~/' + dir_base_dir)) + for _path in (copy_from_file_path, copy_to_abs_dir): + try: + shutil.rmtree(_path, ignore_errors=True) + except Exception: + pass + try: + with open(copy_from_file_path, 'w') as fh: + fh.writelines(['asdf']) + self.client.copy_file(copy_from_file_path, copy_to_file_path) + self.assertTrue(os.path.isfile(copy_to_abs_path)) + finally: + for _path in (copy_from_file_path, copy_to_abs_dir): + try: + shutil.rmtree(_path, ignore_errors=True) + except Exception: + pass + + def test_sftp_mkdir_abspath(self): + remote_dir = '/tmp/dir_to_create/dir1/dir2/dir3' + _sftp = self.client._make_sftp() + try: + self.client.mkdir(_sftp, remote_dir) + self.assertTrue(os.path.isdir(remote_dir)) + self.assertFalse(os.path.exists(os.path.expanduser('~/tmp'))) + finally: + for _dir in (remote_dir, os.path.expanduser('~/tmp')): + try: + shutil.rmtree(_dir) + except Exception: + pass + + def test_sftp_mkdir_rel_path(self): + remote_dir = 'dir_to_create/dir1/dir2/dir3' + try: + shutil.rmtree(os.path.expanduser('~/' + remote_dir)) + except Exception: + pass + _sftp = self.client._make_sftp() + try: + self.client.mkdir(_sftp, remote_dir) + self.assertTrue(os.path.exists(os.path.expanduser('~/' + remote_dir))) + finally: + for _dir in (remote_dir, os.path.expanduser('~/tmp')): + try: + shutil.rmtree(_dir) + except Exception: + pass