diff --git a/ftpsync/sftp_target.py b/ftpsync/sftp_target.py index 5ce08c3..1f69f63 100644 --- a/ftpsync/sftp_target.py +++ b/ftpsync/sftp_target.py @@ -5,6 +5,7 @@ """ import json import logging +import os import stat import time from posixpath import join as join_url @@ -29,6 +30,19 @@ ) +class PatchedPysftpConnection(pysftp.Connection): + """ + Patched version that fixes exception on connect errors: + `AttributeError: 'Connection' object has no attribute '_sftp_live'` + https://stackoverflow.com/a/65060184 + """ + + def __init__(self, *args, **kwargs): + self._sftp_live = False + self._transport = None + super().__init__(*args, **kwargs) + + # =============================================================================== # SFTPTarget # =============================================================================== @@ -45,9 +59,8 @@ class SFTPTarget(_Target): """ DEFAULT_BLOCKSIZE = 8 * 1024 # ftplib uses 8k chunks by default - MAX_SPOOL_MEM = ( - 100 * 1024 - ) # keep open_readable() buffer in memory if smaller than 100kB + # keep open_readable() buffer in memory if smaller than 100kB + MAX_SPOOL_MEM = 100 * 1024 def __init__( self, @@ -140,7 +153,7 @@ def open(self): assert self.sftp is None while True: try: - self.sftp = pysftp.Connection( + self.sftp = PatchedPysftpConnection( self.host, username=self.username, password=self.password, @@ -159,12 +172,10 @@ def open(self): # Continue while-loop except paramiko.ssh_exception.SSHException as e: write_error( - "{exc}: Try `ssh-keyscan HOST` to add it " - "(or pass `--no-verify-host-keys` if you don't care about security).".format( - exc=e - ) + f"{e}: Try `ssh-keyscan HOST` to add it to `USER/.ssh/known_hosts` " + "(or pass `--no-verify-host-keys` if you don't care about security)." ) - raise + raise SystemExit if verbose >= 4: write( @@ -304,9 +315,49 @@ def mkdir(self, dir_name): self.check_write(dir_name) self.sftp.mkdir(dir_name) - def rmdir(self, dir_name): + def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None): + # FTP does not support deletion of non-empty directories. + assert is_native(dir_name) self.check_write(dir_name) - return self.sftp.rmdir(dir_name) + names = [] + + attr_list = self.sftp.listdir_attr(dir_name) + + # write(f"rmdir({dir_name}): {attr_list}") + for dir_attr in attr_list: + name = dir_attr.filename + # name = self.re_encode_to_native(name) + if "/" in name: + name = os.path.basename(name) + if name in (".", ".."): + continue + if predicate and not predicate(name): + continue + names.append(name) + + if len(names) > 0: + self.sftp.cwd(dir_name) + try: + for name in names: + try: + # try to delete this as a file + self.sftp.remove(name) + except IOError: # ftplib.all_errors as _e: + write(f" sftp.delete({name}) failed, trying rmdir()...") + # assume is a folder + self.rmdir(name) + finally: + if dir_name != ".": + self.sftp.cwd("..") + # write("sftp.rmd(%s)..." % (dir_name, )) + if not keep_root_folder: + self.sftp.rmdir(dir_name) + return + + def rmdir(self, dir_name): + # self.check_write(dir_name) + # return self.sftp.rmdir(dir_name) + return self._rmdir_impl(dir_name) _paramiko_py3compat_u = paramiko.py3compat.u diff --git a/tests/test_1x.py b/tests/test_1x.py index 85ef239..74842ac 100644 --- a/tests/test_1x.py +++ b/tests/test_1x.py @@ -31,7 +31,7 @@ def prepare_fixtures_1(): # print("PYFTPSYNC_TEST_FOLDER", os.environ.get("PYFTPSYNC_TEST_FOLDER")) # print("PYFTPSYNC_TEST_FTP_URL", os.environ.get("PYFTPSYNC_TEST_FTP_URL")) - assert os.path.isdir(PYFTPSYNC_TEST_FOLDER) + assert os.path.isdir(PYFTPSYNC_TEST_FOLDER), PYFTPSYNC_TEST_FOLDER # Reset all empty_folder(PYFTPSYNC_TEST_FOLDER) # Add some files to ../local/