diff --git a/_unittests/ut_filehelper/test_folder_transfer.py b/_unittests/ut_filehelper/test_folder_transfer.py index b406c2f13..4c6d49f55 100644 --- a/_unittests/ut_filehelper/test_folder_transfer.py +++ b/_unittests/ut_filehelper/test_folder_transfer.py @@ -28,7 +28,9 @@ def test_folder_transfer(self) : ftn = FileTreeNode(folder) ftp = MockTransferFTP("http://www.xavierdupre.fr/", "login", "password", fLOG=fLOG) - fftp = FolderTransferFTP (ftn, ftp, status) + fftp = FolderTransferFTP (ftn, ftp, status, + footer_html = "footer", + content_filter = lambda c : c) li = list ( fftp.iter_eligible_files() ) assert len(li) > 0 diff --git a/src/pyquickhelper/filehelper/ftp_transfer.py b/src/pyquickhelper/filehelper/ftp_transfer.py index 41911d6fc..4e409f288 100644 --- a/src/pyquickhelper/filehelper/ftp_transfer.py +++ b/src/pyquickhelper/filehelper/ftp_transfer.py @@ -6,7 +6,7 @@ moved from pyensae to pyquickhelper """ from ftplib import FTP -import os +import os, io from ..loghelper.flog import noLOG @@ -89,7 +89,7 @@ def run_command (self, command, *args) : if command == FTP.pwd or command == FTP.dir: return t elif command != FTP.cwd : - self.LOG(" ** run ", str(command), str(args)) + pass return True except Exception as e: if TransferFTP.errorNoDirectory in str(e) : @@ -164,40 +164,37 @@ def ls(self, path = '.'): """ return self.run_command(FTP.dir, path) - def transfer (self, file, to, debug = False) : + def transfer (self, file, to, name, debug = False) : """ transfers a file - @param file file + @param file file name or stream (binary, BytesIO) @param to destination (a folder) + @param name name of the stream on the website @param debug if True, displays more information @return status - """ - if not os.path.exists(file): - raise FileNotFoundError(file) + .. versionchanged:: 1.0 + file can be a file name or a stream, + parameter *name* was added + """ path = to.split("/") path = [ _ for _ in path if len(_) > 0 ] - temp = os.path.split(file)[-1] - self.LOG ("[upload] ", temp, "to", to) - - if debug : - self.LOG (" -- path", path) - self.LOG (" -- pwd", self.pwd()) for p in path : - if debug : self.LOG (" -- cwd", p) self.cwd(p, True) - if debug : - self.LOG (" -- transferring", file) - - with open(file, "rb") as f : - r = self.run_command(FTP.storbinary, 'STOR ' + temp, f) + if isinstance(file, str): + if not os.path.exists(file): + raise FileNotFoundError(file) + with open(file, "rb") as f : + r = self.run_command(FTP.storbinary, 'STOR ' + name, f) + elif isinstance(file, io.BytesIO): + r = self.run_command(FTP.storbinary, 'STOR ' + name, file) + else: + r = self.run_command(FTP.storbinary, 'STOR ' + name, file) for p in path : - if debug : - self.LOG (" -- cwd", "..") self.cwd("..") return r diff --git a/src/pyquickhelper/filehelper/ftp_transfer_files.py b/src/pyquickhelper/filehelper/ftp_transfer_files.py index 4ee02d032..43088a737 100644 --- a/src/pyquickhelper/filehelper/ftp_transfer_files.py +++ b/src/pyquickhelper/filehelper/ftp_transfer_files.py @@ -4,7 +4,7 @@ .. versionadded:: 1.0 """ -import os +import os, io from .files_status import FilesStatus from ..loghelper.flog import noLOG @@ -14,6 +14,25 @@ class FolderTransferFTPException(Exception): """ pass +_text_extensions = { ".ipynb", ".html", ".py", ".cpp", ".h", ".hpp", ".c", + ".cs", ".txt", ".csv", ".xml", ".css", ".js", ".r", ".doc", + ".ind", ".buildinfo", ".rst", ".aux", ".out", ".log", + } + +def content_as_binary(filename): + """ + determines if filename is binary or None before transfering it + + @param filename filename + @return boolean + """ + global _text_extensions + ext = os.path.splitext(filename)[-1].lower() + if ext in _text_extensions: + return False + else: + return True + class FolderTransferFTP: """ This class aims at transfering a folder to a FTP website, @@ -29,7 +48,7 @@ class FolderTransferFTP: ftn = FileTreeNode("c:/somefolder") ftp = TransferFTP("ftp.website.fr", "login", "password", fLOG=print) fftp = FolderTransferFTP (ftn, ftp, "status_file.txt", - root_website = "/www/htdocs/app/pyquickhelper/helpsphinx") + root_web = "/www/htdocs/app/pyquickhelper/helpsphinx") fftp.start_transfering() @@ -70,7 +89,7 @@ class FolderTransferFTP: sfile = os.path.join(this, "status_%s.txt" % module) ftn = FileTreeNode(root) fftp = FolderTransferFTP (ftn, ftp, sfile, - root_website = rootw % module, + root_web = rootw % module, fLOG=print) fftp.start_transfering() @@ -79,7 +98,7 @@ class FolderTransferFTP: ftn = FileTreeNode(os.path.join(root,".."), filter = lambda root, path, f, dir: not dir) fftp = FolderTransferFTP (ftn, ftp, sfile, - root_website = (rootw % module).replace("helpsphinx",""), + root_web = (rootw % module).replace("helpsphinx",""), fLOG=print) fftp.start_transfering() @@ -95,9 +114,12 @@ def __init__(self, file_tree_node, ftp_transfer, file_status, - root_local = None, - root_website = None, - fLOG = noLOG): + root_local = None, + root_web = None, + footer_html = None, + content_filter = None, + is_binary = content_as_binary, + fLOG = noLOG): """ constructor @@ -105,15 +127,24 @@ def __init__(self, @param ftp_transfer @see cl TransferFTP @param file_status file keeping the status file @param root_local local root - @param root_website remote root on the website + @param root_web remote root on the website + @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience) + at the end of the file (before tag ````) + @param content_filter function which transform the content if a specific string is found + in the file, if the result is None, it raises an exception + indicating the file cannot be transfered (applies only on text files) + @param is_binary function which determines if content of a files is binary or not @param fLOG logging function """ - self._ftn = file_tree_node - self._ftp = ftp_transfer - self._status = file_status - self._root_local = root_local if root_local is not None else file_tree_node.root - self._root_website = root_website if root_website is not None else "" - self.fLOG = fLOG + self._ftn = file_tree_node + self._ftp = ftp_transfer + self._status = file_status + self._root_local = root_local if root_local is not None else file_tree_node.root + self._root_web = root_web if root_web is not None else "" + self.fLOG = fLOG + self._footer_html = footer_html + self._content_filter = content_filter + self._is_binary = is_binary self._ft = FilesStatus(file_status) @@ -123,7 +154,7 @@ def __str__(self): """ mes = [ "FolderTransferFTP" ] mes += [ " local root: {0}".format(self._root_local) ] - mes += [ " remote root: {0}".format(self._root_website) ] + mes += [ " remote root: {0}".format(self._root_web) ] return "\n".join(mes) def iter_eligible_files(self): @@ -149,6 +180,55 @@ def update_status(self, file): self._ft.save_dates() return r + def preprocess_before_transfering(self, path): + """ + Applies some preprocessing to the file to transfer. + It adds the footer for example. + It returns a stream which should be closed by using method @see me close_stream + + @param path file name + @return binary stream + """ + if self._is_binary(path): + return open(path, "rb") + else: + if self._footer_html is None and self._content_filter is None: + return open(path, "rb") + else: + with open(path, "r", encoding="utf8") as f : + content = f.read() + + # footer + if self._footer_html is not None and os.path.splitext(path)[-1].lower() in (".htm", ".html") : + spl = content.split("") + if len(spl) == 1: + raise FolderTransferFTPException("tag was not found, it must be written in lower case, file: {0}".format(path)) + + if len(spl) != 2: + spl = [ "".join(spl[:-1]), spl[-1] ] + + content = spl[0] + self._footer_html + "" + spl[-1] + + # filter + content = self._content_filter(content) + if content is None: + raise FolderTransferFTPException("File {0} cannot be transferred due to its content".format(path)) + + # to binary + bcont = content.encode("utf8") + return io.BytesIO(bcont) + + def close_stream(self, stream): + """ + close a stream opened by @see me preprocess_before_transfering + + @param stream stream to close + """ + if isinstance(stream, io.BytesIO): + pass + else: + stream.close() + def start_transfering(self): """ starts transfering files to the remote website @@ -160,19 +240,33 @@ def start_transfering(self): issues = [ ] done = [ ] total = list ( self.iter_eligible_files() ) + sum_bytes = 0 for i, file in enumerate(total): if i % 20 == 0: - self.fLOG("#### transfering",i,len(total)) + self.fLOG("#### transfering %d/%d (so far %d bytes)"% (i,len(total), sum_bytes)) relp = os.path.relpath(file.fullname, self._root_local) if ".." in relp: raise ValueError("the local root is not accurate:\n{0}\nFILE:\n{1}\nRELPATH:\n{2}".format(self, file.fullname, relp)) - path = self._root_website + "/" + os.path.split(relp)[0] + path = self._root_web + "/" + os.path.split(relp)[0] path = path.replace("\\","/") + + size = os.stat(file.fullname).st_size + self.fLOG("[upload % 8d bytes name=%s -- fullname=%s]" % ( + size, + os.path.split(file.fullname)[-1], + file.fullname)) + + data = self.preprocess_before_transfering(file.fullname) + try : - r = self._ftp.transfer (file.fullname, path) - except Exception as e : + r = self._ftp.transfer (data, path, os.path.split(file.fullname)[-1]) + except FileNotFoundError as e : r = False - issues.append( (file.fullname, e) ) + issues.append( (file.fullname, "not found") ) + + self.close_stream(data) + + sum_bytes += size if r : fi = self.update_status(file.fullname)