add possibility to add a footer or to preprocess a file before upload…

…ing it to the website

_unittests/ut_filehelper/test_folder_transfer.py

@@ -28,7 +28,9 @@ def test_folder_transfer(self) :
 
         ftn  = FileTreeNode(folder)
         ftp  = MockTransferFTP("http://www.xavierdupre.fr/", "login", "password", fLOG=fLOG)
-        fftp = FolderTransferFTP (ftn, ftp, status)
+        fftp = FolderTransferFTP (ftn, ftp, status,
+                        footer_html = "<b>footer</b>",
+                        content_filter = lambda c : c)
 
         li = list ( fftp.iter_eligible_files() )
         assert len(li) > 0

src/pyquickhelper/filehelper/ftp_transfer.py

@@ -6,7 +6,7 @@
     moved from pyensae to pyquickhelper
 """
 from ftplib import FTP
-import os
+import os, io
 
 from ..loghelper.flog import noLOG
 
@@ -89,7 +89,7 @@ def run_command (self, command, *args) :
             if command == FTP.pwd or command == FTP.dir:
                 return t
             elif command != FTP.cwd :
-                self.LOG("    ** run ", str(command), str(args))
+                pass
             return True
         except Exception as e:
             if TransferFTP.errorNoDirectory in str(e) :
@@ -164,40 +164,37 @@ def ls(self, path = '.'):
         """
         return self.run_command(FTP.dir, path)
 
-    def transfer (self, file, to, debug = False) :
+    def transfer (self, file, to, name, debug = False) :
         """
         transfers a file
 
-        @param      file        file
+        @param      file        file name or stream (binary, BytesIO)
         @param      to          destination (a folder)
+        @param      name        name of the stream on the website
         @param      debug       if True, displays more information
         @return                 status
-        """
-        if not os.path.exists(file):
-            raise FileNotFoundError(file)
 
+        .. versionchanged:: 1.0
+            file can be a file name or a stream,
+            parameter *name* was added
+        """
         path = to.split("/")
         path = [ _ for _ in path if len(_) > 0 ]
-        temp = os.path.split(file)[-1]
-        self.LOG ("[upload] ", temp, "to", to)
-
-        if debug :
-            self.LOG ("    -- path", path)
-            self.LOG ("    -- pwd", self.pwd())
 
         for p in path :
-            if debug :  self.LOG ("    -- cwd", p)
             self.cwd(p, True)
 
-        if debug :
-            self.LOG ("    -- transferring", file)
-
-        with open(file, "rb") as f :
-            r = self.run_command(FTP.storbinary, 'STOR ' + temp, f)
+        if isinstance(file, str):
+            if not os.path.exists(file):
+                raise FileNotFoundError(file)
+            with open(file, "rb") as f :
+                r = self.run_command(FTP.storbinary, 'STOR ' + name, f)
+        elif isinstance(file, io.BytesIO):
+            r = self.run_command(FTP.storbinary, 'STOR ' + name, file)
+        else:
+            r = self.run_command(FTP.storbinary, 'STOR ' + name, file)
 
         for p in path :
-            if debug :
-                self.LOG ("    -- cwd", "..")
             self.cwd("..")
 
         return r

src/pyquickhelper/filehelper/ftp_transfer_files.py

@@ -4,7 +4,7 @@
 
 .. versionadded:: 1.0
 """
-import os
+import os, io
 from .files_status      import FilesStatus
 from ..loghelper.flog   import noLOG
 
@@ -14,6 +14,25 @@ class FolderTransferFTPException(Exception):
     """
     pass
 
+_text_extensions = { ".ipynb", ".html", ".py", ".cpp", ".h", ".hpp", ".c",
+                ".cs", ".txt", ".csv", ".xml", ".css", ".js", ".r", ".doc",
+                ".ind", ".buildinfo", ".rst", ".aux", ".out", ".log",
+                }
+
+def content_as_binary(filename):
+    """
+    determines if filename is binary or None before transfering it
+
+    @param      filename        filename
+    @return                     boolean
+    """
+    global _text_extensions
+    ext = os.path.splitext(filename)[-1].lower()
+    if ext in _text_extensions:
+        return False
+    else:
+        return True
+
 class FolderTransferFTP:
     """
     This class aims at transfering a folder to a FTP website,
@@ -29,7 +48,7 @@ class FolderTransferFTP:
     ftn  = FileTreeNode("c:/somefolder")
     ftp  = TransferFTP("ftp.website.fr", "login", "password", fLOG=print)
     fftp = FolderTransferFTP (ftn, ftp, "status_file.txt",
-            root_website = "/www/htdocs/app/pyquickhelper/helpsphinx")
+            root_web = "/www/htdocs/app/pyquickhelper/helpsphinx")
 
     fftp.start_transfering()
 
@@ -70,7 +89,7 @@ class FolderTransferFTP:
         sfile = os.path.join(this, "status_%s.txt" % module)
         ftn  = FileTreeNode(root)
         fftp = FolderTransferFTP (ftn, ftp, sfile,
-                            root_website = rootw % module,
+                            root_web = rootw % module,
                             fLOG=print)
 
         fftp.start_transfering()
@@ -79,7 +98,7 @@ class FolderTransferFTP:
 
         ftn  = FileTreeNode(os.path.join(root,".."), filter = lambda root, path, f, dir: not dir)
         fftp = FolderTransferFTP (ftn, ftp, sfile,
-                            root_website = (rootw % module).replace("helpsphinx",""),
+                            root_web = (rootw % module).replace("helpsphinx",""),
                             fLOG=print)
 
         fftp.start_transfering()
@@ -95,25 +114,37 @@ def __init__(self,
                     file_tree_node,
                     ftp_transfer,
                     file_status,
-                    root_local = None,
-                    root_website = None,
-                    fLOG = noLOG):
+                    root_local      = None,
+                    root_web        = None,
+                    footer_html     = None,
+                    content_filter  = None,
+                    is_binary       = content_as_binary,
+                    fLOG            = noLOG):
         """
         constructor
 
         @param      file_tree_node      @see cl FileTreeNode
         @param      ftp_transfer        @see cl TransferFTP
         @param      file_status         file keeping the status file
         @param      root_local          local root
-        @param      root_website        remote root on the website
+        @param      root_web            remote root on the website
+        @param      footer_html         append  this HTML code to any uploaded page (such a javascript code to count the audience)
+                                        at the end of the file (before tag ``</body>``)
+        @param      content_filter      function which transform the content if a specific string is found
+                                        in the file, if the result is None, it raises an exception
+                                        indicating  the file cannot be transfered (applies only on text files)
+        @param      is_binary           function which determines if content of a files is binary or not
         @param      fLOG                logging function
         """
-        self._ftn = file_tree_node
-        self._ftp = ftp_transfer
-        self._status = file_status
-        self._root_local = root_local if root_local is not None else file_tree_node.root
-        self._root_website = root_website if root_website is not None else ""
-        self.fLOG = fLOG
+        self._ftn               = file_tree_node
+        self._ftp               = ftp_transfer
+        self._status            = file_status
+        self._root_local        = root_local if root_local is not None else file_tree_node.root
+        self._root_web          = root_web if root_web is not None else ""
+        self.fLOG               = fLOG
+        self._footer_html       = footer_html
+        self._content_filter    = content_filter
+        self._is_binary         = is_binary
 
         self._ft = FilesStatus(file_status)
 
@@ -123,7 +154,7 @@ def __str__(self):
         """
         mes  = [ "FolderTransferFTP" ]
         mes += [ "    local root: {0}".format(self._root_local) ]
-        mes += [ "    remote root: {0}".format(self._root_website) ]
+        mes += [ "    remote root: {0}".format(self._root_web) ]
         return "\n".join(mes)
 
     def iter_eligible_files(self):
@@ -149,6 +180,55 @@ def update_status(self, file):
         self._ft.save_dates()
         return r
 
+    def preprocess_before_transfering(self, path):
+        """
+        Applies some preprocessing to the file to transfer.
+        It adds the footer for example.
+        It returns a stream which should be closed by using method @see me close_stream
+
+        @param      path        file name
+        @return                 binary stream
+        """
+        if self._is_binary(path):
+            return open(path, "rb")
+        else:
+            if self._footer_html is None and self._content_filter is None:
+                return open(path, "rb")
+            else:
+                with open(path, "r", encoding="utf8") as f :
+                    content = f.read()
+
+                # footer
+                if self._footer_html is not None and os.path.splitext(path)[-1].lower() in (".htm", ".html") :
+                    spl = content.split("</body>")
+                    if len(spl) == 1:
+                        raise FolderTransferFTPException("tag </body> was not found, it must be written in lower case, file: {0}".format(path))
+
+                    if len(spl) != 2:
+                        spl = [ "</body>".join(spl[:-1]), spl[-1] ]
+
+                    content = spl[0] + self._footer_html + "</body>" + spl[-1]
+
+                # filter
+                content = self._content_filter(content)
+                if content is None:
+                    raise FolderTransferFTPException("File {0} cannot be transferred due to its content".format(path))
+
+                # to binary
+                bcont = content.encode("utf8")
+                return io.BytesIO(bcont)
+
+    def close_stream(self, stream):
+        """
+        close a stream opened by @see me preprocess_before_transfering
+
+        @param      stream      stream to close
+        """
+        if isinstance(stream, io.BytesIO):
+            pass
+        else:
+            stream.close()
+
     def start_transfering(self):
         """
         starts transfering files to the remote website
@@ -160,19 +240,33 @@ def start_transfering(self):
         issues = [ ]
         done = [ ]
         total = list ( self.iter_eligible_files() )
+        sum_bytes = 0
         for i, file in enumerate(total):
             if i % 20 == 0:
-                self.fLOG("#### transfering",i,len(total))
+                self.fLOG("#### transfering %d/%d (so far %d bytes)"% (i,len(total), sum_bytes))
             relp = os.path.relpath(file.fullname, self._root_local)
             if ".." in relp:
                 raise ValueError("the local root is not accurate:\n{0}\nFILE:\n{1}\nRELPATH:\n{2}".format(self, file.fullname, relp))
-            path = self._root_website + "/" + os.path.split(relp)[0]
+            path = self._root_web + "/" + os.path.split(relp)[0]
             path = path.replace("\\","/")
+
+            size = os.stat(file.fullname).st_size
+            self.fLOG("[upload % 8d bytes name=%s -- fullname=%s]" % (
+                                        size,
+                                        os.path.split(file.fullname)[-1],
+                                        file.fullname))
+
+            data = self.preprocess_before_transfering(file.fullname)
+
             try :
-                r = self._ftp.transfer (file.fullname, path)
-            except Exception as e :
+                r = self._ftp.transfer (data, path, os.path.split(file.fullname)[-1])
+            except FileNotFoundError as e :
                 r = False
-                issues.append( (file.fullname, e) )
+                issues.append( (file.fullname, "not found") )
+
+            self.close_stream(data)
+
+            sum_bytes += size
 
             if r :
                 fi = self.update_status(file.fullname)