add logging information while synchronizing big folders

_unittests/ut_sync/test_syncho_hash.py

@@ -55,7 +55,8 @@ def filter_copy(file):
                         hash_size = 0,
                         repo1 = True,
                         filter_copy = filter_copy,
-                        file_date = file_date)
+                        file_date = file_date,
+                        log1 = True)
 
         assert os.path.exists(file_date)
         assert os.path.exists(os.path.join(troi,"sub","filetwo.txt"))

src/pyquickhelper/sync/file_tree_node.py

@@ -72,7 +72,8 @@ def __init__ (self, root,
                         level               = 0,
                         parent              = None,
                         repository          = False,
-                        log                 = False) :
+                        log                 = False,
+                        log1                = False) :
         """
         define a file, relative to a root
         @param      root            root (it must exist)
@@ -85,6 +86,7 @@ def __init__ (self, root,
         @param      parent          link to the parent
         @param      repository      use SVN or GIT if True
         @param      log             log every explored folder
+        @param      log1            intermediate logs (first level)
         """
         self._root      = root
         self._file      = None if file == None else file
@@ -95,6 +97,7 @@ def __init__ (self, root,
         self._level     = level
         self._parent    = parent
         self._log       = log
+        self._log1      = log1
         self.module     = None
 
         if not os.path.exists (root) : raise PQHException ("path %s does not exist"  % root)
@@ -216,6 +219,8 @@ def _fill (self, filter, repository) :
         self._children = []
         for a in all :
             #af = os.path.join (fi, a)
+            if self._log1:
+                fLOG("[FileTreeNode], entering", a)
             fu = os.path.join (full, a)
             if filter == None or filter (self._root, fi, a, os.path.isdir (fu)) :
                 try :

src/pyquickhelper/sync/file_tree_status.py

@@ -237,17 +237,22 @@ def has_been_modified_and_reason (self, file) :
             self.modifiedFile.append( (file, reason) )
         return res, reason
 
-    def difference(self, files, u4 = False):
+    def difference(self, files, u4 = False, nlog = None):
         """
         goes through the list of files and tells which one has changed
         
         @param      files   @see cl FileTreeNode
         @param      u4      @see cl FileTreeNode (changes the output)
+        @param      nlog    if not None, print something every ``nlog`` processed files
         @return             iterator on files which changed
         """
         if u4 :
+            nb = 0
             for file in files :
                 if file._file == None : continue
+                nb += 1
+                if nlog != None and nb % nlog == 0 :
+                    self.LOG("[FileTreeStatus], processed", nb, "files")
                 full = file.fullname
                 r, reason = self.has_been_modified_and_reason(full)
                 if r :
@@ -261,7 +266,11 @@ def difference(self, files, u4 = False):
                     r = ( "==", file._file, file, None )
                     yield r
         else :
+            nb = 0
             for file in files :
+                nb += 1
+                if nlog != None and nb % nlog == 0 :
+                    self.LOG("[FileTreeStatus], processed", nb, "files")
                 full = file.fullname
                 if self.has_been_modified_and_reason(file):
                     yield file

src/pyquickhelper/sync/synchelper.py

@@ -142,7 +142,8 @@ def synchronize_folder (   p1,
                     filter_copy     = None,
                     avoid_copy      = False,
                     operations      = None,
-                    file_date       = None) :
+                    file_date       = None,
+                    log1            = False) :
     """
     synchronize two folders (or copy if the second is empty), it only copies more recent files.
     
@@ -165,6 +166,7 @@ def synchronize_folder (   p1,
     @param      avoid_copy          if True, just return the list of files which should be copied but does not do the copy
     @param      operations          if None, this function is called with the following parameters: ``operations(op,n1,n2)``
     @param      file_date           filename which contains information about when the last sync was done
+    @param      log1                @see cl FileTreeNode
     @return                         list of operations done by the function
                                         list of 3-uple: action, source_file, dest_file
                                         
@@ -205,11 +207,12 @@ def pr_filter (root, path, f, d) :
     node1 = FileTreeNode (f1, filter = pr_filter, repository = repo1, log = True)
     fLOG ("     number of found files (p1)", len (node1), node1.max_date ())
     if file_date != None and os.path.exists(file_date) :
+        log1n = 1000 if log1  else None
         status = FileTreeStatus(file_date)
-        res = list(status.difference(node1, u4=True))
+        res = list(status.difference(node1, u4=True, nlog = log1n))
     else :
         fLOG ("   exploring ", f2)
-        node2 = FileTreeNode (f2, filter = pr_filter, repository = repo2, log = True)
+        node2 = FileTreeNode (f2, filter = pr_filter, repository = repo2, log = True, log1 = True)
         fLOG ("     number of found files (p2)", len (node2), node2.max_date ())
         res = node1.difference (node2, hash_size = hash_size)
         status = None