remove sync, move files to filehelper, add function download

README.rst

@@ -34,7 +34,7 @@ This project contains various helper about logging functions, unit tests and hel
 Versions
 --------
 
-* **0.9 - 2014/??/??**
+* **0.9 - 2015/??/??**
     * **add:** function to remove extra spaces in a file :func:`remove_extra_spaces <pyquickhelper.pycode.code_helper.remove_extra_spaces>`
     * **add:** function :func:`create_visual_diff_through_html_files <pyquickhelper.sync.visual_sync.create_visual_diff_through_html_files>`
     * **fix:** the setup does not need the file ``README.rst`` anymore
@@ -49,6 +49,7 @@ Versions
     * **add:** add simple statistics while generation the documentation
     * **add:** add function :func:`clone <pyquickhelper.loghelper.repositories.pygit_helper.clone>` and :func:`rebase <pyquickhelper.loghelper.repositories.pygit_helper.rebase>` to clone or pull rebase a git repository
     * **new:** function :func:`set_sphinx_variables <pyquickhelper.helpgen.default_conf.set_sphinx_variables>` to avoid copying the same configuration file over multiple projects
+    * **del:** remove folder *sync*, move everything to *filehelper*
 * **0.8 - 2014/11/03**
     * **add:** Python version is now checked, ImportError is raised if it used on Python 2
     * **new:** function :func:`run_doc_server <pyquickhelper.serverdoc.documentation_server.run_doc_server>` creates a local server to display documentation

_unittests/ut_filehelper/test_download.py

@@ -0,0 +1,33 @@
+"""
+@brief      test log(time=2s)
+"""
+
+import sys, os, unittest
+
+if "temp_" in os.path.abspath(__file__):
+    raise ImportError("this file should not be imported in that location: " + os.path.abspath(__file__))
+
+try :
+    import src
+except ImportError :
+    path = os.path.normpath(os.path.abspath( os.path.join( os.path.split(__file__)[0], "..", "..")))
+    if path not in sys.path : sys.path.append (path)
+    import src
+
+from src.pyquickhelper import download, get_temp_folder, fLOG
+
+
+class TestDownload (unittest.TestCase):
+
+    def test_download(self) :
+        fLOG (__file__, self._testMethodName, OutputPrint = __name__ == "__main__")
+        fold = get_temp_folder(__file__,"temp_download")
+        url = "https://docs.python.org/3.5/library/ftplib.html"
+        f = download(url, fold)
+        fLOG(f)
+        assert os.path.exists(f)
+        assert f.endswith("ftplib.html")
+
+
+if __name__ == "__main__"  :
+    unittest.main ()

_unittests/ut_helpgen/test_utils_sphinxdoc.py

@@ -13,10 +13,10 @@
     if path not in sys.path : sys.path.append(path)
     import src
 
-from src.pyquickhelper.loghelper.flog           import fLOG
-from src.pyquickhelper.sync.synchelper          import explore_folder
+from src.pyquickhelper.loghelper.flog               import fLOG
+from src.pyquickhelper.filehelper.synchelper        import explore_folder
+from src.pyquickhelper.loghelper.pyrepo_helper      import SourceRepository
 import src.pyquickhelper.helpgen.utils_sphinx_doc as utils_sphinx_doc
-from src.pyquickhelper.loghelper.pyrepo_helper  import SourceRepository
 
 class TestSphinxDoc (unittest.TestCase):
 

_unittests/ut_helpgen/test_utils_sphinxdoc_full.py

@@ -13,8 +13,8 @@
     import src
 
 from src.pyquickhelper.loghelper.flog           import fLOG, removedirs
+from src.pyquickhelper.filehelper.synchelper    import synchronize_folder
 import src.pyquickhelper.helpgen.utils_sphinx_doc as utils_sphinx_doc
-from src.pyquickhelper.sync.synchelper          import synchronize_folder
 
 
 class TestSphinxDocFull (unittest.TestCase):

_unittests/ut_loghelper/test_log.py

@@ -25,7 +25,6 @@ def test_random_curve (self) :
         fLOG ("message", "ok", option1 = "k", option2 = 2,
                 LogFile=os.path.join(fold,"temp_log.txt"))
         assert os.path.exists (os.path.join(fold,"temp_log.txt"))
-        #os.remove ("hal_log.txt")
 
     def test_import_problem (self) :
         fLOG (__file__, self._testMethodName, OutputPrint = __name__ == "__main__")

_unittests/ut_loghelper/test_syncho.py

@@ -13,7 +13,7 @@
     import src
 
 from src.pyquickhelper.loghelper.flog import fLOG
-from src.pyquickhelper.sync.file_tree_node import FileTreeNode
+from src.pyquickhelper.filehelper.file_tree_node import FileTreeNode
 
 class TestFileCol (unittest.TestCase):
 

_unittests/ut_sync/test_folder.py

@@ -13,7 +13,7 @@
     import src
 
 from src.pyquickhelper.loghelper.flog import fLOG
-import src.pyquickhelper.sync.synchelper as foldermod
+import src.pyquickhelper.filehelper.synchelper as foldermod
 
 
 class TestFolder (unittest.TestCase):

_unittests/ut_sync/test_jsdifflib.py

@@ -13,8 +13,8 @@
     if path not in sys.path : sys.path.append (path)
     import src
 
-from src.pyquickhelper.loghelper.flog        import fLOG
-from src.pyquickhelper.sync.visual_sync      import create_visual_diff_through_html
+from src.pyquickhelper.loghelper.flog           import fLOG
+from src.pyquickhelper.filehelper.visual_sync   import create_visual_diff_through_html
 
 class TestJsDiffLib(unittest.TestCase):
 
@@ -33,7 +33,7 @@ def test_jsdifflib(self) :
                 warnings.warn("unable to test TestJsDiffLib.test_jsdifflib")
                 return
 
-        tt = os.path.split(src.pyquickhelper.sync.visual_sync.__file__)[0]
+        tt = os.path.split(src.pyquickhelper.filehelper.visual_sync.__file__)[0]
         ma = os.path.join(tt, "temp_difflibjs", "jsdifflib-master")
         if os.path.exists(ma):
             for i in os.listdir(ma) :

_unittests/ut_sync/test_syncho.py

@@ -12,9 +12,9 @@
     if path not in sys.path : sys.path.append (path)
     import src
 
-from src.pyquickhelper.loghelper.flog        import fLOG
-from src.pyquickhelper.sync.file_tree_node   import FileTreeNode
-from src.pyquickhelper.sync.synchelper       import synchronize_folder, remove_folder
+from src.pyquickhelper.loghelper.flog              import fLOG
+from src.pyquickhelper.filehelper.file_tree_node   import FileTreeNode
+from src.pyquickhelper.filehelper.synchelper       import synchronize_folder, remove_folder
 
 class TestFileCol (unittest.TestCase):
 

_unittests/ut_sync/test_syncho_bug.py

@@ -14,7 +14,7 @@
     import src
 
 from src.pyquickhelper.loghelper.flog        import fLOG
-from src.pyquickhelper.sync.synchelper       import synchronize_folder, remove_folder
+from src.pyquickhelper.filehelper.synchelper import synchronize_folder, remove_folder
 
 class TestSyncFolder (unittest.TestCase):
 

_unittests/ut_sync/test_syncho_hash.py

@@ -13,7 +13,7 @@
     import src
 
 from src.pyquickhelper.loghelper.flog        import fLOG
-from src.pyquickhelper.sync.synchelper       import synchronize_folder, remove_folder
+from src.pyquickhelper.filehelper.synchelper import synchronize_folder, remove_folder
 
 class TestSynchoHash (unittest.TestCase):
 

src/pyquickhelper/__init__.py

@@ -33,18 +33,19 @@ def check():
 from .funcwin.frame_function            import open_window_function
 from .funcwin.main_window               import main_loop_functions
 from .loghelper.convert_helper          import str_to_datetime
-from .sync.synchelper                   import explore_folder, synchronize_folder, has_been_updated, remove_folder, gzip_files, zip_files
+from .filehelper.synchelper             import explore_folder, synchronize_folder, has_been_updated, remove_folder
 from .pandashelper.readh                import read_url
 from .pandashelper.tblformat            import df_to_rst, df_to_html
 from .pandashelper.tblfunction          import isempty, isnan
 from .helpgen                           import get_help_usage
 from .helpgen.sphinx_main               import generate_help_sphinx, process_notebooks
 from .ipythonhelper.kindofcompletion    import AutoCompletion, AutoCompletionFile
-from .sync.visual_sync                  import create_visual_diff_through_html, create_visual_diff_through_html_files
+from .filehelper.visual_sync            import create_visual_diff_through_html, create_visual_diff_through_html_files
 from .serverdoc.documentation_server    import run_doc_server
 from .pycode.code_helper                import remove_extra_spaces, remove_extra_spaces_folder
 from .ipythonhelper.html_forms          import open_html_form
 from .ipythonhelper.magic_parser        import MagicCommandParser
 from .helpgen.utils_sphinx_config       import NbImage
-from .unittests.utils_tests             import main_wrapper_tests
-from .loghelper.repositories.gitlab_helper import GitLabAPI, GitLabException
+from .loghelper.repositories.gitlab_helper import GitLabAPI, GitLabException
+from .filehelper.internet_helper        import download
+from .unittests.utils_tests             import get_temp_folder, main_wrapper_tests

src/pyquickhelper/filehelper/compression_helper.py

@@ -0,0 +1,64 @@
+"""
+@file
+@brief Functions about compressing files.
+"""
+
+import os, re, zipfile, datetime, gzip
+
+from ..loghelper.flog import noLOG
+
+def zip_files (filename, fileSet, fLOG = noLOG) :
+    """
+    put all files from an iterator in a zip file
+
+    @param      filename        final zip file
+    @param      fileSet         iterator on file to add
+    @param      fLOG            logging function
+    @return                     number of added files
+    """
+    nb = 0
+    a1980 = datetime.datetime(1980,1,1)
+    with zipfile.ZipFile(filename, 'w') as myzip:
+        for file in fileSet :
+            st    = os.stat(file)
+            atime = datetime.datetime.fromtimestamp(st.st_atime)
+            mtime = datetime.datetime.fromtimestamp(st.st_mtime)
+            if atime < a1980 or mtime < a1980 :
+                new_mtime = st.st_mtime + (4*3600) #new modification time
+                while datetime.datetime.fromtimestamp(new_mtime) < a1980 :
+                    new_mtime += (4*3600) #new modification time
+
+                fLOG("zip_files: changing time timestamp for file ", file)
+                os.utime(file,(st.st_atime,new_mtime))
+
+            myzip.write(file)
+            nb += 1
+    return nb
+
+def gzip_files (filename_gz, fileSet, fLOG = noLOG, filename_zip = None) :
+    """
+    put all files from an iterator in a zip file and then in a gzip file
+
+    @param      filename_gz     final gzip file (double compression, extension should something like .zip.gz)
+    @param      filename_zip    temporary zip file (will be removed after the zipping unless it is different from None)
+    @param      fileSet         iterator on file to add
+    @param      log             log function
+    @return                     number of added files
+    """
+    if filename_zip is None :
+        zipf = filename_gz + ".temp.zip"
+    else : zipf = filename_zip
+    nb = zip_files (zipf, fileSet, fLOG = fLOG)
+
+    f = gzip.open(filename_gz, 'wb')
+    with open(zipf, "rb") as gr :
+        bb = gr.read(1000000)
+        while len(bb) > 0 :
+            f.write(bb)
+            bb = gr.read(1000000)
+    f.close()
+
+    if filename_zip is None :
+        os.remove (zipf)
+
+    return nb

src/pyquickhelper/filehelper/fexceptions.py

@@ -0,0 +1,10 @@
+"""
+@file
+@brief exceptions
+"""
+
+class FileException(Exception):
+    """
+    exception related to files
+    """
+    pass

src/pyquickhelper/filehelper/internet_helper.py

@@ -0,0 +1,96 @@
+"""
+@file
+@brief Gather functions about downloading from internet, ...
+"""
+
+import os, urllib, urllib.request, sys
+
+from ..loghelper.flog import noLOG, _get_file_url
+from .fexceptions import FileException
+
+def download (url, path_download = ".", outfile = None, fLOG = noLOG) :
+    """
+    Download a file
+
+    If url is an url, downloads the file and return the downloaded
+    If it has already been downloaded, it is not downloaded again
+    The function raises an exception if the url does not contain ``http://`` or ``https://`` or ``ftp://``.
+
+    @param      url                 url
+    @param      path_download       download the file here
+    @param      outfile             see below
+    @param      fLOG                logging function
+    @return                         the filename
+
+    If *outfile* is None, the function will give a relative name
+    based on the last part of the url.
+    If *outfile* is "", the function will remove every weird character.
+    If *outfile* is not null, the function will use it. It will be relative to
+    the current folder and not *path_download*.
+
+    .. versionadded:: 0.9
+
+    """
+    lurl = url.lower()
+    if "http://" in lurl or "https://" in lurl or "ftp://":
+        if outfile is None: dest = os.path.join(path_download, os.path.split(url)[-1])
+        elif outfile == "" : dest = _get_file_url (url, path_download)
+        else: dest = outfile
+
+        down = False
+        nyet = dest + ".notyet"
+
+        if os.path.exists (dest) and not os.path.exists (nyet) :
+            try :
+                f1      = urllib.urlopen (url)
+                down    = _first_more_recent (f1, dest)
+                newdate = down
+                f1.close ()
+            except IOError as e :
+                raise FileException("unable to access url: " + url) from e
+        else :
+            down    = True
+            newdate = False
+
+        if down :
+            if newdate :    fLOG (" downloading (updated) ", url)
+            else :          fLOG (" downloading ", url)
+
+            if len (url) > 4 and url [-4].lower () in [".txt", ".csv", ".tsv", ".log"] :
+                fLOG ("creating text file ", dest)
+                format = "w"
+            else :
+                fLOG ("creating binary file ", dest)
+                format = "wb"
+
+            if os.path.exists (nyet) :
+                size    = os.stat (dest).st_size
+                fLOG ("resume downloading (stop at", size, ") from ", url)
+                request = urllib.request.Request(url)
+                request.add_header("Range", "bytes=%d-" % size)
+                fu      = urllib.request.urlopen (request)
+                f       = open (dest, format.replace ("w", "a"))
+            else :
+                fLOG ("downloading ", url)
+                request = urllib.request.Request(url)
+                fu      = urllib.request.urlopen (url)
+                f       = open (dest, format)
+
+            open (nyet, "w").close ()
+            c       = fu.read (2**21)
+            size    = 0
+            while len (c) > 0 :
+                size += len (c)
+                fLOG("    size", size)
+                f.write (c)
+                f.flush ()
+                c = fu.read (2**21)
+            fLOG ("end downloading")
+            f.close ()
+            fu.close ()
+            os.remove (nyet)
+
+        url = dest
+        return url
+    else:
+        raise FileException("This url does not seem to be one: " + url)