feat(download): Add zip_all function

autotest/t012_test.py

@@ -109,6 +109,8 @@ def test_compile_gsflow():
     # download and compile GSFLOW
     pymake.build_program(target=target,
                          include_subdirs=True,
+                         fflags='-O1 -fno-second-underscore',
+                         cflags='-O1',
                          download_dir=dstpth,
                          exe_dir=dstpth)
     return

autotest/t999_test.py

@@ -57,7 +57,7 @@ def test_previous_assets():
     return
 
 
-def test_download_and_unzip():
+def test_download_and_unzip_and_zip():
     exclude_files = ['code.json']
     pth = './temp/t999'
     pymake.getmfexes(pth)
@@ -67,6 +67,35 @@ def test_download_and_unzip():
             errmsg = '{} not executable'.format(fpth)
             assert which(fpth) is not None, errmsg
 
+    # zip up exe's using files
+    zip_pth = os.path.join('temp', 'ziptest01.zip')
+    success = pymake.zip_all(zip_pth,
+                            file_pths=[os.path.join(pth, e)
+                                       for e in os.listdir(pth)])
+    assert success, 'could not create zipfile using file names'
+    os.remove(zip_pth)
+
+    # zip up exe's using directories
+    zip_pth = os.path.join('temp', 'ziptest02.zip')
+    success = pymake.zip_all(zip_pth, dir_pths=pth)
+    assert success, 'could not create zipfile using directories'
+    os.remove(zip_pth)
+
+    # zip up exe's using directories and a pattern
+    zip_pth = os.path.join('temp', 'ziptest03.zip')
+    success = pymake.zip_all(zip_pth, dir_pths=pth, patterns='mf')
+    assert success, 'could not create zipfile using directories and a pattern'
+    os.remove(zip_pth)
+
+    # zip up exe's using files and directories
+    zip_pth = os.path.join('temp', 'ziptest04.zip')
+    success = pymake.zip_all(zip_pth,
+                             file_pths=[os.path.join(pth, e)
+                                        for e in os.listdir(pth)],
+                             dir_pths=pth)
+    assert success, 'could not create zipfile using files and directories'
+    os.remove(zip_pth)
+
     # clean up exe's
     for f in os.listdir(pth):
         fpth = os.path.join(pth, f)
@@ -83,7 +112,7 @@ def test_download_and_unzip():
 
 
 if __name__ == '__main__':
+    test_download_and_unzip_and_zip()
     test_previous_assets()
     test_latest_version()
     test_latest_assets()
-    test_download_and_unzip()

examples/buildall.bat

@@ -1,5 +1,5 @@
 rem 64-bit executables
-python buildall.py --appdir win64 --ifort --icl 
-python buildall.py --appdir win32 --ifort --icl --ia32
+python buildall.py --appdir win64 --ifort --icl --zip win64.zip --keep
+python buildall.py --appdir win32 --ifort --icl --ia32 --zip win32.zip --keep
 
 pause

examples/buildall.py

@@ -15,6 +15,9 @@ def build_all():
         # build code json
         pymake.usgs_program_data.export_json(current=True)
 
+        # compress files
+        pymake.compress_apps()
+
 
 if __name__ == '__main__':
     build_all()

pymake/__init__.py

@@ -7,7 +7,7 @@
 from .pymake import main, parser, get_ordered_srcfiles
 from .dag import order_source_files, order_c_source_files, get_f_nodelist
 from .download import download_and_unzip, getmfexes, \
-    repo_latest_version, get_repo_assets
+    repo_latest_version, get_repo_assets, zip_all
 from .visualize import make_plots
 from .autotest import setup, setup_comparison, teardown, \
     get_namefiles, get_entries_from_namefile, \
@@ -16,4 +16,4 @@
     compare_stages, compare, \
     setup_mf6, setup_mf6_comparison, get_mf6_comparison, get_mf6_files
 from .build_program import build_program, build_apps, build_replace, \
-    set_compiler, set_bindir
+    set_compiler, set_bindir, compress_apps

pymake/build_program.py

@@ -12,7 +12,7 @@
     from distutils.spawn import find_executable as which
 
 from .pymake import main
-from .download import download_and_unzip
+from .download import download_and_unzip, zip_all
 from .usgsprograms import usgs_program_data
 
 
@@ -283,7 +283,9 @@ def set_fflags(target, fc='gfortran'):
             else:
                 fflags = '-fp-model source'
         elif fc == 'gfortran':
-            fflags = '-fno-second-underscore'
+            fflags = '-O1 -fno-second-underscore'
+            # if 'win32' in sys.platform.lower():
+            #     fflags += ' -Bstatic -Wall'
 
     # add additional fflags from the command line
     for idx, arg in enumerate(sys.argv):
@@ -331,11 +333,14 @@ def set_cflags(target, cc='gcc'):
     elif target == 'gsflow':
         if cc == 'icc' or cc == 'icl':
             if 'win32' in sys.platform.lower():
-                cflags = '/Wall /D_CRT_SECURE_NO_WARNINGS'
+                cflags = '/D_CRT_SECURE_NO_WARNINGS'
             else:
-                cflags = '-D_UF -Wall'
+                cflags = '-D_UF'
         elif cc == 'gcc':
-            cflags = '-D_UF'
+            # cflags = '-O -D_UF'
+            cflags = '-O1'
+            # if 'win32' in sys.platform.lower():
+            #     cflags += ' -Bstatic -Wall'
 
     # add additional cflags from the command line
     for idx, arg in enumerate(sys.argv):
@@ -393,6 +398,10 @@ def set_syslibs(target, fc, cc):
         if 'win32' not in sys.platform.lower():
             if 'ifort' in fc:
                 syslibs = '-nofor_main'
+        # else:
+        #     if 'gfortran' in fc:
+        #         if 'win32' in sys.platform.lower():
+        #             syslibs = '-lgfortran -lgcc -lm'
 
     # write syslibs
     msg = '{} will use the following predefined syslibs:\n'.format(target)
@@ -532,6 +541,34 @@ def set_extrafiles(target, download_dir):
     return extrafiles
 
 
+def set_zip():
+    """
+    Set file path for zip file
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+    zip_pth : str
+        path for zip file
+
+    """
+    zip_pth = None
+    for idx, arg in enumerate(sys.argv):
+        if arg.lower() == '--zip':
+            if idx < len(sys.argv) - 1:
+                zip_pth = sys.argv[idx + 1]
+
+    # write c/c++ flags
+    if zip_pth is not None:
+        msg = 'compiled executibles will be compressed to:\n'
+        msg += '    {}\n'.format(zip_pth)
+        print(msg)
+
+    return zip_pth
+
+
 def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True,
                   expedite=False, dryrun=False, double=False, debug=False,
                   include_subdirs=False,
@@ -1003,6 +1040,55 @@ def build_apps(targets=None):
     return returncode
 
 
+def compress_apps(targets=None):
+    """
+
+    Parameters
+    ----------
+    targets : str or list of str
+
+    Returns
+    -------
+    returncode : int
+
+    """
+    # intialize the return code
+    returncode = 0
+
+    # get the specified zip file path
+    zip_pth = set_zip()
+
+    # compress the compiled executables
+    if zip_pth is not None:
+        # determine targets if not defined
+        if targets is None:
+            targets = build_targets()
+
+        # add code.json
+        if 'code.json' not in targets:
+            targets.append('code.json')
+
+        # delete the zip file if it exists
+        if os.path.exists(zip_pth):
+            print("Deleting existing zipfile '{}'".format(zip_pth))
+            os.remove(zip_pth)
+
+        # set the bin dir
+        bindir = set_bindir()
+
+        # compress the compiled executables
+        msg = "Compressing files in '{}' ".format(bindir) + \
+              "directory to zip file '{}'".format(zip_pth)
+        print(msg)
+        success = zip_all(zip_pth, dir_pths=bindir, patterns=targets)
+
+        # set return code
+        if not success:
+            returncode = 1
+
+    return returncode
+
+
 # routines for updating source files locations and to compile
 # with gfortran, gcc, and g++
 def update_triangle_files(srcdir, fc, cc, arch, double):

pymake/download.py

@@ -4,11 +4,11 @@
 import sys
 import shutil
 import timeit
-from zipfile import ZipFile, ZipInfo
+from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
 import tarfile
 
 
-class MyZipFile(ZipFile):
+class pymakeZipFile(ZipFile):
     """
     ZipFile file attributes are not being preserved.  This preserves file
     attributes as described here
@@ -43,6 +43,71 @@ def extractall(self, path=None, members=None, pwd=None):
         for zipinfo in members:
             self.extract(zipinfo, path, pwd)
 
+    @staticmethod
+    def compressall(path, file_pths=None, dir_pths=None, patterns=None):
+
+        # create an empty list
+        if file_pths is None:
+            file_pths = []
+        # convert files to a list
+        else:
+            if isinstance(file_pths, str):
+                file_pths = [file_pths]
+            elif isinstance(file_pths, tuple):
+                file_pths = list(file_pths)
+
+        # remove directories from the file list
+        if len(file_pths) > 0:
+            file_pths = [e for e in file_pths if os.path.isfile(e)]
+
+        # convert dirs to a list if a str (a tuple is allowed)
+        if dir_pths is None:
+            dir_pths = []
+        else:
+            if isinstance(dir_pths, str):
+                dir_pths = [dir_pths]
+
+        # convert find to a list if a str (a tuple is allowed)
+        if patterns is not None:
+            if isinstance(patterns, str):
+                patterns = [patterns]
+
+        # walk through dirs and add files to the list
+        for dir_pth in dir_pths:
+            for dirname, subdirs, files in os.walk(dir_pth):
+                for filename in files:
+                    fpth = os.path.join(dirname, filename)
+                    # add the file if it does not exist in file_pths
+                    if fpth not in file_pths:
+                        file_pths.append(fpth)
+
+        # remove file_paths that do not match the patterns
+        if patterns is not None:
+            tlist = []
+            for file_pth in file_pths:
+                if any(p in os.path.basename(file_pth) for p in patterns):
+                    tlist.append(file_pth)
+            file_pths = tlist
+
+        # write the zipfile
+        success = True
+        if len(file_pths) > 0:
+            zf = ZipFile(path, 'w', ZIP_DEFLATED)
+
+            # write files to zip file
+            for file_pth in file_pths:
+                arcname = os.path.basename(file_pth)
+                zf.write(file_pth, arcname=arcname)
+
+            # close the zip file
+            zf.close()
+        else:
+            msg = 'No files to add to the zip file'
+            print(msg)
+            success = False
+
+        return success
+
 
 def download_and_unzip(url, pth='./', delete_zip=True, verify=True,
                        timeout=30, nattempts=10, chunk_size=2048000):
@@ -120,7 +185,7 @@ def download_and_unzip(url, pth='./', delete_zip=True, verify=True,
     # Unzip the file, and delete zip file if successful.
     if 'zip' in os.path.basename(file_name) or \
             'exe' in os.path.basename(file_name):
-        z = MyZipFile(file_name)
+        z = pymakeZipFile(file_name)
         try:
             print('Extracting the zipfile...')
             z.extractall(pth)
@@ -140,6 +205,36 @@ def download_and_unzip(url, pth='./', delete_zip=True, verify=True,
     print('Done downloading and extracting...\n')
 
 
+def zip_all(path, file_pths=None, dir_pths=None, patterns=None):
+    """
+    compress all files in the user-provided list of file paths and directory
+    paths that match the provided file patterns
+
+    Parameters
+    ----------
+    path : str
+        path of the zip file that will be created
+
+    file_pths : str or list
+        file path or list of file paths to be compressed
+
+    dir_pths : str or list
+        directory path or list of directory paths to search for files that
+        will be compressed
+
+    patterns : str or list
+        file pattern or list of file patterns s to match to when creating a
+        list of files that will be compressed
+
+    Returns
+    -------
+
+
+    """
+    return pymakeZipFile.compressall(path, file_pths=file_pths,
+                                     dir_pths=dir_pths, patterns=patterns)
+
+
 def get_default_repo():
     """
     Return the default repo name
@@ -268,8 +363,10 @@ def repo_json(github_repo, tag_name=None):
             msg = 'Could not get release catalog from ' + request_url
             raise Exception(msg)
 
-    msg = "Requesting asset data for tag_name '{}' ".format(tag_name) + \
-          "from: {}".format(request_url)
+    msg = "Requesting asset data "
+    if tag_name is not None:
+        msg += "for tag_name '{}' ".format(tag_name)
+    msg += "from: {}".format(request_url)
     print(msg)
 
     # process the request