From b288afb0fb453acf7d5a6375134890bf8a87b178 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 29 Apr 2020 13:33:28 -0400 Subject: [PATCH 1/3] feat(download): Add zip_all function zip_all() function to zip executables after build. Add sutra to usgsprograms.txt --- autotest/t999_test.py | 33 ++++++++++++- examples/buildall.bat | 4 +- examples/buildall.py | 3 ++ pymake/__init__.py | 4 +- pymake/build_program.py | 78 ++++++++++++++++++++++++++++- pymake/download.py | 107 ++++++++++++++++++++++++++++++++++++++-- pymake/usgsprograms.txt | 1 + 7 files changed, 218 insertions(+), 12 deletions(-) diff --git a/autotest/t999_test.py b/autotest/t999_test.py index 59e3c35a..85dafa8e 100644 --- a/autotest/t999_test.py +++ b/autotest/t999_test.py @@ -57,7 +57,7 @@ def test_previous_assets(): return -def test_download_and_unzip(): +def test_download_and_unzip_and_zip(): exclude_files = ['code.json'] pth = './temp/t999' pymake.getmfexes(pth) @@ -67,6 +67,35 @@ def test_download_and_unzip(): errmsg = '{} not executable'.format(fpth) assert which(fpth) is not None, errmsg + # zip up exe's using files + zip_pth = os.path.join('temp', 'ziptest01.zip') + success = pymake.zip_all(zip_pth, + file_pths=[os.path.join(pth, e) + for e in os.listdir(pth)]) + assert success, 'could not create zipfile using file names' + os.remove(zip_pth) + + # zip up exe's using directories + zip_pth = os.path.join('temp', 'ziptest02.zip') + success = pymake.zip_all(zip_pth, dir_pths=pth) + assert success, 'could not create zipfile using directories' + os.remove(zip_pth) + + # zip up exe's using directories and a pattern + zip_pth = os.path.join('temp', 'ziptest03.zip') + success = pymake.zip_all(zip_pth, dir_pths=pth, patterns='mf') + assert success, 'could not create zipfile using directories and a pattern' + os.remove(zip_pth) + + # zip up exe's using files and directories + zip_pth = os.path.join('temp', 'ziptest04.zip') + success = pymake.zip_all(zip_pth, + file_pths=[os.path.join(pth, e) + for e in os.listdir(pth)], + dir_pths=pth) + assert success, 'could not create zipfile using files and directories' + os.remove(zip_pth) + # clean up exe's for f in os.listdir(pth): fpth = os.path.join(pth, f) @@ -83,7 +112,7 @@ def test_download_and_unzip(): if __name__ == '__main__': + test_download_and_unzip_and_zip() test_previous_assets() test_latest_version() test_latest_assets() - test_download_and_unzip() diff --git a/examples/buildall.bat b/examples/buildall.bat index f7e05dc0..b75489d6 100644 --- a/examples/buildall.bat +++ b/examples/buildall.bat @@ -1,5 +1,5 @@ rem 64-bit executables -python buildall.py --appdir win64 --ifort --icl -python buildall.py --appdir win32 --ifort --icl --ia32 +python buildall.py --appdir win64 --ifort --icl --zip win64.zip +python buildall.py --appdir win32 --ifort --icl --ia32 --zip win32.zip pause diff --git a/examples/buildall.py b/examples/buildall.py index ef6fa932..eb6f2cd9 100644 --- a/examples/buildall.py +++ b/examples/buildall.py @@ -15,6 +15,9 @@ def build_all(): # build code json pymake.usgs_program_data.export_json(current=True) + # compress files + pymake.compress_apps() + if __name__ == '__main__': build_all() diff --git a/pymake/__init__.py b/pymake/__init__.py index e58b7592..42577ff0 100644 --- a/pymake/__init__.py +++ b/pymake/__init__.py @@ -7,7 +7,7 @@ from .pymake import main, parser, get_ordered_srcfiles from .dag import order_source_files, order_c_source_files, get_f_nodelist from .download import download_and_unzip, getmfexes, \ - repo_latest_version, get_repo_assets + repo_latest_version, get_repo_assets, zip_all from .visualize import make_plots from .autotest import setup, setup_comparison, teardown, \ get_namefiles, get_entries_from_namefile, \ @@ -16,4 +16,4 @@ compare_stages, compare, \ setup_mf6, setup_mf6_comparison, get_mf6_comparison, get_mf6_files from .build_program import build_program, build_apps, build_replace, \ - set_compiler, set_bindir + set_compiler, set_bindir, compress_apps diff --git a/pymake/build_program.py b/pymake/build_program.py index 7d90d98c..ecd5c10f 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -12,7 +12,7 @@ from distutils.spawn import find_executable as which from .pymake import main -from .download import download_and_unzip +from .download import download_and_unzip, zip_all from .usgsprograms import usgs_program_data @@ -532,6 +532,34 @@ def set_extrafiles(target, download_dir): return extrafiles +def set_zip(): + """ + Set file path for zip file + + Parameters + ---------- + + Returns + ------- + zip_pth : str + path for zip file + + """ + zip_pth = None + for idx, arg in enumerate(sys.argv): + if arg.lower() == '--zip': + if idx < len(sys.argv) - 1: + zip_pth = sys.argv[idx + 1] + + # write c/c++ flags + if zip_pth is not None: + msg = 'compiled executibles will be compressed to:\n' + msg += ' {}\n'.format(zip_pth) + print(msg) + + return zip_pth + + def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, expedite=False, dryrun=False, double=False, debug=False, include_subdirs=False, @@ -1002,6 +1030,54 @@ def build_apps(targets=None): return returncode +def compress_apps(targets=None): + """ + + Parameters + ---------- + targets : str or list of str + + Returns + ------- + returncode : int + + """ + # intialize the return code + returncode = 0 + + # get the specified zip file path + zip_pth = set_zip() + + # compress the compiled executables + if zip_pth is not None: + # determine targets if not defined + if targets is None: + targets = build_targets() + + # add code.json + if 'code.json' not in targets: + targets.append('code.json') + + # delete the zip file if it exists + if os.path.exists(zip_pth): + print("Deleting existing zipfile '{}'".format(zip_pth)) + os.remove(zip_pth) + + # set the bin dir + bindir = set_bindir() + + # compress the compiled executables + msg = "Compressing files in '{}' ".format(bindir) + \ + "directory to zip file '{}'".format(zip_pth) + print(msg) + success = zip_all(zip_pth, dir_pths=bindir, patterns=targets) + + # set return code + if not success: + returncode = 1 + + return returncode + # routines for updating source files locations and to compile # with gfortran, gcc, and g++ diff --git a/pymake/download.py b/pymake/download.py index a4a90bfd..f3c3f190 100644 --- a/pymake/download.py +++ b/pymake/download.py @@ -4,11 +4,11 @@ import sys import shutil import timeit -from zipfile import ZipFile, ZipInfo +from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED import tarfile -class MyZipFile(ZipFile): +class pymakeZipFile(ZipFile): """ ZipFile file attributes are not being preserved. This preserves file attributes as described here @@ -43,6 +43,71 @@ def extractall(self, path=None, members=None, pwd=None): for zipinfo in members: self.extract(zipinfo, path, pwd) + @staticmethod + def compressall(path, file_pths=None, dir_pths=None, patterns=None): + + # create an empty list + if file_pths is None: + file_pths = [] + # convert files to a list + else: + if isinstance(file_pths, str): + file_pths = [file_pths] + elif isinstance(file_pths, tuple): + file_pths = list(file_pths) + + # remove directories from the file list + if len(file_pths) > 0: + file_pths = [e for e in file_pths if os.path.isfile(e)] + + # convert dirs to a list if a str (a tuple is allowed) + if dir_pths is None: + dir_pths = [] + else: + if isinstance(dir_pths, str): + dir_pths = [dir_pths] + + # convert find to a list if a str (a tuple is allowed) + if patterns is not None: + if isinstance(patterns, str): + patterns = [patterns] + + # walk through dirs and add files to the list + for dir_pth in dir_pths: + for dirname, subdirs, files in os.walk(dir_pth): + for filename in files: + fpth = os.path.join(dirname, filename) + # add the file if it does not exist in file_pths + if fpth not in file_pths: + file_pths.append(fpth) + + # remove file_paths that do not match the patterns + if patterns is not None: + tlist = [] + for file_pth in file_pths: + if any(p in os.path.basename(file_pth) for p in patterns): + tlist.append(file_pth) + file_pths = tlist + + # write the zipfile + success = True + if len(file_pths) > 0: + zf = ZipFile(path, 'w', ZIP_DEFLATED) + + # write files to zip file + for file_pth in file_pths: + arcname = os.path.basename(file_pth) + zf.write(file_pth, arcname=arcname) + + # close the zip file + zf.close() + else: + msg = 'No files to add to the zip file' + print(msg) + success = False + + return success + def download_and_unzip(url, pth='./', delete_zip=True, verify=True, timeout=30, nattempts=10, chunk_size=2048000): @@ -120,7 +185,7 @@ def download_and_unzip(url, pth='./', delete_zip=True, verify=True, # Unzip the file, and delete zip file if successful. if 'zip' in os.path.basename(file_name) or \ 'exe' in os.path.basename(file_name): - z = MyZipFile(file_name) + z = pymakeZipFile(file_name) try: print('Extracting the zipfile...') z.extractall(pth) @@ -140,6 +205,36 @@ def download_and_unzip(url, pth='./', delete_zip=True, verify=True, print('Done downloading and extracting...\n') +def zip_all(path, file_pths=None, dir_pths=None, patterns=None): + """ + compress all files in the user-provided list of file paths and directory + paths that match the provided file patterns + + Parameters + ---------- + path : str + path of the zip file that will be created + + file_pths : str or list + file path or list of file paths to be compressed + + dir_pths : str or list + directory path or list of directory paths to search for files that + will be compressed + + patterns : str or list + file pattern or list of file patterns s to match to when creating a + list of files that will be compressed + + Returns + ------- + + + """ + return pymakeZipFile.compressall(path, file_pths=file_pths, + dir_pths=dir_pths, patterns=patterns) + + def get_default_repo(): """ Return the default repo name @@ -268,8 +363,10 @@ def repo_json(github_repo, tag_name=None): msg = 'Could not get release catalog from ' + request_url raise Exception(msg) - msg = "Requesting asset data for tag_name '{}' ".format(tag_name) + \ - "from: {}".format(request_url) + msg = "Requesting asset data " + if tag_name is not None: + msg += "for tag_name '{}' ".format(tag_name) + msg += "from: {}".format(request_url) print(msg) # process the request diff --git a/pymake/usgsprograms.txt b/pymake/usgsprograms.txt index 5f2b0667..31ae5648 100644 --- a/pymake/usgsprograms.txt +++ b/pymake/usgsprograms.txt @@ -20,3 +20,4 @@ gridgen 1.0.02 True https://water.usgs.gov/water-resources/software/GR zonbud3 3.01 True https://water.usgs.gov/water-resources/software/ZONEBUDGET/zonbud3_01.exe Zonbud.3_01 Src True False crt 1.3.1 True https://water.usgs.gov/ogw/CRT/CRT_1.3.1.zip CRT_1.3.1 SOURCE True False gsflow 2.1.0 True https://water.usgs.gov/water-resources/software/gsflow/gsflow_2.1.0_linux.zip gsflow_2.1.0 src True False +sutra 3.0 True http://water.usgs.gov/water-resources/software/sutra/SUTRA_3_0_0.zip SutraSuite SUTRA_3_0/source True False From b4976d453d615da408ad89ea8ffe646d608aa0f6 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 29 Apr 2020 18:23:54 -0400 Subject: [PATCH 2/3] feat(download): Add zip_all function zip_all() function to zip executables after build. Add sutra to usgsprograms.txt --- autotest/t012_test.py | 2 ++ examples/buildall.bat | 4 +-- pymake/build_program.py | 18 +++++++--- pymake/pymake.py | 75 ++++++++++++++++++++++++++++------------- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/autotest/t012_test.py b/autotest/t012_test.py index b19a3195..ba9d8abe 100644 --- a/autotest/t012_test.py +++ b/autotest/t012_test.py @@ -109,6 +109,8 @@ def test_compile_gsflow(): # download and compile GSFLOW pymake.build_program(target=target, include_subdirs=True, + fflags='-O1 -fno-second-underscore', + cflags='-O1', download_dir=dstpth, exe_dir=dstpth) return diff --git a/examples/buildall.bat b/examples/buildall.bat index b75489d6..4d96817f 100644 --- a/examples/buildall.bat +++ b/examples/buildall.bat @@ -1,5 +1,5 @@ rem 64-bit executables -python buildall.py --appdir win64 --ifort --icl --zip win64.zip -python buildall.py --appdir win32 --ifort --icl --ia32 --zip win32.zip +python buildall.py --appdir win64 --ifort --icl --zip win64.zip --keep +python buildall.py --appdir win32 --ifort --icl --ia32 --zip win32.zip --keep pause diff --git a/pymake/build_program.py b/pymake/build_program.py index ecd5c10f..c82ea0dd 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -283,7 +283,9 @@ def set_fflags(target, fc='gfortran'): else: fflags = '-fp-model source' elif fc == 'gfortran': - fflags = '-fno-second-underscore' + fflags = '-O1 -fno-second-underscore' + # if 'win32' in sys.platform.lower(): + # fflags += ' -Bstatic -Wall' # add additional fflags from the command line for idx, arg in enumerate(sys.argv): @@ -331,11 +333,14 @@ def set_cflags(target, cc='gcc'): elif target == 'gsflow': if cc == 'icc' or cc == 'icl': if 'win32' in sys.platform.lower(): - cflags = '/Wall /D_CRT_SECURE_NO_WARNINGS' + cflags = '/D_CRT_SECURE_NO_WARNINGS' else: - cflags = '-D_UF -Wall' + cflags = '-D_UF' elif cc == 'gcc': - cflags = '-D_UF' + # cflags = '-O -D_UF' + cflags = '-O1' + # if 'win32' in sys.platform.lower(): + # cflags += ' -Bstatic -Wall' # add additional cflags from the command line for idx, arg in enumerate(sys.argv): @@ -393,6 +398,10 @@ def set_syslibs(target, fc, cc): if 'win32' not in sys.platform.lower(): if 'ifort' in fc: syslibs = '-nofor_main' + # else: + # if 'gfortran' in fc: + # if 'win32' in sys.platform.lower(): + # syslibs = '-lgfortran -lgcc -lm' # write syslibs msg = '{} will use the following predefined syslibs:\n'.format(target) @@ -1030,6 +1039,7 @@ def build_apps(targets=None): return returncode + def compress_apps(targets=None): """ diff --git a/pymake/pymake.py b/pymake/pymake.py index 86f5acd9..fcd2ac46 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -449,10 +449,13 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # define the platform platform = sys.platform - # For horrible windows issue + # set shellflg for popen shellflg = False - if platform == 'win32': - shellflg = True + + # jdh commented out 4/29/2020 since is working with python 3 + # For horrible windows issue + # if platform == 'win32': + # shellflg = True # define the OS macro for gfortran if platform == 'win32': @@ -498,7 +501,7 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, compileflags.append('-fPIC') if debug: - compileflags += ['-fcheck=all', '-fbounds-check'] + compileflags += ['-fcheck=all', '-fbounds-check', '-Wall'] lflag = flag_available('-ffpe-trap') if lflag: compileflags.append( @@ -514,6 +517,10 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # add fbacktrace to debug and release versions compileflags.append('-fbacktrace') + # add static + if sys.platform == 'win32': + compileflags.append('-Bstatic') + # add double precision switches if double: compileflags.append('-fdefault-real-8') @@ -557,6 +564,8 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, cflags += [opt] if cc.startswith('g'): + if sys.platform == 'win32': + cflags += ['-Bstatic'] if debug: lflag = flag_available('-Wall') if lflag: @@ -564,17 +573,33 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, else: pass - # syslibs + # determine if any c, cpp or fortran files + iscfiles = False + isfortranfiles = False + for srcfile in srcfiles: + ext = os.path.splitext(srcfile)[1].lower() + if ext in ['.c', '.cpp']: # mja + iscfiles = True + elif ext in ['.f', '.for', '.f90', '.fpp']: + isfortranfiles = True + # reset syslibs for windows if sys.platform == 'win32': syslibs = [] + if isfortranfiles: + syslibs.append('-lgfortran') + if iscfiles: + syslibs.append('-lgcc') + syslibs.append('-lm') # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # code that is linked to C/C++ code - # -D_UF defines UNIX naming conventions for mixed language compilation. - use_iso_c = get_iso_c(srcfiles) - if not use_iso_c: - cflags.append('-D_UF') + # code that is linked to C/C++ code. Only needed if there are + # any fortran files. -D_UF defines UNIX naming conventions for + # mixed language compilation. + if isfortranfiles: + use_iso_c = get_iso_c(srcfiles) + if not use_iso_c: + cflags.append('-D_UF') # build object files print('\nCompiling object files for ' + @@ -591,7 +616,8 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, for srcfile in srcfiles: cmdlist = [] iscfile = False - if srcfile.endswith('.c') or srcfile.endswith('.cpp'): # mja + ext = os.path.splitext(srcfile)[1].lower() + if ext in ['.c', '.cpp']: # mja iscfile = True cmdlist.append(cc) # mja for switch in cflags: # mja @@ -601,9 +627,14 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, for switch in compileflags: cmdlist.append(switch) - # add search path for any header files - for sd in searchdir: - cmdlist.append('-I{}'.format(sd)) + # add search path for any c and c++ header files + if iscfile: + for sd in searchdir: + cmdlist.append('-I{}'.format(sd)) + # put object files and module files in objdir_temp and moddir_temp + else: + cmdlist.append('-I{}'.format(objdir_temp)) + cmdlist.append('-J{}'.format(moddir_temp)) cmdlist.append('-c') cmdlist.append(srcfile) @@ -615,12 +646,6 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, cmdlist.append('-o') cmdlist.append(objfile) - if not iscfile: - # put object files in objdir_temp - cmdlist.append('-I' + objdir_temp) - # put module files in moddir_temp - cmdlist.append('-J' + moddir_temp) - # If expedited, then check if object file is out of date (if exists). # No need to compile if object file is newer. compilefile = True @@ -1304,7 +1329,7 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, expedite=False, dryrun=False, double=False, debug=False, - include_subdirs=False, fflags=None, cflags=None, syslibs='-lc', + include_subdirs=False, fflags=None, cflags=None, syslibs=None, arch='intel64', makefile=False, srcdir2=None, extrafiles=None, excludefiles=None, cmake=None, sharedobject=False): """ @@ -1351,9 +1376,14 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, # get ordered list of files to compile srcfiles = get_ordered_srcfiles(srcdir_temp, include_subdirs) + # add default syslibs + if syslibs is None: + if sys.platform != 'win32': + syslibs = '-lc' + # convert syslibs to a list if isinstance(syslibs, str): - syslibs = [syslibs] + syslibs = syslibs.split() # compile with gfortran or ifort winifort = False @@ -1389,7 +1419,6 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, else: winifort = True objext = '.obj' - # cc = 'cl.exe' returncode = compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, expedite, dryrun, double, debug, From 4d03ba717813ca8b08be2793124544611e6a4a47 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 29 Apr 2020 19:43:44 -0400 Subject: [PATCH 3/3] feat(download): cleanup --- pymake/pymake.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymake/pymake.py b/pymake/pymake.py index fcd2ac46..bad3ab17 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -696,6 +696,8 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, compileflags.insert(ipos, '-shared') for switch in compileflags: + if switch[:2] == '-I' or switch[:2] == '-J': + continue cmd += switch + ' ' cmdlist.append(switch)