From 41d8199e875c0e28f707715c604243b7646fdc1b Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Mon, 27 Apr 2020 16:23:06 -0400 Subject: [PATCH 1/7] ci(gsflow): Added gsflow build autotest Add gsflow build autotest (t012_test.py) that includes a test run of the sagehen model for a shortened simulation period. Update build_apps() to build single and double precision versions of targets based on the standard_switch and double_switch columns in usgsprograms.txt. If --appdir command line argument is specified when running buildall.py code.json is also copied to the specified --appdir directory. --- .gitignore | 8 ++ autotest/t012_test.py | 137 +++++++++++++++++++++++++++ examples/buildall.bat | 4 +- pymake/__init__.py | 3 +- pymake/build_program.py | 204 ++++++++++++++++++++++++++-------------- pymake/usgsprograms.py | 37 +++++++- pymake/usgsprograms.txt | 45 +++++---- 7 files changed, 336 insertions(+), 102 deletions(-) create mode 100644 autotest/t012_test.py diff --git a/.gitignore b/.gitignore index 9654085f..8d94e4a6 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,11 @@ obj_temp/ src_temp/ *.exe Dependencies + +# files in the examples directory +examples/win32/ +examples/win64/ +examples/mac/ +examples/linux/ +examples/temp/ + diff --git a/autotest/t012_test.py b/autotest/t012_test.py new file mode 100644 index 00000000..b19a3195 --- /dev/null +++ b/autotest/t012_test.py @@ -0,0 +1,137 @@ +from __future__ import print_function +import os +import sys +import shutil +import pymake +import flopy + +# define program data +target = 'gsflow' +prog_dict = pymake.usgs_program_data.get_target(target) + +# set up paths +dstpth = os.path.join('temp') +if not os.path.exists(dstpth): + os.makedirs(dstpth) + +gsflowver = prog_dict.version +gsflowpth = os.path.join(dstpth, prog_dict.dirname) +egsflow = os.path.abspath(os.path.join(dstpth, target)) + +# example path +expth = os.path.join(gsflowpth, 'data') +examples = ['sagehen'] +control_files = ['gsflow.control'] + +# set up pths and exes +pths = [gsflowpth] +exes = [egsflow] + + +def copy_example_dir(epth): + src = os.path.join(expth, epth) + dst = os.path.join(dstpth, epth) + + # delete dst if it exists + if os.path.isdir(dst): + shutil.rmtree(dst) + + # copy the files + try: + shutil.copytree(src, dst) + except: + msg = "could not move files from {} to '{}'".format(src, dst) + raise NameError(msg) + + # edit the control file for a shorter run + # sagehen + if epth == examples[0]: + fpth = os.path.join(dstpth, epth, 'linux', control_files[0]) + with open(fpth) as f: + lines = f.readlines() + with open(fpth, 'w') as f: + idx = 0 + while idx < len(lines): + line = lines[idx] + if 'end_time' in line: + line += '6\n1\n1981\n' + idx += 3 + f.write(line) + idx += 1 + return + + +def run_gsflow(example, control_file): + # copy files + copy_example_dir(example) + + model_ws = os.path.join(dstpth, example, 'linux') + + # run the flow model + msg = '{}'.format(egsflow) + success, buff = flopy.run_model(egsflow, control_file, model_ws=model_ws, + silent=False) + + assert success, 'could not run...{}'.format(msg) + + return + + +def clean_up(): + # clean up downloaded directories + if os.path.isdir(gsflowpth): + print('Removing folder ' + gsflowpth) + shutil.rmtree(gsflowpth) + + # clean up examples + for example in examples: + pth = os.path.join(dstpth, example) + if os.path.isdir(pth): + print('Removing example folder ' + example) + shutil.rmtree(pth) + + exe = egsflow + if sys.platform == 'win32': + exe += '.exe' + + # clean up compiled executables + if os.path.isfile(exe): + print('Removing ' + exe) + os.remove(exe) + return + + +def test_compile_gsflow(): + # Remove the existing GSFLOW directory if it exists + if os.path.isdir(gsflowpth): + shutil.rmtree(gsflowpth) + + # download and compile GSFLOW + pymake.build_program(target=target, + include_subdirs=True, + download_dir=dstpth, + exe_dir=dstpth) + return + + +def test_gsflow(): + for ex, cf in zip(examples, control_files): + yield run_gsflow, ex, cf + + +def test_clean_up(): + clean_up() + return + + +if __name__ == "__main__": + + # compile GSFLOW + test_compile_gsflow() + + # run example problems + for ex, cf in zip(examples, control_files): + run_gsflow(ex, cf) + + # clean up test + clean_up() diff --git a/examples/buildall.bat b/examples/buildall.bat index ca873eeb..f7e05dc0 100644 --- a/examples/buildall.bat +++ b/examples/buildall.bat @@ -1,5 +1,5 @@ rem 64-bit executables -python make_mfnwt.py --appdir win64 --ifort --icl --double -python make_mt3dusgs.py --appdir win64 --ifort --icl --double +python buildall.py --appdir win64 --ifort --icl +python buildall.py --appdir win32 --ifort --icl --ia32 pause diff --git a/pymake/__init__.py b/pymake/__init__.py index 50d25746..7a311706 100644 --- a/pymake/__init__.py +++ b/pymake/__init__.py @@ -16,5 +16,4 @@ compare_stages, compare, \ setup_mf6, setup_mf6_comparison, get_mf6_comparison, get_mf6_files from .build_program import build_program, build_apps, build_replace, \ - set_compiler - + set_compiler, set_bindir diff --git a/pymake/build_program.py b/pymake/build_program.py index 75bb3a6c..374bad92 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -51,7 +51,7 @@ def get_function_names(module, select_name=None): return func -def set_bindir(target): +def set_bindir(target=None): """ Set path for target based on --travis or --appdir command line arguments @@ -88,8 +88,9 @@ def set_bindir(target): bindir = '.' if not os.path.isdir(bindir): bindir = '.' - print('{} will be placed in the directory:\n'.format(target) + - ' "{}"\n'.format(bindir)) + if target is not None: + print('{} will be placed in the directory:\n'.format(target) + + ' "{}"\n'.format(bindir)) return bindir @@ -374,45 +375,6 @@ def set_syslibs(target, fc, cc): return syslibs -def set_double(target): - """ - Set boolean that defines if the target should use double precision reals - based on -dbl or --double command line arguments. - - Parameters - ---------- - target : str - target to build - - Returns - ------- - double : bool - - """ - double = False - if 'PYMAKE_DOUBLE' in os.environ: - double = True - - for idx, arg in enumerate(sys.argv): - if arg.lower() == '-dbl' or arg.lower() == '--double': - double = True - break - - if target in ['swtv4']: - double = True - - # write a message - if double: - prec = 'double' - else: - prec = 'single' - msg = '{} will be built using "{}" precision floats.\n'.format(target, - prec) - print(msg) - - return double - - def set_debug(target): """ Set boolean that defines if the target should be compiled with debug @@ -656,6 +618,13 @@ def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, if exe_name is None: exe_name = target + msg = 'float variable precision: ' + if double: + msg += '{}'.format('double') + else: + msg += '{}'.format('single') + print(msg) + if modify_exe_name: if double: filename, file_extension = os.path.splitext(exe_name) @@ -666,6 +635,11 @@ def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, if filename.lower()[-1] != 'd': exe_name = filename + 'd' + file_extension + # if a user-defined replace_function has not been provided then + # use the standard replace function, if one is available + if replace_function is None: + replace_function = build_replace(target) + if platform.system().lower() == 'windows': filename, file_extension = os.path.splitext(exe_name) if file_extension.lower() != '.exe': @@ -861,7 +835,6 @@ def build_apps(targets=None): # set double precision flag and whether the executable name # can be modified - double = set_double(target) if target == 'swtv4': modify_exe_name = False else: @@ -899,7 +872,7 @@ def build_apps(targets=None): replace_function = build_replace(target) # set download information - download = True + download_now = True download_clean = True download_dir = 'temp' @@ -911,7 +884,7 @@ def build_apps(targets=None): elif target == 'zbud6': if idt > 0: if targets[idt - 1] == 'mf6': - download = False + download_now = False # modify download if mfusg and also building zonbudusg if target == 'mfusg': @@ -921,7 +894,7 @@ def build_apps(targets=None): elif target == 'zonbudusg': if idt > 0: if targets[idt - 1] == 'mfusg': - download = False + download_now = False if target in ['mt3dms', 'triangle', 'mf6beta']: download_verify = False @@ -930,36 +903,54 @@ def build_apps(targets=None): download_verify = True timeout = 30 - # print download information - msg = 'downloading file: {}\n'.format(download) - msg += 'verified download: {}\n'.format(download_verify) - msg += 'download timeout: {} sec.\n'.format(timeout) - msg += 'cleaning extracted files: {}\n'.format(download_clean) - print(msg) - # set extrafiles extrafiles = set_extrafiles(target, download_dir) - # build the code - returncode = build_program(target=target, - fc=fc, - cc=cc, - double=double, - debug=debug, - fflags=fflags, - cflags=cflags, - syslibs=syslibs, - arch=arch, - include_subdirs=include_subdirs, - extrafiles=extrafiles, - replace_function=replace_function, - modify_exe_name=modify_exe_name, - exe_dir=bindir, - download=download, - download_dir=download_dir, - download_clean=download_clean, - download_verify=download_verify, - timeout=timeout) + # determine the + precision = usgs_program_data.get_precision(target) + + for idx, double in enumerate(precision): + # set clean flag + if len(precision) > 1: + if idx == 0: + download = download_now + else: + download = False + if idx < len(precision) - 1: + clean = False + else: + clean = download_clean + else: + download = download_now + clean = download_clean + + # print download information + msg = 'downloading file: {}\n'.format(download) + msg += 'verified download: {}\n'.format(download_verify) + msg += 'download timeout: {} sec.\n'.format(timeout) + msg += 'cleaning extracted files: {}\n'.format(clean) + print(msg) + + # build the code + returncode = build_program(target=target, + fc=fc, + cc=cc, + double=double, + debug=debug, + fflags=fflags, + cflags=cflags, + syslibs=syslibs, + arch=arch, + include_subdirs=include_subdirs, + extrafiles=extrafiles, + replace_function=replace_function, + modify_exe_name=modify_exe_name, + exe_dir=bindir, + download=download, + download_dir=download_dir, + download_clean=clean, + download_verify=download_verify, + timeout=timeout) # calculate download and compile time end_downcomp = datetime.now() @@ -1255,6 +1246,19 @@ def update_mfnwt_files(srcdir, fc, cc, arch, double): if os.path.exists(fpth): os.remove(fpth) + # update gwf2swi27.f + fpth = os.path.join(srcdir, 'gwf2swi27.f') + tag = '(i,csolver(i),i=1,3)' + new_tag = '(i,csolver(i),i=1,2)' + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() + def update_gsflow_files(srcdir, fc, cc, arch, double): # update gwf2swt7.f @@ -1270,6 +1274,60 @@ def update_gsflow_files(srcdir, fc, cc, arch, double): f.write(line) f.close() + # update gwf2swi27.f + fpth = os.path.join(srcdir, 'modflow', 'gwf2swi27.f') + tag = '(i,csolver(i),i=1,3)' + new_tag = '(i,csolver(i),i=1,2)' + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() + + # remove merge and lib directories + pths = [os.path.join(srcdir, 'merge'), os.path.join(srcdir, 'lib')] + for pth in pths: + if os.path.isdir(pth): + shutil.rmtree(pth) + + # remove existing *.mod, *.o, and *.a (if any are left) files + dirs = [os.path.join(srcdir, o) for o in os.listdir(srcdir) + if os.path.isdir(os.path.join(srcdir, o))] + + for dir in dirs: + for f in os.listdir(dir): + ext = os.path.splitext(f)[1] + fpth = os.path.join(dir, f) + if ext in ['.mod', '.o', '.a']: + os.remove(fpth) + + # edit and remove os specific files + if sys.platform.lower() == 'win32': + tag = "FORM='BINARY'" + new_tag = "FORM='UNFORMATTED', ACCESS='STREAM'" + fpth = os.path.join(srcdir, 'prms', 'utils_prms_windows.f90') + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() + + fpths = [os.path.join(srcdir, 'prms', 'utils_prms_linux.f90'), + os.path.join(srcdir, 'prms', 'utils_prms.f90')] + else: + fpths = [os.path.join(srcdir, 'prms', 'utils_prms_linux.f90'), + os.path.join(srcdir, 'prms', 'utils_prms_windows.f90')] + for fpth in fpths: + os.remove(fpth) + + return + def update_mf2000_files(srcdir, fc, cc, arch, double): # Remove six src folders diff --git a/pymake/usgsprograms.py b/pymake/usgsprograms.py index b7d5d517..edded031 100644 --- a/pymake/usgsprograms.py +++ b/pymake/usgsprograms.py @@ -1,4 +1,5 @@ import os +import shutil import json from collections import OrderedDict @@ -21,7 +22,8 @@ class dotdict(dict): program_data_file = 'usgsprograms.txt' # keys to create for each target -target_keys = ['version', 'current', 'url', 'dirname', 'srcdir'] +target_keys = ['version', 'current', 'url', 'dirname', 'srcdir', + 'standard_switch', 'double_switch'] def str_to_bool(s): @@ -79,7 +81,7 @@ def _build_usgs_database(self): d = OrderedDict() for idx, key in enumerate(target_keys): v = t[idx + 1] - if key == 'current': + if key in ['current', 'standard_switch', 'double_switch']: v = str_to_bool(v) d[key] = v @@ -165,6 +167,31 @@ def get_program_dict(): """ return usgs_program_data()._program_dict + @staticmethod + def get_precision(key): + """ + Get the dictionary for a specified target + + Parameters + ---------- + key : str + Target USGS program + + Returns + ------- + precision : list + List + + + """ + target = usgs_program_data().get_target(key) + precision = [] + if target.standard_switch: + precision.append(False) + if target.double_switch: + precision.append(True) + return precision + @staticmethod def list_targets(current=False): """ @@ -264,6 +291,12 @@ def export_json(fpth='code.json', prog_data=None, current=False, msg = 'could not export json file "{}"'.format(fpth) raise IOError(msg) + # + bindir = pymake.set_bindir() + if bindir != '.': + dst = os.path.join(bindir, fpth) + shutil.copyfile(fpth, dst) + return @staticmethod diff --git a/pymake/usgsprograms.txt b/pymake/usgsprograms.txt index 7ceeabfa..5f2b0667 100644 --- a/pymake/usgsprograms.txt +++ b/pymake/usgsprograms.txt @@ -1,23 +1,22 @@ -#target version current url dirname srcdir -mf2000 1.19.01 True https://water.usgs.gov/nrp/gwsoftware/modflow2000/mf2k1_19_01.tar.gz mf2k.1_19 src -mf2005 1.12.00 True https://water.usgs.gov/water-resources/software/MODFLOW-2005/MF2005.1_12u.zip MF2005.1_12u src -mf2005.1.11 1.11.00 False https://water.usgs.gov/ogw/modflow/archive-mf2005/MODFLOW-2005_v1.11.00/mf2005v1_11_00_unix.zip Unix src -mfnwt1.1.4 1.1.4 False https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.1.4.zip MODFLOW-NWT_1.1.4 src -mfnwt 1.2.0 True https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.2.0.zip MODFLOW-NWT_1.2.0 src -mfusg 1.5 True https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip mfusg1_5 src -zonbudusg 1.5 True https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip mfusg1_5 src/zonebudusg -mf6 6.1.0 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.0.zip mf6.1.0 src -zbud6 6.1.0 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.0.zip mf6.1.0 utils/zonebudget/src -swtv4 4.00.05 True https://water.usgs.gov/water-resources/software/SEAWAT/swt_v4_00_05.zip swt_v4_00_05 source -mp6 6.0.1 True https://water.usgs.gov/water-resources/software/MODPATH/modpath.6_0_01.zip modpath.6_0 src -mp7 7.2.001 True https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip modpath_7_2_001 source -mflgr 2.0.0 True https://water.usgs.gov/ogw/modflow-lgr/modflow-lgr-v2.0.0/mflgrv2_0_00.zip mflgr.2_0 src -mt3dms 5.3.0 True https://hydro.geo.ua.edu/mt3d/mt3dms_530.exe mt3dms5.3.0 src/true-binary -mt3dusgs 1.1.0 True https://water.usgs.gov/water-resources/software/MT3D-USGS/mt3dusgs1.1.0.zip mt3dusgs1.1.0 src -vs2dt 3.3 True https://water.usgs.gov/water-resources/software/VS2DI/vs2dt3_3.zip vs2dt3_3 include -triangle 1.6 True https://www.netlib.org/voronoi/triangle.zip triangle1.6 src -gridgen 1.0.02 True https://water.usgs.gov/water-resources/software/GRIDGEN/gridgen.1.0.02.zip gridgen.1.0.02 src -zonbud3 3.01 True https://water.usgs.gov/water-resources/software/ZONEBUDGET/zonbud3_01.exe Zonbud.3_01 Src -crt 1.3.1 True https://water.usgs.gov/ogw/CRT/CRT_1.3.1.zip CRT_1.3.1 SOURCE -gsflow 2.0.0 False https://water.usgs.gov/water-resources/software/gsflow/gsflow_2.0.0_linux.zip GSFLOW_2.0.0 src -mf6beta 6.3.0 False https://github.com/MODFLOW-USGS/modflow6/archive/mf6beta.zip modflow6-mf6beta src +#target version current url dirname srcdir standard_switch double_switch +mf2000 1.19.01 True https://water.usgs.gov/nrp/gwsoftware/modflow2000/mf2k1_19_01.tar.gz mf2k.1_19 src True False +mf2005 1.12.00 True https://water.usgs.gov/water-resources/software/MODFLOW-2005/MF2005.1_12u.zip MF2005.1_12u src True True +mf2005.1.11 1.11.00 False https://water.usgs.gov/ogw/modflow/archive-mf2005/MODFLOW-2005_v1.11.00/mf2005v1_11_00_unix.zip Unix src True False +mfnwt1.1.4 1.1.4 False https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.1.4.zip MODFLOW-NWT_1.1.4 src True False +mfnwt 1.2.0 True https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.2.0.zip MODFLOW-NWT_1.2.0 src True True +mfusg 1.5 True https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip mfusg1_5 src True True +zonbudusg 1.5 True https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip mfusg1_5 src/zonebudusg True False +mf6 6.1.0 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.0.zip mf6.1.0 src True False +zbud6 6.1.0 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.0.zip mf6.1.0 utils/zonebudget/src True False +swtv4 4.00.05 True https://water.usgs.gov/water-resources/software/SEAWAT/swt_v4_00_05.zip swt_v4_00_05 source False True +mp6 6.0.1 True https://water.usgs.gov/water-resources/software/MODPATH/modpath.6_0_01.zip modpath.6_0 src True False +mp7 7.2.001 True https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip modpath_7_2_001 source True False +mflgr 2.0.0 True https://water.usgs.gov/ogw/modflow-lgr/modflow-lgr-v2.0.0/mflgrv2_0_00.zip mflgr.2_0 src True True +mt3dms 5.3.0 True https://hydro.geo.ua.edu/mt3d/mt3dms_530.exe mt3dms5.3.0 src/true-binary True False +mt3dusgs 1.1.0 True https://water.usgs.gov/water-resources/software/MT3D-USGS/mt3dusgs1.1.0.zip mt3dusgs1.1.0 src True False +vs2dt 3.3 True https://water.usgs.gov/water-resources/software/VS2DI/vs2dt3_3.zip vs2dt3_3 include True False +triangle 1.6 True https://www.netlib.org/voronoi/triangle.zip triangle1.6 src True False +gridgen 1.0.02 True https://water.usgs.gov/water-resources/software/GRIDGEN/gridgen.1.0.02.zip gridgen.1.0.02 src True False +zonbud3 3.01 True https://water.usgs.gov/water-resources/software/ZONEBUDGET/zonbud3_01.exe Zonbud.3_01 Src True False +crt 1.3.1 True https://water.usgs.gov/ogw/CRT/CRT_1.3.1.zip CRT_1.3.1 SOURCE True False +gsflow 2.1.0 True https://water.usgs.gov/water-resources/software/gsflow/gsflow_2.1.0_linux.zip gsflow_2.1.0 src True False From 52c4533d94f994abfbf0f87d4ff4a9c23596b4a7 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Mon, 27 Apr 2020 17:36:39 -0400 Subject: [PATCH 2/7] ci(gsflow): Added gsflow build autotest Add gsflow build autotest (t012_test.py) that includes a test run of the sagehen model for a shortened simulation period. Update build_apps() to build single and double precision versions of targets based on the standard_switch and double_switch columns in usgsprograms.txt. If --appdir command line argument is specified when running buildall.py code.json is also copied to the specified --appdir directory. --- pymake/build_program.py | 92 ++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 374bad92..24c3a9cd 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -1231,33 +1231,37 @@ def update_mfnwt_files(srcdir, fc, cc, arch, double): # update gwf2swt7.f tag = 'EST(J,I,N)=0.0' fpth = os.path.join(srcdir, 'gwf2swt7.f') - with open(fpth) as f: - lines = f.readlines() - f = open(fpth, 'w') - for line in lines: - if tag in line: - indent = len(line) - len(line.lstrip()) - line += indent * ' ' + 'PCS(J,I,N)=0.0\n' - f.write(line) - f.close() + if os.path.exists(fpth): + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + indent = len(line) - len(line.lstrip()) + line += indent * ' ' + 'PCS(J,I,N)=0.0\n' + f.write(line) + f.close() # remove lrestart.f fpth = os.path.join(srcdir, 'Irestart.f') if os.path.exists(fpth): os.remove(fpth) - # update gwf2swi27.f + # update gwf2swi27.f or gwf2swi27.f fpth = os.path.join(srcdir, 'gwf2swi27.f') - tag = '(i,csolver(i),i=1,3)' - new_tag = '(i,csolver(i),i=1,2)' - with open(fpth) as f: - lines = f.readlines() - f = open(fpth, 'w') - for line in lines: - if tag in line: - line = line.replace(tag, new_tag) - f.write(line) - f.close() + if not os.path.exists(fpth): + fpth = os.path.join(srcdir, 'gwf2swi27.fpp') + if os.path.exists(fpth): + tag = '(i,csolver(i),i=1,3)' + new_tag = '(i,csolver(i),i=1,2)' + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() def update_gsflow_files(srcdir, fc, cc, arch, double): @@ -1274,18 +1278,21 @@ def update_gsflow_files(srcdir, fc, cc, arch, double): f.write(line) f.close() - # update gwf2swi27.f + # update gwf2swi27.f or gwf2swi27.fpp fpth = os.path.join(srcdir, 'modflow', 'gwf2swi27.f') - tag = '(i,csolver(i),i=1,3)' - new_tag = '(i,csolver(i),i=1,2)' - with open(fpth) as f: - lines = f.readlines() - f = open(fpth, 'w') - for line in lines: - if tag in line: - line = line.replace(tag, new_tag) - f.write(line) - f.close() + if not os.path.exists(fpth): + fpth = os.path.join(srcdir, 'modflow', 'gwf2swi27.fpp') + if os.path.exists(fpth): + tag = '(i,csolver(i),i=1,3)' + new_tag = '(i,csolver(i),i=1,2)' + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() # remove merge and lib directories pths = [os.path.join(srcdir, 'merge'), os.path.join(srcdir, 'lib')] @@ -1306,17 +1313,18 @@ def update_gsflow_files(srcdir, fc, cc, arch, double): # edit and remove os specific files if sys.platform.lower() == 'win32': - tag = "FORM='BINARY'" - new_tag = "FORM='UNFORMATTED', ACCESS='STREAM'" - fpth = os.path.join(srcdir, 'prms', 'utils_prms_windows.f90') - with open(fpth) as f: - lines = f.readlines() - f = open(fpth, 'w') - for line in lines: - if tag in line: - line = line.replace(tag, new_tag) - f.write(line) - f.close() + if 'ifort' not in fc: + tag = "FORM='BINARY'" + new_tag = "FORM='UNFORMATTED', ACCESS='STREAM'" + fpth = os.path.join(srcdir, 'prms', 'utils_prms_windows.f90') + with open(fpth) as f: + lines = f.readlines() + f = open(fpth, 'w') + for line in lines: + if tag in line: + line = line.replace(tag, new_tag) + f.write(line) + f.close() fpths = [os.path.join(srcdir, 'prms', 'utils_prms_linux.f90'), os.path.join(srcdir, 'prms', 'utils_prms.f90')] From 43d519f30ddc37a50b16dba39f05d2bbac994b6f Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Mon, 27 Apr 2020 18:07:30 -0400 Subject: [PATCH 3/7] ci(gsflow): Added gsflow build autotest Add gsflow build autotest (t012_test.py) that includes a test run of the sagehen model for a shortened simulation period. Update build_apps() to build single and double precision versions of targets based on the standard_switch and double_switch columns in usgsprograms.txt. If --appdir command line argument is specified when running buildall.py code.json is also copied to the specified --appdir directory. --- pymake/build_program.py | 10 +++++----- pymake/download.py | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 24c3a9cd..0dde4ce1 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -1301,13 +1301,13 @@ def update_gsflow_files(srcdir, fc, cc, arch, double): shutil.rmtree(pth) # remove existing *.mod, *.o, and *.a (if any are left) files - dirs = [os.path.join(srcdir, o) for o in os.listdir(srcdir) - if os.path.isdir(os.path.join(srcdir, o))] + dpths = [os.path.join(srcdir, o) for o in os.listdir(srcdir) + if os.path.isdir(os.path.join(srcdir, o))] - for dir in dirs: - for f in os.listdir(dir): + for dpth in dpths: + for f in os.listdpth(dpth): ext = os.path.splitext(f)[1] - fpth = os.path.join(dir, f) + fpth = os.path.join(dpth, f) if ext in ['.mod', '.o', '.a']: os.remove(fpth) diff --git a/pymake/download.py b/pymake/download.py index 26a88d54..fa917c3a 100644 --- a/pymake/download.py +++ b/pymake/download.py @@ -62,8 +62,19 @@ def download_and_unzip(url, pth='./', delete_zip=True, verify=True, tic = timeit.default_timer() for idx in range(nattempts): print(' download attempt: {}'.format(idx + 1)) - # - req = requests.get(url, stream=True, verify=verify) + + # open request + try: + req = requests.get(url, stream=True, verify=verify) + except TimeoutError: + continue + except requests.ConnectionError: + continue + except: + e = sys.exc_info()[0] + raise Exception(e) + + # connection established - download the file fs = 0 lenfs = 0 if 'Content-length' in req.headers: From 935eb41bea8a66dd4798f014bf062afca98cb890 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Mon, 27 Apr 2020 18:39:58 -0400 Subject: [PATCH 4/7] ci(gsflow): Added gsflow build autotest --- pymake/build_program.py | 21 ++++++++++++++------- pymake/usgsprograms.py | 6 +++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 0dde4ce1..6e8c93a5 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -84,13 +84,15 @@ def set_bindir(target=None): bindir = sys.argv[idx + 1] if not os.path.isdir(bindir): os.mkdir(bindir) + + # set default directory to current directory if bindir is None: bindir = '.' - if not os.path.isdir(bindir): - bindir = '.' - if target is not None: - print('{} will be placed in the directory:\n'.format(target) + - ' "{}"\n'.format(bindir)) + + # write message + if target is not None: + print('{} will be placed in the directory:\n'.format(target) + + ' "{}"\n'.format(bindir)) return bindir @@ -141,7 +143,12 @@ def set_build(target, exe_name): if keep: print('Determining if {} needs to be built'.format(exe_name)) - exe_exists = which(exe_name) + bindir = set_bindir() + if bindir != '.': + fpth = os.path.join(bindir, exe_name) + exe_exists = which(fpth) + else: + exe_exists = which(exe_name) # determine if it is in the current directory if exe_exists is None: @@ -1305,7 +1312,7 @@ def update_gsflow_files(srcdir, fc, cc, arch, double): if os.path.isdir(os.path.join(srcdir, o))] for dpth in dpths: - for f in os.listdpth(dpth): + for f in os.listdir(dpth): ext = os.path.splitext(f)[1] fpth = os.path.join(dpth, f) if ext in ['.mod', '.o', '.a']: diff --git a/pymake/usgsprograms.py b/pymake/usgsprograms.py index edded031..ba0b109d 100644 --- a/pymake/usgsprograms.py +++ b/pymake/usgsprograms.py @@ -1,5 +1,4 @@ import os -import shutil import json from collections import OrderedDict @@ -291,11 +290,12 @@ def export_json(fpth='code.json', prog_data=None, current=False, msg = 'could not export json file "{}"'.format(fpth) raise IOError(msg) - # + # export code.json to --appdir directory, if it exists bindir = pymake.set_bindir() if bindir != '.': dst = os.path.join(bindir, fpth) - shutil.copyfile(fpth, dst) + with open(fpth, 'w') as f: + json.dump(prog_data, f, indent=4) return From 0d08b3f4ce70bc9d7781a4fc7b880a757d256fff Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Mon, 27 Apr 2020 19:07:57 -0400 Subject: [PATCH 5/7] ci(gsflow): Added gsflow build autotest --- pymake/usgsprograms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymake/usgsprograms.py b/pymake/usgsprograms.py index ba0b109d..b3fb457b 100644 --- a/pymake/usgsprograms.py +++ b/pymake/usgsprograms.py @@ -290,11 +290,12 @@ def export_json(fpth='code.json', prog_data=None, current=False, msg = 'could not export json file "{}"'.format(fpth) raise IOError(msg) - # export code.json to --appdir directory, if it exists + # export code.json to --appdir directory, if the + # command line argument was specified bindir = pymake.set_bindir() if bindir != '.': dst = os.path.join(bindir, fpth) - with open(fpth, 'w') as f: + with open(dst, 'w') as f: json.dump(prog_data, f, indent=4) return From afa75b5f544ce9d07690b4573ff1ae7a58fad101 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Tue, 28 Apr 2020 00:40:54 -0400 Subject: [PATCH 6/7] ci(gsflow): Added gsflow build autotest --- pymake/build_program.py | 32 ++++++++++++++++++++++++++++++++ pymake/usgsprograms.py | 7 +++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 6e8c93a5..5527a332 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -276,6 +276,14 @@ def set_fflags(target, fc='gfortran'): if target == 'mp7': if fc == 'gfortran': fflags = '-ffree-line-length-512' + elif target == 'gsflow': + if fc == 'ifort': + if 'win32' in sys.platform.lower(): + fflags = '/fp:source /names:lowercase /assume:underscore' + else: + fflags = '-fp-model source' + elif fc == 'gfortran': + fflags = '-fno-second-underscore' # add additional fflags from the command line for idx, arg in enumerate(sys.argv): @@ -320,6 +328,14 @@ def set_cflags(target, cc='gcc'): cflags = '-lm' else: cflags = '-DNO_TIMER' + elif target == 'gsflow': + if cc == 'icc' or cc == 'icl': + if 'win32' in sys.platform.lower(): + cflags = '/Wall /D_CRT_SECURE_NO_WARNINGS' + else: + cflags = '-D_UF -Wall' + elif cc == 'gcc': + cflags = '-D_UF' # add additional cflags from the command line for idx, arg in enumerate(sys.argv): @@ -373,6 +389,10 @@ def set_syslibs(target, fc, cc): lcc = True if lfc and lcc: syslibs = '-lm' + elif target == 'gsflow': + if 'win32' not in sys.platform.lower(): + if 'ifort' in fc: + syslibs = '-nofor_main' # write syslibs msg = '{} will use the following predefined syslibs:\n'.format(target) @@ -710,6 +730,18 @@ def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, msg = '{} build failure.'.format(app) assert os.path.isfile(exe_name), msg + # remove *.exp and *.lib from windows builds + if target in ['gsflow', 'mfnwt']: + if fc == 'ifort': + if 'win32' in sys.platform.lower(): + fpth = exe_name.replace('.exe', '.exp') + if os.path.exists(fpth): + os.remove(fpth) + fpth = exe_name.replace('.exe', '.lib') + if os.path.exists(fpth): + os.remove(fpth) + + # clean download directory if different than directory with executable download_clean = set_download_clean(download_clean) if download_clean: diff --git a/pymake/usgsprograms.py b/pymake/usgsprograms.py index b3fb457b..68e332e0 100644 --- a/pymake/usgsprograms.py +++ b/pymake/usgsprograms.py @@ -290,10 +290,13 @@ def export_json(fpth='code.json', prog_data=None, current=False, msg = 'could not export json file "{}"'.format(fpth) raise IOError(msg) + # determine if running on Travis + is_travis = 'TRAVIS' in os.environ + # export code.json to --appdir directory, if the - # command line argument was specified + # command line argument was specified. Only done if not travis bindir = pymake.set_bindir() - if bindir != '.': + if bindir != '.' and not is_travis: dst = os.path.join(bindir, fpth) with open(dst, 'w') as f: json.dump(prog_data, f, indent=4) From ef792ba85c570e22a4124a2fc40fc24d718e9c90 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Tue, 28 Apr 2020 01:23:44 -0400 Subject: [PATCH 7/7] ci(gsflow): Added gsflow build autotest --- pymake/build_program.py | 1 - pymake/pymake.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 5527a332..740ff71f 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -741,7 +741,6 @@ def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, if os.path.exists(fpth): os.remove(fpth) - # clean download directory if different than directory with executable download_clean = set_download_clean(download_clean) if download_clean: diff --git a/pymake/pymake.py b/pymake/pymake.py index e6d40226..86f5acd9 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -828,7 +828,7 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, # put module files in moddir_temp cmdlist.append('-module') - cmdlist.append('./' + moddir_temp + '/') + cmdlist.append(moddir_temp + '/') # add search path for any header files for sd in searchdir: @@ -840,7 +840,7 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, # object file name and location srcname, srcext = os.path.splitext(srcfile) srcname = srcname.split(os.path.sep)[-1] - objfile = os.path.join('.', objdir_temp, srcname + '.o') + objfile = os.path.join(objdir_temp, srcname + '.o') cmdlist.append('-o') cmdlist.append(objfile)