From f99e6b4fa20eaf485903febb83dc15bf76a321e2 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Tue, 16 Jun 2020 13:36:25 -0400 Subject: [PATCH 1/8] refactor(pymake): refactor makefile generation --- pymake/pymake.py | 154 +++++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 65 deletions(-) diff --git a/pymake/pymake.py b/pymake/pymake.py index 59800cbf..8a45c5bc 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -56,7 +56,7 @@ def parser(): 'gfortran']) parser.add_argument('-cc', help='C compiler to use (default is gcc)', default='gcc', choices=['gcc', 'clang', 'icc', - 'mpiicc', 'g++']) + 'mpiicc', 'g++', 'cl']) parser.add_argument('-ar', '--arch', help='Architecture to use for ifort (default is intel64)', default='intel64', @@ -469,11 +469,13 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, else: os_macro = None - # fortran compiler switches + # compiler optimization level if debug: - opt = '-O0' + optlevel = '-O0' else: - opt = '-O2' + optlevel = '-O2' + + # fortran compiler switches if fflags is None: fflags = [] elif isinstance(fflags, str): @@ -482,17 +484,16 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, for fflag in fflags: if fflag[:2] == '-O': if not debug: - opt = fflag + optlevel = fflag fflags.remove(fflag) break # after first optimization (O) flag # set fortran flags + compileflags = [] + + # Debug flags if debug: - # Debug flags - compileflags = ['-g', - opt] - else: - compileflags = [opt] + compileflags = ['-g'] # add gfortran specific compiler switches if fc is not None: @@ -536,11 +537,6 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, compileflags.append(fflag) # C/C++ compiler switches -- thanks to mja - if debug: - opt = '-O0' - else: - opt = '-O2' - if cflags is None: cflags = [] else: @@ -551,17 +547,14 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, for cflag in cflags: if cflag[:2] == '-O': if not debug: - opt = cflag + optlevel = cflag cflags.remove(cflag) break # after first optimization (O) flag # set additional c flags + # Debug flags if debug: - # Debug flags - cflags += ['-g', - opt] - else: - cflags += [opt] + cflags += ['-g'] if cc.startswith('g'): if sys.platform == 'win32': @@ -620,10 +613,12 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if ext in ['.c', '.cpp']: # mja iscfile = True cmdlist.append(cc) # mja + cmdlist.append(optlevel) for switch in cflags: # mja cmdlist.append(switch) # mja else: # mja cmdlist.append(fc) + cmdlist.append(optlevel) for switch in compileflags: cmdlist.append(switch) @@ -656,7 +651,6 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # Compile if compilefile: if not dryrun: - # subprocess.check_call(cmdlist, shell=shellflg) proc = Popen(cmdlist, shell=shellflg, stdout=PIPE, stderr=PIPE) process_Popen_command(shellflg, cmdlist) @@ -684,12 +678,14 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if fc is None: cmd = cc + ' ' cmdlist.append(cc) + cmdlist.append(optlevel) for switch in cflags: cmd += switch + ' ' cmdlist.append(switch) else: cmd = fc + ' ' cmdlist.append(fc) + cmdlist.append(optlevel) if sharedobject: ipos = compileflags.index('-fPIC') @@ -702,8 +698,7 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, cmdlist.append(switch) cmdlist.append('-o') - cmdlist.append(os.path.join('.', target)) - + cmdlist.append(target) for objfile in objfiles: cmdlist.append(objfile) @@ -735,6 +730,36 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # return return 0 +def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, + osname=None): + if osname is None: + osname = sys.platform.lower() + + if osname in ['linux', 'darwin']: + switch = '-' + else: + switch = '/' + + compileflags = [] + + # set optimization + if debug: + compileflags.append(switch + 'O0') + else: + compileflags.append(switch + 'O2') + + # add + if fc == 'gfortran': + if 'win32' in os: + compileflags.append('-Bstatic') + elif fc in ['ifort', 'ifortmpi']: + if 'darwin' in os or 'linux' in os: + print('do it') + + else: + msg = "unsupported system '{}'".format(os) + print('do it') + def compile_with_macnix_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, @@ -746,11 +771,11 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, Make target on Mac OSX """ # fortran compiler switches - if debug: - opt = '-O0' + optlevel = '-O0' else: - opt = '-O2' + optlevel = '-O2' + if fflags is None: fflags = [] elif isinstance(fflags, str): @@ -760,21 +785,19 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, for fflag in fflags: if fflag[:2] == '-O' or fflag == '-fast': if not debug: - opt = fflag + optlevel = fflag fflags.remove(fflag) break # after first optimization (O) flag # add ifort specific compiler switches compileflags = [] if fc is not None: - compileflags.append(opt) - # add shared object switches if sharedobject: compileflags.append('-fpic') + # Debug flags if debug: - # Debug flags compileflags += ['-debug', 'all', '-no-heap-arrays', '-fpe0', @@ -796,11 +819,6 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, compileflags.append(fflag) # C/C++ compiler switches -- thanks to mja - if debug: - opt = '-O0' - else: - opt = '-O2' - if cflags is None: cflags = [] else: @@ -811,17 +829,14 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, for cflag in cflags: if cflag[:2] == '-O': if not debug: - opt = cflag + optlevel = cflag cflags.remove(cflag) break # after first optimization (O) flag # set additional c flags + # Debug flags if debug: - # Debug flags - cflags += ['-g', - opt] - else: - cflags += [opt] + cflags += ['-g'] # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran # code that is linked to C/C++ code @@ -846,20 +861,29 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, cmdlist = [] if srcfile.endswith('.c') or srcfile.endswith('.cpp'): # mja cmdlist.append(cc) # mja + cmdlist.append(optlevel) for switch in cflags: # mja cmdlist.append(switch) # mja + + # add search path for any header files + for sd in searchdir: + cmdlist.append('-I {}'.format(sd)) else: # mja cmdlist.append(fc) + cmdlist.append(optlevel) for switch in compileflags: cmdlist.append(switch) + # put object files in objdir_temp + cmdlist.append('-I {}/'.format(objdir_temp)) + # put module files in moddir_temp cmdlist.append('-module') cmdlist.append(moddir_temp + '/') - # add search path for any header files - for sd in searchdir: - cmdlist.append('-I{}'.format(sd)) + # # add search path for any header files + # for sd in searchdir: + # cmdlist.append('-I {}'.format(sd)) cmdlist.append('-c') cmdlist.append(srcfile) @@ -906,14 +930,13 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, cmdlist = [] if fc is None: - cmd = cc + ' ' cmdlist.append(cc) + cmdlist.append(optlevel) for switch in cflags: - cmd += switch + ' ' cmdlist.append(switch) else: - cmd = fc + ' ' cmdlist.append(fc) + cmdlist.append(optlevel) if sharedobject: ipos = compileflags.index('-fpic') @@ -924,15 +947,16 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, compileflags.insert(ipos, copt) for switch in compileflags: - cmd += switch + ' ' cmdlist.append(switch) cmdlist.append('-o') - cmdlist.append(os.path.join('.', target)) + # cmdlist.append(os.path.join('.', target)) + cmdlist.append(target) for objfile in objfiles: cmdlist.append(objfile) for switch in syslibs: cmdlist.append(switch) + if not dryrun: # subprocess.check_call(cmdlist) proc = Popen(cmdlist, stdout=PIPE, stderr=PIPE) @@ -987,9 +1011,9 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, fflags = ['/heap-arrays:0', '/fpe:0', '/traceback', '/nologo'] if debug: - opt = '/debug' + optlevel = '/debug' else: - opt = '/O2' + optlevel = '/O2' if fflagsu is None: fflagsu = [] elif isinstance(fflagsu, str): @@ -1002,16 +1026,16 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, for fflag in fflagsu: if fflag[:2] in ('-O', '/O') or fflag in ('-fast', '/fast'): if not debug: - opt = fflag + optlevel = fflag fflagsu.remove(fflag) break # after first optimization (O) flag if debug: - fflags.append(opt) + # fflags.append(optlevel) cflags.append('/Zi') - else: - # production version compile flags - fflags.append(opt) - cflags.append('/O2') + # else: + # # production version compile flags + # fflags.append(optlevel) + # cflags.append('/O2') if double: fflags.append('/real-size:64') fflags.append('/double-size:64') @@ -1036,7 +1060,7 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if flopy_avail: if flopy_is_exe(target): os.remove(target) - makebatch(batchfile, fc, cc, fflags, cflags, srcfiles, target, + makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, arch, objdir_temp, moddir_temp) proc = Popen([batchfile, ], stdout=PIPE, stderr=STDOUT) while True: @@ -1064,8 +1088,8 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, return 0 -def makebatch(batchfile, fc, cc, fflags, cflags, srcfiles, target, arch, - objdir_temp, moddir_temp): +def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, + arch, objdir_temp, moddir_temp): """ Make an ifort batch file @@ -1096,7 +1120,7 @@ def makebatch(batchfile, fc, cc, fflags, cflags, srcfiles, target, arch, # write commands to build object files for srcfile in srcfiles: if srcfile.endswith('.c') or srcfile.endswith('.cpp'): - cmd = cc + ' ' + cmd = cc + ' ' + optlevel + ' ' for switch in cflags: cmd += switch + ' ' @@ -1110,7 +1134,7 @@ def makebatch(batchfile, fc, cc, fflags, cflags, srcfiles, target, arch, cmd += '/Fo' + obj + ' ' cmd += srcfile else: - cmd = fc + ' ' + cmd = fc + ' ' + optlevel + ' ' for switch in fflags: cmd += switch + ' ' cmd += '-c' + ' ' @@ -1122,9 +1146,9 @@ def makebatch(batchfile, fc, cc, fflags, cflags, srcfiles, target, arch, # write commands to link if fc is None: - cmd = cc + ' ' + cmd = cc + ' ' + optlevel + ' ' else: - cmd = fc + ' ' + cmd = fc + ' ' + optlevel + ' ' for switch in fflags: cmd += switch + ' ' cmd += '-o' + ' ' + target + ' ' + objdir_temp + '\\*.obj' + '\n' From afa319eda0d654bdecf4b9637c5d725faad067f1 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Tue, 16 Jun 2020 18:43:17 -0400 Subject: [PATCH 2/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, and CFLAGS. --- pymake/pymake.py | 1095 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 774 insertions(+), 321 deletions(-) diff --git a/pymake/pymake.py b/pymake/pymake.py index 8a45c5bc..70fabd41 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -1,7 +1,5 @@ #! /usr/bin/env python -""" -Make a binary executable for a FORTRAN program, such as MODFLOW. -""" +"""Make a binary executable for a FORTRAN program, such as MODFLOW.""" from __future__ import print_function __author__ = "Christian D. Langevin" @@ -38,9 +36,7 @@ def parser(): - ''' - Construct the parser and return argument values - ''' + """Construct the parser and return argument values.""" description = __description__ parser = argparse.ArgumentParser(description=description, epilog='''Note that the source directory @@ -119,8 +115,7 @@ def parser(): def process_Popen_command(shellflg, cmdlist): - """ - Generic function to write Popen command data to the screen + """Generic function to write Popen command data to the screen. Parameters ---------- @@ -132,7 +127,6 @@ def process_Popen_command(shellflg, cmdlist): Returns ------- - """ if not shellflg: print(' '.join(cmdlist)) @@ -140,9 +134,8 @@ def process_Popen_command(shellflg, cmdlist): def process_Popen_communicate(stdout, stderr): - """ - Generic function to write communication information from Popen - to the screen + """Generic function to write communication information from Popen to the + screen. Parameters ---------- @@ -154,7 +147,6 @@ def process_Popen_communicate(stdout, stderr): Returns ------- - """ if stdout: if PY3: @@ -168,10 +160,11 @@ def process_Popen_communicate(stdout, stderr): def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): - ''' - Remove temp source directory and target, and then copy source into - source temp directory. Return temp directory path. - ''' + """Remove temp source directory and target, and then copy source into + source temp directory. + + Return temp directory path. + """ # remove the target if it already exists srcdir_temp = os.path.join('.', 'src_temp') objdir_temp = os.path.join('.', 'obj_temp') @@ -272,10 +265,7 @@ def parse_extrafiles(extrafiles): def clean(srcdir_temp, objdir_temp, moddir_temp, objext, winifort): - """ - Remove mod and object files, and remove the temp source directory. - - """ + """Remove mod and object files, and remove the temp source directory.""" # clean things up print('\nCleaning up temporary source, object, and module files...') filelist = os.listdir('.') @@ -293,9 +283,10 @@ def clean(srcdir_temp, objdir_temp, moddir_temp, objext, winifort): def get_ordered_srcfiles(srcdir_temp, include_subdir=False): - """ - Create a list of ordered source files (both fortran and c). Ordering - is build using a directed acyclic graph to determine module dependencies. + """Create a list of ordered source files (both fortran and c). + + Ordering is build using a directed acyclic graph to determine module + dependencies. """ # create a list of all c(pp), f and f90 source files @@ -344,9 +335,10 @@ def get_ordered_srcfiles(srcdir_temp, include_subdir=False): def create_openspec(srcdir_temp): - """ - Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses - STREAM ACCESS. This is specific to MODFLOW and MT3D based targets. + """Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses + STREAM ACCESS. + + This is specific to MODFLOW and MT3D based targets. """ files = ['openspec.inc', 'filespec.inc'] dirs = [d[0] for d in os.walk(srcdir_temp)] @@ -402,8 +394,7 @@ def get_iso_c(srcfiles): def flag_available(flag): - """ - Determine if a specified flag exists + """Determine if a specified flag exists. Not all flags will be detected, for example -O2 -fbounds-check=on """ @@ -441,10 +432,7 @@ def flag_available(flag): def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, expedite, dryrun, double, debug, fflags, cflags, syslibs, srcdir, srcdir2, extrafiles, makefile, sharedobject): - """ - Compile the program using the gnu compilers (gfortran and gcc) - - """ + """Compile the program using the gnu compilers (gfortran and gcc)""" # define the platform platform = sys.platform @@ -456,143 +444,180 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # For horrible windows issue # if platform == 'win32': # shellflg = True + # + # # define the OS macro for gfortran + # if platform == 'win32': + # os_macro = '-D_WIN32' + # elif platform == 'darwin': + # os_macro = '-D__APPLE__' + # elif platform == 'linux' or platform == 'linux2': + # os_macro = '-D__linux__' + # elif 'bsd' in platform: + # os_macro = '-D__unix__' + # else: + # os_macro = None - # define the OS macro for gfortran - if platform == 'win32': - os_macro = '-D_WIN32' - elif platform == 'darwin': - os_macro = '-D__APPLE__' - elif platform == 'linux' or platform == 'linux2': - os_macro = '-D__linux__' - elif 'bsd' in platform: - os_macro = '-D__unix__' - else: - os_macro = None - - # compiler optimization level - if debug: - optlevel = '-O0' - else: - optlevel = '-O2' - - # fortran compiler switches + # convert fflags and cflags to lists if fflags is None: fflags = [] elif isinstance(fflags, str): fflags = fflags.split() - # look for optimization levels in fflags - for fflag in fflags: - if fflag[:2] == '-O': - if not debug: - optlevel = fflag - fflags.remove(fflag) - break # after first optimization (O) flag - - # set fortran flags - compileflags = [] - - # Debug flags - if debug: - compileflags = ['-g'] - - # add gfortran specific compiler switches - if fc is not None: - # add shared object switches - if sharedobject: - compileflags.append('-fPIC') - - if debug: - compileflags += ['-fcheck=all', '-fbounds-check', '-Wall'] - lflag = flag_available('-ffpe-trap') - if lflag: - compileflags.append( - '-ffpe-trap=overflow,zero,invalid,denormal') - else: - lflag = flag_available('-ffpe-summary') - if lflag: - compileflags.append('-ffpe-summary=overflow') - lflag = flag_available('-ffpe-trap') - if lflag: - compileflags.append('-ffpe-trap=overflow,zero,invalid') - - # add fbacktrace to debug and release versions - compileflags.append('-fbacktrace') - - # add static - if sys.platform == 'win32': - compileflags.append('-Bstatic') - - # add double precision switches - if double: - compileflags.append('-fdefault-real-8') - compileflags.append('-fdefault-double-8') - - # add defined OS macro - if os_macro is not None: - compileflags.append(os_macro) - - # Split all tokens by spaces - for fflag in ' '.join(fflags).split(): - if fflag not in compileflags: - compileflags.append(fflag) - - # C/C++ compiler switches -- thanks to mja if cflags is None: cflags = [] - else: - if isinstance(cflags, str): - cflags = cflags.split() + elif isinstance(cflags, str): + cflags = cflags.split() - # look for optimization levels in cflags - for cflag in cflags: - if cflag[:2] == '-O': - if not debug: - optlevel = cflag - cflags.remove(cflag) - break # after first optimization (O) flag - - # set additional c flags - # Debug flags - if debug: - cflags += ['-g'] - - if cc.startswith('g'): - if sys.platform == 'win32': - cflags += ['-Bstatic'] - if debug: - lflag = flag_available('-Wall') - if lflag: - cflags += ['-Wall'] - else: - pass - - # determine if any c, cpp or fortran files - iscfiles = False - isfortranfiles = False - for srcfile in srcfiles: - ext = os.path.splitext(srcfile)[1].lower() - if ext in ['.c', '.cpp']: # mja - iscfiles = True - elif ext in ['.f', '.for', '.f90', '.fpp']: - isfortranfiles = True - - # reset syslibs for windows - if sys.platform == 'win32': - syslibs = [] - if isfortranfiles: - syslibs.append('-lgfortran') - if iscfiles: - syslibs.append('-lgcc') - syslibs.append('-lm') - - # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # code that is linked to C/C++ code. Only needed if there are - # any fortran files. -D_UF defines UNIX naming conventions for - # mixed language compilation. - if isfortranfiles: - use_iso_c = get_iso_c(srcfiles) - if not use_iso_c: - cflags.append('-D_UF') + # set optimization levels + optlevel = get_optlevel(fc, cc, debug, fflags, cflags) + # if debug: + # optlevel = '-O0' + # else: + # optlevel = '-O2' + # # look for optimization levels in fflags + # for flag in fflags: + # if flag[:2] == '-O' or flag == '-fast': + # if not debug: + # optlevel = flag + # fflags.remove(flag) + # break # after first optimization (O) flag + # # look for optimization levels in cflags + # for flag in cflags: + # if flag[:2] == '-O': + # if not debug: + # optlevel = flag + # cflags.remove(flag) + # break # after first optimization (O) flag + + # get fortran and c compiler switches + tfflags = get_fortran_flags(fc, fflags, debug, double, + sharedobject=sharedobject) + tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, + sharedobject=sharedobject) + + # # compiler optimization level + # if debug: + # optlevel = '-O0' + # else: + # optlevel = '-O2' + # + # # fortran compiler switches + # if fflags is None: + # fflags = [] + # elif isinstance(fflags, str): + # fflags = fflags.split() + # # look for optimization levels in fflags + # for fflag in fflags: + # if fflag[:2] == '-O': + # if not debug: + # optlevel = fflag + # fflags.remove(fflag) + # break # after first optimization (O) flag + # + # # set fortran flags + # compileflags = [] + # + # # Debug flags + # if debug: + # compileflags = ['-g'] + # + # # add gfortran specific compiler switches + # if fc is not None: + # # add shared object switches + # if sharedobject: + # compileflags.append('-fPIC') + # + # if debug: + # compileflags += ['-fcheck=all', '-fbounds-check', '-Wall'] + # lflag = flag_available('-ffpe-trap') + # if lflag: + # compileflags.append( + # '-ffpe-trap=overflow,zero,invalid,denormal') + # else: + # lflag = flag_available('-ffpe-summary') + # if lflag: + # compileflags.append('-ffpe-summary=overflow') + # lflag = flag_available('-ffpe-trap') + # if lflag: + # compileflags.append('-ffpe-trap=overflow,zero,invalid') + # + # # add fbacktrace to debug and release versions + # compileflags.append('-fbacktrace') + # + # # add static + # if sys.platform == 'win32': + # compileflags.append('-Bstatic') + # + # # add double precision switches + # if double: + # compileflags.append('-fdefault-real-8') + # compileflags.append('-fdefault-double-8') + # + # # add defined OS macro + # if os_macro is not None: + # compileflags.append(os_macro) + # + # # Split all tokens by spaces + # for fflag in ' '.join(fflags).split(): + # if fflag not in compileflags: + # compileflags.append(fflag) + # + # # C/C++ compiler switches -- thanks to mja + # if cflags is None: + # cflags = [] + # else: + # if isinstance(cflags, str): + # cflags = cflags.split() + # + # # look for optimization levels in cflags + # for cflag in cflags: + # if cflag[:2] == '-O': + # if not debug: + # optlevel = cflag + # cflags.remove(cflag) + # break # after first optimization (O) flag + # + # # set additional c flags + # # Debug flags + # if debug: + # cflags += ['-g'] + # + # if cc.startswith('g'): + # if sys.platform == 'win32': + # cflags += ['-Bstatic'] + # if debug: + # lflag = flag_available('-Wall') + # if lflag: + # cflags += ['-Wall'] + # else: + # pass + # + # # determine if any c, cpp or fortran files + # iscfiles = False + # isfortranfiles = False + # for srcfile in srcfiles: + # ext = os.path.splitext(srcfile)[1].lower() + # if ext in ['.c', '.cpp']: # mja + # iscfiles = True + # elif ext in ['.f', '.for', '.f90', '.fpp']: + # isfortranfiles = True + # + # # reset syslibs for windows + # if sys.platform == 'win32': + # syslibs = [] + # if isfortranfiles: + # syslibs.append('-lgfortran') + # if iscfiles: + # syslibs.append('-lgcc') + # syslibs.append('-lm') + # + # # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran + # # code that is linked to C/C++ code. Only needed if there are + # # any fortran files. -D_UF defines UNIX naming conventions for + # # mixed language compilation. + # if isfortranfiles: + # use_iso_c = get_iso_c(srcfiles) + # if not use_iso_c: + # cflags.append('-D_UF') # build object files print('\nCompiling object files for ' + @@ -614,12 +639,12 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, iscfile = True cmdlist.append(cc) # mja cmdlist.append(optlevel) - for switch in cflags: # mja + for switch in tcflags: # mja cmdlist.append(switch) # mja else: # mja cmdlist.append(fc) cmdlist.append(optlevel) - for switch in compileflags: + for switch in tfflags: cmdlist.append(switch) # add search path for any c and c++ header files @@ -679,7 +704,7 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, cmd = cc + ' ' cmdlist.append(cc) cmdlist.append(optlevel) - for switch in cflags: + for switch in tcflags: cmd += switch + ' ' cmdlist.append(switch) else: @@ -688,10 +713,10 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, cmdlist.append(optlevel) if sharedobject: - ipos = compileflags.index('-fPIC') - compileflags.insert(ipos, '-shared') + ipos = tfflags.index('-fPIC') + tfflags.insert(ipos, '-shared') - for switch in compileflags: + for switch in tfflags: if switch[:2] == '-I' or switch[:2] == '-J': continue cmd += switch + ' ' @@ -724,126 +749,530 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if makefile: create_makefile(target, srcdir, srcdir2, extrafiles, srcfiles, objfiles, - fc, compileflags, cc, cflags, syslibs, + fc, tfflags, cc, tcflags, syslibs, modules=['-I', '-J']) # return return 0 -def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, - osname=None): - if osname is None: - osname = sys.platform.lower() - if osname in ['linux', 'darwin']: - switch = '-' - else: - switch = '/' +def get_osname(): + """Return the lower case OS platform name. + + Parameters + ------- + + Returns + ------- + str : str + lower case OS platform name + """ + return sys.platform.lower() - compileflags = [] - # set optimization - if debug: - compileflags.append(switch + 'O0') - else: - compileflags.append(switch + 'O2') +def get_prepend(compiler, osname): + """Return the appropriate prepend for a compiler switch for a OS. - # add - if fc == 'gfortran': - if 'win32' in os: - compileflags.append('-Bstatic') - elif fc in ['ifort', 'ifortmpi']: - if 'darwin' in os or 'linux' in os: - print('do it') + Parameters + ------- + compiler : str + compiler name + osname : str + lower case OS name + Returns + ------- + str : str + prepend string for a compiler switch for a OS + """ + if compiler in ['gfortran', 'gcc', 'g++', 'clang']: + prepend = '-' + else: + if osname in ['linux', 'darwin']: + prepend = '-' else: - msg = "unsupported system '{}'".format(os) - print('do it') + prepend = '/' + return prepend -def compile_with_macnix_ifort(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, - srcdir, srcdir2, extrafiles, makefile, - sharedobject): +def fortran_files(srcfiles, extensions=False): + """Return a list of fortran files or unique fortran file extensions. + + Parameters + ------- + srcfiles : list + list of sourcefile names + extensions : bool + flag controls return of either a list of fortran files or + a list of unique fortran file extensions + + Returns + ------- + list : list + list of fortran files or unique fortran file extensions """ - Make target on Mac OSX + l = [] + for srcfile in srcfiles: + ext = os.path.splitext(srcfile)[1] + if ext.lower() in ['.f', '.for', '.f90', '.fpp']: + if extensions: + # save unique extension + if ext not in l: + l.append(ext) + else: + l.append(srcfile) + if len(l) < 1: + l = None + return l + + +def c_files(srcfiles, extensions=False): + """Return a list of c and cpp files or unique c and cpp file extensions. + + Parameters + ------- + srcfiles : list + list of sourcefile names + extensions : bool + flag controls return of either a list of c and cpp files or + a list of unique c and cpp file extensions + + Returns + ------- + list : list + list of c or cpp files or uniques c and cpp file extensions """ - # fortran compiler switches + l = [] + for srcfile in srcfiles: + ext = os.path.splitext(srcfile)[1] + if ext.lower() in ['.c', '.cpp']: + if extensions: + if ext not in l: + l.append(ext) + else: + l.append(srcfile) + if len(l) < 1: + l = None + return l + +def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): + """Return a compiler optimization switch. + + Parameters + ------- + fc : str + fortran compiler + cc : str + c or cpp compiler + debug : bool + flag indicating is a debug executible will be built + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + osname : str + optional lower case OS name. If not passed it will be determined + using sys.platform + + Returns + ------- + optlevel : str + compiler optimization switch + """ + # get lower case OS string + if osname is None: + osname = get_osname() + + compiler = None + if fc is not None: + compiler = fc + elif compiler is None: + compiler = cc + + # get - or / to prepend for compiler switches + prepend = get_prepend(fc, osname) + + # set basic optimization level if debug: - optlevel = '-O0' + if osname == 'win32': + optlevel = 'Od' + else: + optlevel = 'O0' else: - optlevel = '-O2' - - if fflags is None: - fflags = [] - elif isinstance(fflags, str): - fflags = fflags.split() + optlevel = 'O2' # look for optimization levels in fflags - for fflag in fflags: - if fflag[:2] == '-O' or fflag == '-fast': + for flag in fflags: + if flag[:2] == '-O' or flag == '-fast': if not debug: - optlevel = fflag - fflags.remove(fflag) + optlevel = flag[1:] break # after first optimization (O) flag - # add ifort specific compiler switches - compileflags = [] - if fc is not None: - # add shared object switches - if sharedobject: - compileflags.append('-fpic') + # look for optimization levels in cflags + for flag in cflags: + if flag[:2] == '-O': + if not debug: + optlevel = flag[1:] + break # after first optimization (O) flag + + # prepend optlevel + optlevel = prepend + optlevel - # Debug flags + return optlevel + +def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, + osname=None): + """Return a list of standard pymake and user specified fortran compiler + switches. + + Parameters + ------- + fc : str + fortran compiler + fflags : list + user provided list of fortran compiler flags + debug : bool + flag indicating a debug executable will be built + double : bool + flag indicating a double precision executable will be built + sharedobject : bool + flag indicating a shared object (.so or .dll) will be built + osname : str + optional lower case OS name. If not passed it will be determined + using sys.platform + + Returns + ------- + flags : str + fortran compiler switches + """ + # remove .exe extension of necessary + if '.exe' in fc.lower(): + fc = fc[:-4] + + # get lower case OS string + if osname is None: + osname = get_osname() + + # get - or / to prepend for compiler switches + prepend = get_prepend(fc, osname) + + # generate standard fortran flags + flags = [] + if fc == 'gfortran': + if sharedobject: + flags.append('fPIC') + flags.append('fbacktrace') + if osname == 'win32': + flags.append('Bstatic') if debug: - compileflags += ['-debug', 'all', - '-no-heap-arrays', - '-fpe0', - '-traceback'] + flags += ['g', 'fcheck=all', 'fbounds-check', 'Wall'] + if flag_available('-ffpe-trap'): + flags.append('ffpe-trap=overflow,zero,invalid,denormal') else: - # production version compile flags - compileflags += ['-no-heap-arrays', - '-fpe0', - '-traceback'] - - # add double precision compiler switches + if flag_available('-ffpe-summary'): + flags.append('ffpe-summary=overflow') + if flag_available('-ffpe-trap'): + flags.append('ffpe-trap=overflow,zero,invalid') if double: - compileflags += ['-real-size', '64'] - compileflags += ['-double-size', '64'] + flags += ['fdefault-real-8', 'fdefault-double-8'] + # define the OS macro for gfortran + if osname == 'win32': + os_macro = 'D_WIN32' + elif osname == 'darwin': + os_macro = 'D__APPLE__' + elif 'linux' in osname: + os_macro = 'D__linux__' + elif 'bsd' in osname: + os_macro = 'D__unix__' + else: + os_macro = None + if os_macro is not None: + flags.append(os_macro) + elif fc in ['ifort', 'mpiifort']: + if osname == 'win32': + flags += ['heap-arrays:0', 'fpe:0', 'traceback', 'nologo'] + if debug: + flags += ['debug:full', 'Zi'] + if double: + flags += ['real-size:64', 'double-size:64'] + else: + if sharedobject: + flags.append('fpic') + if debug: + flags += ['g'] + flags += ['no-heap-arrays', 'fpe0', 'traceback'] + if double: + flags += ['real-size 64', 'double-size 64'] - # Split all tokens by spaces - for fflag in ' '.join(fflags).split(): - if fflag not in compileflags: - compileflags.append(fflag) + # Add passed fortran flags - assume that flags have - or / as the + # first character. fortran flags starting with O are excluded + for flag in fflags: + if flag[1] is not 'O': + if flag[1:] not in flags: + flags.append(flag[1:]) - # C/C++ compiler switches -- thanks to mja - if cflags is None: - cflags = [] - else: - if isinstance(cflags, str): - cflags = cflags.split() + # add prepend to compiler flags + for idx, flag in enumerate(flags): + flags[idx] = prepend + flag - # look for optimization levels in cflags - for cflag in cflags: - if cflag[:2] == '-O': - if not debug: - optlevel = cflag - cflags.remove(cflag) - break # after first optimization (O) flag + return flags + + +def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, + osname=None): + """Return a list of standard pymake and user specified c or cpp compiler + switches. + + Parameters + ------- + cc : str + c or cpp compiler + cflags : list + user provided list of c or cpp compiler flags + debug : bool + flag indicating a debug executable will be built + double : bool + flag indicating a double precision executable will be built + srcfiles : list + list of sourcefile names + sharedobject : bool + flag indicating a shared object (.so or .dll) will be built + osname : str + optional lower case OS name. If not passed it will be determined + using sys.platform + + Returns + ------- + flags : str + c or cpp compiler switches + """ + # remove .exe extension of necessary + if '.exe' in cc.lower(): + cc = cc[:-4] + + # get lower case OS string + if osname is None: + osname = get_osname() + + # get - or / to prepend for compiler switches + prepend = get_prepend(cc, osname) + + # generate c flags + flags = [] + if cc in ['gcc', 'g++', 'clang']: + if sharedobject: + flags.append('fPIC') + if osname == 'win32': + flags.append('Bstatic') + if debug: + flags += ['g'] + if flag_available('-Wall'): + flags.append('Wall') + else: + pass + elif cc in ['icc', 'icpc', 'mpiicc', 'mpiicpc']: + if osname == 'win32': + if debug: + flags.append('/debug:full') + else: + if sharedobject: + flags.append('fpic') + if debug: + flags += ['debug full'] + elif cc in ['cl']: + if osname == 'win32': + if debug: + flags.append('Zi') - # set additional c flags - # Debug flags - if debug: - cflags += ['-g'] # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # code that is linked to C/C++ code - # -D_UF defines UNIX naming conventions for mixed language compilation. - use_iso_c = get_iso_c(srcfiles) - if not use_iso_c: - cflags.append('-D_UF') + # code that is linked to C/C++ code. Only needed if there are + # any fortran files. -D_UF defines UNIX naming conventions for + # mixed language compilation. + ffiles = fortran_files(srcfiles) + cfiles = c_files(srcfiles) + if ffiles is not None: + use_iso_c = get_iso_c(ffiles) + if not use_iso_c and cfiles is not None: + flags.append('D_UF') + + # add passed c flags - assume that flags have - or / as the + # first character. c flags starting with O are excluded + for flag in cflags: + if flag[1] is not 'O': + if flag[1:] not in flags: + flags.append(flag[1:]) + + # add prepend to compiler flags + for idx, flag in enumerate(flags): + flags[idx] = prepend + flag + + return flags + + +def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, + syslibs, sharedobject=False, osname=None): + """Return a list of standard pymake and user specified c or cpp compiler + switches. + + Parameters + ------- + cc : str + c or cpp compiler + cflags : list + user provided list of c or cpp compiler flags + debug : bool + flag indicating a debug executable will be built + double : bool + flag indicating a double precision executable will be built + srcfiles : list + list of sourcefile names + sharedobject : bool + flag indicating a shared object (.so or .dll) will be built + osname : str + optional lower case OS name. If not passed it will be determined + using sys.platform + + Returns + ------- + flags : str + c or cpp compiler switches + """ + compiler = fc + if compiler is None: + compiler = cc + + # remove .exe extension of necessary + if '.exe' in compiler.lower(): + compiler = compiler[:-4] + + # get lower case OS string + if osname is None: + osname = get_osname() + + # get - or / to prepend for compiler switches + prepend = get_prepend(compiler, osname) + + if compiler in ['gfortran', 'ifort', 'mpiifort']: + flags = get_fortran_flags(compiler, fflags, debug, double, + sharedobject=sharedobject, osname=osname) + elif compiler in ['gcc', 'g++', 'clang', 'icc', 'icpc', + 'mpiicc', 'mpiicpc']: + flags = get_c_flags(compiler, cflags, debug, double, srcfiles, + sharedobject=sharedobject, osname=osname) + + if sharedobject: + tag = prepend + 'pic' + ipos = flags.index(tag) + if osname == 'darwin': + copt = '-dynamiclib' + else: + copt = '-shared' + flags.insert(ipos, copt) + + return flags, syslibs + + +def compile_with_macnix_ifort(srcfiles, target, fc, cc, + objdir_temp, moddir_temp, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, + srcdir, srcdir2, extrafiles, makefile, + sharedobject): + """Make target on Mac OSX.""" + # convert fflags and cflags to lists + if fflags is None: + fflags = [] + elif isinstance(fflags, str): + fflags = fflags.split() + if cflags is None: + cflags = [] + elif isinstance(cflags, str): + cflags = cflags.split() + + # set optimization levels + optlevel = get_optlevel(fc, cc, debug, fflags, cflags) + # if debug: + # optlevel = '-O0' + # else: + # optlevel = '-O2' + # # look for optimization levels in fflags + # for flag in fflags: + # if flag[:2] == '-O' or flag == '-fast': + # if not debug: + # optlevel = flag + # fflags.remove(flag) + # break # after first optimization (O) flag + # # look for optimization levels in cflags + # for flag in cflags: + # if flag[:2] == '-O': + # if not debug: + # optlevel = flag + # cflags.remove(flag) + # break # after first optimization (O) flag + + # get fortran and c compiler switches + tfflags = get_fortran_flags(fc, fflags, debug, double, + sharedobject=sharedobject) + tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, + sharedobject=sharedobject) + + # # add ifort specific compiler switches + # compileflags = [] + # if fc is not None: + # # add shared object switches + # if sharedobject: + # compileflags.append('-fpic') + # + # # Debug flags + # if debug: + # compileflags += ['-debug', 'all', + # '-no-heap-arrays', + # '-fpe0', + # '-traceback'] + # else: + # # production version compile flags + # compileflags += ['-no-heap-arrays', + # '-fpe0', + # '-traceback'] + # + # # add double precision compiler switches + # if double: + # compileflags += ['-real-size', '64'] + # compileflags += ['-double-size', '64'] + # + # # Split all tokens by spaces + # for fflag in ' '.join(fflags).split(): + # if fflag not in compileflags: + # compileflags.append(fflag) + # + # # C/C++ compiler switches -- thanks to mja + # if cflags is None: + # cflags = [] + # else: + # if isinstance(cflags, str): + # cflags = cflags.split() + # + # # look for optimization levels in cflags + # for cflag in cflags: + # if cflag[:2] == '-O': + # if not debug: + # optlevel = cflag + # cflags.remove(cflag) + # break # after first optimization (O) flag + # + # # set additional c flags + # # Debug flags + # if debug: + # cflags += ['-g'] + # + # # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran + # # code that is linked to C/C++ code + # # -D_UF defines UNIX naming conventions for mixed language compilation. + # use_iso_c = get_iso_c(srcfiles) + # if not use_iso_c: + # cflags.append('-D_UF') # build object files print('\nCompiling object files for ' + @@ -862,7 +1291,7 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, if srcfile.endswith('.c') or srcfile.endswith('.cpp'): # mja cmdlist.append(cc) # mja cmdlist.append(optlevel) - for switch in cflags: # mja + for switch in tcflags: # mja cmdlist.append(switch) # mja # add search path for any header files @@ -871,7 +1300,7 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, else: # mja cmdlist.append(fc) cmdlist.append(optlevel) - for switch in compileflags: + for switch in tfflags: cmdlist.append(switch) # put object files in objdir_temp @@ -881,10 +1310,6 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, cmdlist.append('-module') cmdlist.append(moddir_temp + '/') - # # add search path for any header files - # for sd in searchdir: - # cmdlist.append('-I {}'.format(sd)) - cmdlist.append('-c') cmdlist.append(srcfile) @@ -939,14 +1364,14 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, cmdlist.append(optlevel) if sharedobject: - ipos = compileflags.index('-fpic') + ipos = tfflags.index('-fpic') if 'darwin' in sys.platform.lower(): copt = '-dynamiclib' else: copt = '-shared' - compileflags.insert(ipos, copt) + tfflags.insert(ipos, copt) - for switch in compileflags: + for switch in tfflags: cmdlist.append(switch) cmdlist.append('-o') @@ -977,7 +1402,7 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, if makefile: create_makefile(target, srcdir, srcdir2, extrafiles, srcfiles, objfiles, - fc, compileflags, cc, cflags, syslibs, + fc, tfflags, cc, tcflags, syslibs, modules=['-module ']) # return @@ -985,11 +1410,9 @@ def compile_with_macnix_ifort(srcfiles, target, fc, cc, def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, - expedite, dryrun, double, debug, fflagsu, cflagsu, + expedite, dryrun, double, debug, fflags, cflags, syslibs, arch, srcdir, srcdir2, extrafiles, makefile): - """ - Make target on Windows OS - """ + """Make target on Windows OS.""" if fc == 'ifort': fc = 'ifort.exe' @@ -1002,50 +1425,86 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, else: cc = 'cl.exe' - # C/C++ compiler switches - cflags = ['/nologo', '/c'] + # convert fflags and cflags to lists + if fflags is None: + fflags = [] + elif isinstance(fflags, str): + fflags = fflags.split() + if cflags is None: + cflags = [] + elif isinstance(cflags, str): + cflags = cflags.split() + + # set optimization levels + optlevel = get_optlevel(fc, cc, debug, fflags, cflags) # if debug: - # cflags += ['/O0', '/g'] + # optlevel = '-O0' # else: - # cflags += ['/O3'] - - fflags = ['/heap-arrays:0', '/fpe:0', '/traceback', '/nologo'] - if debug: - optlevel = '/debug' - else: - optlevel = '/O2' - if fflagsu is None: - fflagsu = [] - elif isinstance(fflagsu, str): - fflagsu = fflagsu.split() - if cflagsu is None: - cflagsu = [] - elif isinstance(cflagsu, str): - cflagsu = cflagsu.split() - # look for optimization levels in fflags - for fflag in fflagsu: - if fflag[:2] in ('-O', '/O') or fflag in ('-fast', '/fast'): - if not debug: - optlevel = fflag - fflagsu.remove(fflag) - break # after first optimization (O) flag - if debug: - # fflags.append(optlevel) - cflags.append('/Zi') + # optlevel = '-O2' + # # look for optimization levels in fflags + # for flag in fflags: + # if flag[:2] == '-O' or flag == '-fast': + # if not debug: + # optlevel = flag + # fflags.remove(flag) + # break # after first optimization (O) flag + # # look for optimization levels in cflags + # for flag in cflags: + # if flag[:2] == '-O': + # if not debug: + # optlevel = flag + # cflags.remove(flag) + # break # after first optimization (O) flag + + # get fortran and c compiler switches + tfflags = get_fortran_flags(fc, fflags, debug, double) + tcflags = get_c_flags(cc, cflags, debug, double, srcfiles) + + # + # # C/C++ compiler switches + # cflags = ['/nologo', '/c'] + # # if debug: + # # cflags += ['/O0', '/g'] + # # else: + # # cflags += ['/O3'] + # + # fflags = ['/heap-arrays:0', '/fpe:0', '/traceback', '/nologo'] + # if debug: + # optlevel = '/debug' # else: - # # production version compile flags - # fflags.append(optlevel) - # cflags.append('/O2') - if double: - fflags.append('/real-size:64') - fflags.append('/double-size:64') - # Split all tokens by spaces - for fflag in ' '.join(fflagsu).split(): - if fflag not in fflags: - fflags.append(fflag) - for cflag in ' '.join(cflagsu).split(): - if cflag not in cflags: - cflags.append(cflag) + # optlevel = '/O2' + # if fflagsu is None: + # fflagsu = [] + # elif isinstance(fflagsu, str): + # fflagsu = fflagsu.split() + # if cflagsu is None: + # cflagsu = [] + # elif isinstance(cflagsu, str): + # cflagsu = cflagsu.split() + # # look for optimization levels in fflags + # for fflag in fflagsu: + # if fflag[:2] in ('-O', '/O') or fflag in ('-fast', '/fast'): + # if not debug: + # optlevel = fflag + # fflagsu.remove(fflag) + # break # after first optimization (O) flag + # if debug: + # # fflags.append(optlevel) + # cflags.append('/Zi') + # # else: + # # # production version compile flags + # # fflags.append(optlevel) + # # cflags.append('/O2') + # if double: + # fflags.append('/real-size:64') + # fflags.append('/double-size:64') + # # Split all tokens by spaces + # for fflag in ' '.join(fflagsu).split(): + # if fflag not in fflags: + # fflags.append(fflag) + # for cflag in ' '.join(cflagsu).split(): + # if cflag not in cflags: + # cflags.append(cflag) batchfile = 'compile.bat' if os.path.isfile(batchfile): try: @@ -1060,8 +1519,8 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if flopy_avail: if flopy_is_exe(target): os.remove(target) - makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, - arch, objdir_temp, moddir_temp) + makebatch(batchfile, fc, cc, optlevel, tfflags, tcflags, srcfiles, + target, arch, objdir_temp, moddir_temp) proc = Popen([batchfile, ], stdout=PIPE, stderr=STDOUT) while True: line = proc.stdout.readline() @@ -1090,10 +1549,7 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, arch, objdir_temp, moddir_temp): - """ - Make an ifort batch file - - """ + """Make an ifort batch file.""" iflist = ['IFORT_COMPILER{}'.format(i) for i in range(30, 12, -1)] found = False for ift in iflist: @@ -1358,10 +1814,7 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, include_subdirs=False, fflags=None, cflags=None, syslibs=None, arch='intel64', makefile=False, srcdir2=None, extrafiles=None, excludefiles=None, cmake=None, sharedobject=False): - """ - Main part of program - - """ + """Main part of program.""" # initialize success success = 0 From e1ad3bebea6abfb58caf88e7457f077bceee7308 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 17 Jun 2020 14:49:29 -0400 Subject: [PATCH 3/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to one method for compiling on Windows using Intel compilers and another method for everything else. --- pymake/build_program.py | 100 ++-- pymake/pymake.py | 990 +++++++++++----------------------------- 2 files changed, 317 insertions(+), 773 deletions(-) diff --git a/pymake/build_program.py b/pymake/build_program.py index 676185f6..c31a3908 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -276,36 +276,38 @@ def set_fflags(target, fc='gfortran'): fortran compiler flags. Default is None """ - fflags = None + fflags = [] if target == 'mp7': if fc == 'gfortran': - fflags = '-ffree-line-length-512' + fflags.append('-ffree-line-length-512') elif target == 'gsflow': if fc == 'ifort': if 'win32' in sys.platform.lower(): - fflags = '/fp:source /names:lowercase /assume:underscore' + fflags += ['-fp:source', '-names:lowercase', + '-assume:underscore'] else: - fflags = '-fp-model source' + # fflags.append('-fp-model source') + pass elif fc == 'gfortran': - fflags = '-O1 -fno-second-underscore' - # if 'win32' in sys.platform.lower(): - # fflags += ' -Bstatic -Wall' + fflags += ['-O1', '-fno-second-underscore'] # add additional fflags from the command line for idx, arg in enumerate(sys.argv): if '--fflags' in arg.lower(): - if fflags is None: - fflags = '' - if len(fflags) > 0: - fflags += ' ' - fflags += sys.argv[idx + 1] + s = sys.argv[idx + 1] + delim = ' -' + if ' /' in s: + delim = ' /' + fflags += s.split(delim) # write fortran flags - if fflags is not None: + if len(fflags) > 0: msg = '{} fortran code '.format(target) + \ 'will be built with the following predefined flags:\n' - msg += ' {}\n'.format(fflags) + msg += ' {}\n'.format(' '.join(fflags)) print(msg) + else: + fflags = None return fflags @@ -327,40 +329,39 @@ def set_cflags(target, cc='gcc'): c compiler flags. Default is None """ - cflags = None + cflags = [] if target == 'triangle': if 'linux' in sys.platform.lower() or 'darwin' in sys.platform.lower(): if cc.startswith('g'): - cflags = '-lm' + cflags += ['-lm'] else: - cflags = '-DNO_TIMER' + cflags += ['-DNO_TIMER'] elif target == 'gsflow': - if cc == 'icc' or cc == 'icl': + if cc in ['icc', 'icpl', 'icl']: if 'win32' in sys.platform.lower(): - cflags = '/D_CRT_SECURE_NO_WARNINGS' + cflags += ['-D_CRT_SECURE_NO_WARNINGS'] else: - cflags = '-D_UF' + cflags += ['-D_UF'] elif cc == 'gcc': - # cflags = '-O -D_UF' - cflags = '-O1' - # if 'win32' in sys.platform.lower(): - # cflags += ' -Bstatic -Wall' + cflags += ['-O1'] # add additional cflags from the command line for idx, arg in enumerate(sys.argv): if '--cflags' in arg.lower(): - if cflags is None: - cflags = '' - if len(cflags) > 0: - cflags += ' ' - cflags += sys.argv[idx + 1] + s = sys.argv[idx + 1] + delim = ' -' + if ' /' in s: + delim = ' /' + cflags += s.split(delim) # write c/c++ flags - if cflags is not None: + if len(cflags) > 0: msg = '{} c/c++ code '.format(target) + \ 'will be built with the following predefined flags:\n' - msg += ' {}\n'.format(cflags) + msg += ' {}\n'.format(' '.join(cflags)) print(msg) + else: + cflags = None return cflags @@ -386,9 +387,30 @@ def set_syslibs(target, fc, cc): fortran compiler flags. Default is None """ - syslibs = '-lc' + # set osname + osname = sys.platform.lower() + + # initialize syslibs + syslibs = [] + + # determine if default syslibs will be defined + default_syslibs = True + if osname == 'win32': + if fc is not None: + if fc in ['ifort', 'gfortran']: + default_syslibs = False + if default_syslibs: + if cc is not None: + if cc in ['cl', 'icl', 'gcc', 'g++']: + default_syslibs = False + + # set default syslibs + if default_syslibs: + syslibs.append('-lc') + + # add additional syslibs for select programs if target == 'triangle': - if 'linux' in sys.platform.lower() or 'darwin' in sys.platform.lower(): + if osname in ['linux', 'darwin']: if fc is None: lfc = True else: @@ -397,19 +419,15 @@ def set_syslibs(target, fc, cc): if cc in ['gcc', 'g++', 'clang', 'clang++']: lcc = True if lfc and lcc: - syslibs = '-lm' + syslibs += ['-lm'] elif target == 'gsflow': - if 'win32' not in sys.platform.lower(): + if 'win32' not in osname: if 'ifort' in fc: - syslibs = '-nofor_main' - # else: - # if 'gfortran' in fc: - # if 'win32' in sys.platform.lower(): - # syslibs = '-lgfortran -lgcc -lm' + syslibs += ['-nofor_main'] # write syslibs msg = '{} will use the following predefined syslibs:\n'.format(target) - msg += ' {}\n'.format(syslibs) + msg += ' {}\n'.format(' '.join(syslibs)) print(msg) return syslibs diff --git a/pymake/pymake.py b/pymake/pymake.py index 70fabd41..2a4db1b7 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -264,7 +264,7 @@ def parse_extrafiles(extrafiles): return files -def clean(srcdir_temp, objdir_temp, moddir_temp, objext, winifort): +def clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin): """Remove mod and object files, and remove the temp source directory.""" # clean things up print('\nCleaning up temporary source, object, and module files...') @@ -277,7 +277,7 @@ def clean(srcdir_temp, objdir_temp, moddir_temp, objext, winifort): shutil.rmtree(srcdir_temp) shutil.rmtree(objdir_temp) shutil.rmtree(moddir_temp) - if winifort: + if intelwin: os.remove('compile.bat') return @@ -381,6 +381,7 @@ def get_iso_c(srcfiles): continue lines = f.read() lines = lines.decode('ascii', 'replace').splitlines() + # develop a list of modules in the file for idx, line in enumerate(lines): linelist = line.strip().split() @@ -411,13 +412,6 @@ def flag_available(flag): # establish communicator stdout, stderr = proc.communicate() - # process_Popen_communicate(stdout, stderr) - - # # catch non-zero return code - # if proc.returncode != 0: - # msg = '{} failed, status code {}\n' \ - # .format(' '.join(cmdlist), proc.returncode) - # raise RuntimeError(msg) if PY3: stdout = stdout.decode() @@ -429,64 +423,12 @@ def flag_available(flag): return avail -def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, - expedite, dryrun, double, debug, fflags, cflags, syslibs, - srcdir, srcdir2, extrafiles, makefile, sharedobject): - """Compile the program using the gnu compilers (gfortran and gcc)""" - - # define the platform - platform = sys.platform - - # set shellflg for popen - shellflg = False - - # jdh commented out 4/29/2020 since is working with python 3 - # For horrible windows issue - # if platform == 'win32': - # shellflg = True - # - # # define the OS macro for gfortran - # if platform == 'win32': - # os_macro = '-D_WIN32' - # elif platform == 'darwin': - # os_macro = '-D__APPLE__' - # elif platform == 'linux' or platform == 'linux2': - # os_macro = '-D__linux__' - # elif 'bsd' in platform: - # os_macro = '-D__unix__' - # else: - # os_macro = None - - # convert fflags and cflags to lists - if fflags is None: - fflags = [] - elif isinstance(fflags, str): - fflags = fflags.split() - if cflags is None: - cflags = [] - elif isinstance(cflags, str): - cflags = cflags.split() - +def compile_std(srcfiles, target, fc, cc, objdir_temp, moddir_temp, + expedite, dryrun, double, debug, fflags, cflags, syslibs, + srcdir, srcdir2, extrafiles, makefile, sharedobject): + """Standard program compile""" # set optimization levels optlevel = get_optlevel(fc, cc, debug, fflags, cflags) - # if debug: - # optlevel = '-O0' - # else: - # optlevel = '-O2' - # # look for optimization levels in fflags - # for flag in fflags: - # if flag[:2] == '-O' or flag == '-fast': - # if not debug: - # optlevel = flag - # fflags.remove(flag) - # break # after first optimization (O) flag - # # look for optimization levels in cflags - # for flag in cflags: - # if flag[:2] == '-O': - # if not debug: - # optlevel = flag - # cflags.remove(flag) - # break # after first optimization (O) flag # get fortran and c compiler switches tfflags = get_fortran_flags(fc, fflags, debug, double, @@ -494,131 +436,6 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=sharedobject) - # # compiler optimization level - # if debug: - # optlevel = '-O0' - # else: - # optlevel = '-O2' - # - # # fortran compiler switches - # if fflags is None: - # fflags = [] - # elif isinstance(fflags, str): - # fflags = fflags.split() - # # look for optimization levels in fflags - # for fflag in fflags: - # if fflag[:2] == '-O': - # if not debug: - # optlevel = fflag - # fflags.remove(fflag) - # break # after first optimization (O) flag - # - # # set fortran flags - # compileflags = [] - # - # # Debug flags - # if debug: - # compileflags = ['-g'] - # - # # add gfortran specific compiler switches - # if fc is not None: - # # add shared object switches - # if sharedobject: - # compileflags.append('-fPIC') - # - # if debug: - # compileflags += ['-fcheck=all', '-fbounds-check', '-Wall'] - # lflag = flag_available('-ffpe-trap') - # if lflag: - # compileflags.append( - # '-ffpe-trap=overflow,zero,invalid,denormal') - # else: - # lflag = flag_available('-ffpe-summary') - # if lflag: - # compileflags.append('-ffpe-summary=overflow') - # lflag = flag_available('-ffpe-trap') - # if lflag: - # compileflags.append('-ffpe-trap=overflow,zero,invalid') - # - # # add fbacktrace to debug and release versions - # compileflags.append('-fbacktrace') - # - # # add static - # if sys.platform == 'win32': - # compileflags.append('-Bstatic') - # - # # add double precision switches - # if double: - # compileflags.append('-fdefault-real-8') - # compileflags.append('-fdefault-double-8') - # - # # add defined OS macro - # if os_macro is not None: - # compileflags.append(os_macro) - # - # # Split all tokens by spaces - # for fflag in ' '.join(fflags).split(): - # if fflag not in compileflags: - # compileflags.append(fflag) - # - # # C/C++ compiler switches -- thanks to mja - # if cflags is None: - # cflags = [] - # else: - # if isinstance(cflags, str): - # cflags = cflags.split() - # - # # look for optimization levels in cflags - # for cflag in cflags: - # if cflag[:2] == '-O': - # if not debug: - # optlevel = cflag - # cflags.remove(cflag) - # break # after first optimization (O) flag - # - # # set additional c flags - # # Debug flags - # if debug: - # cflags += ['-g'] - # - # if cc.startswith('g'): - # if sys.platform == 'win32': - # cflags += ['-Bstatic'] - # if debug: - # lflag = flag_available('-Wall') - # if lflag: - # cflags += ['-Wall'] - # else: - # pass - # - # # determine if any c, cpp or fortran files - # iscfiles = False - # isfortranfiles = False - # for srcfile in srcfiles: - # ext = os.path.splitext(srcfile)[1].lower() - # if ext in ['.c', '.cpp']: # mja - # iscfiles = True - # elif ext in ['.f', '.for', '.f90', '.fpp']: - # isfortranfiles = True - # - # # reset syslibs for windows - # if sys.platform == 'win32': - # syslibs = [] - # if isfortranfiles: - # syslibs.append('-lgfortran') - # if iscfiles: - # syslibs.append('-lgcc') - # syslibs.append('-lm') - # - # # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # # code that is linked to C/C++ code. Only needed if there are - # # any fortran files. -D_UF defines UNIX naming conventions for - # # mixed language compilation. - # if isfortranfiles: - # use_iso_c = get_iso_c(srcfiles) - # if not use_iso_c: - # cflags.append('-D_UF') - # build object files print('\nCompiling object files for ' + '{}...'.format(os.path.basename(target))) @@ -654,7 +471,11 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # put object files and module files in objdir_temp and moddir_temp else: cmdlist.append('-I{}'.format(objdir_temp)) - cmdlist.append('-J{}'.format(moddir_temp)) + if fc in ['ifort', 'mpiifort']: + cmdlist.append('-module') + cmdlist.append(moddir_temp + '/') + else: + cmdlist.append('-J{}'.format(moddir_temp)) cmdlist.append('-c') cmdlist.append(srcfile) @@ -673,11 +494,11 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if not out_of_date(srcfile, objfile): compilefile = False - # Compile + # Compile the source code if compilefile: if not dryrun: - proc = Popen(cmdlist, shell=shellflg, stdout=PIPE, stderr=PIPE) - process_Popen_command(shellflg, cmdlist) + proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) + process_Popen_command(False, cmdlist) # establish communicator stdout, stderr = proc.communicate() @@ -694,45 +515,30 @@ def compile_with_gnu(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # at the end objfiles.append(objfile) - # Build the link command and then link + # Build the link command and then link to create the executable msg = '\nLinking object files ' + \ 'to make {}...'.format(os.path.basename(target)) print(msg) - cmdlist = [] - if fc is None: - cmd = cc + ' ' - cmdlist.append(cc) - cmdlist.append(optlevel) - for switch in tcflags: - cmd += switch + ' ' - cmdlist.append(switch) - else: - cmd = fc + ' ' - cmdlist.append(fc) - cmdlist.append(optlevel) - - if sharedobject: - ipos = tfflags.index('-fPIC') - tfflags.insert(ipos, '-shared') - - for switch in tfflags: - if switch[:2] == '-I' or switch[:2] == '-J': - continue - cmd += switch + ' ' - cmdlist.append(switch) + tcomp, tlink_flags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, + debug, double, + srcfiles, syslibs, + sharedobject=sharedobject) + cmdlist = [tcomp, optlevel] + for switch in tlink_flags: + cmdlist.append(switch) cmdlist.append('-o') cmdlist.append(target) for objfile in objfiles: cmdlist.append(objfile) - for switch in syslibs: + for switch in tsyslibs: cmdlist.append(switch) if not dryrun: - proc = Popen(cmdlist, shell=shellflg, stdout=PIPE, stderr=PIPE) - process_Popen_command(shellflg, cmdlist) + proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) + process_Popen_command(False, cmdlist) # establish communicator stdout, stderr = proc.communicate() @@ -855,6 +661,7 @@ def c_files(srcfiles, extensions=False): l = None return l + def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): """Return a compiler optimization switch. @@ -886,11 +693,11 @@ def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): compiler = None if fc is not None: compiler = fc - elif compiler is None: + if compiler is None: compiler = cc # get - or / to prepend for compiler switches - prepend = get_prepend(fc, osname) + prepend = get_prepend(compiler, osname) # set basic optimization level if debug: @@ -920,6 +727,7 @@ def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): return optlevel + def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, osname=None): """Return a list of standard pymake and user specified fortran compiler @@ -946,75 +754,78 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, flags : str fortran compiler switches """ - # remove .exe extension of necessary - if '.exe' in fc.lower(): - fc = fc[:-4] + flags = [] - # get lower case OS string - if osname is None: - osname = get_osname() + # define fortran flags + if fc is not None: + # remove .exe extension of necessary + if '.exe' in fc.lower(): + fc = fc[:-4] - # get - or / to prepend for compiler switches - prepend = get_prepend(fc, osname) + # get lower case OS string + if osname is None: + osname = get_osname() - # generate standard fortran flags - flags = [] - if fc == 'gfortran': - if sharedobject: - flags.append('fPIC') - flags.append('fbacktrace') - if osname == 'win32': - flags.append('Bstatic') - if debug: - flags += ['g', 'fcheck=all', 'fbounds-check', 'Wall'] - if flag_available('-ffpe-trap'): - flags.append('ffpe-trap=overflow,zero,invalid,denormal') - else: - if flag_available('-ffpe-summary'): - flags.append('ffpe-summary=overflow') - if flag_available('-ffpe-trap'): - flags.append('ffpe-trap=overflow,zero,invalid') - if double: - flags += ['fdefault-real-8', 'fdefault-double-8'] - # define the OS macro for gfortran - if osname == 'win32': - os_macro = 'D_WIN32' - elif osname == 'darwin': - os_macro = 'D__APPLE__' - elif 'linux' in osname: - os_macro = 'D__linux__' - elif 'bsd' in osname: - os_macro = 'D__unix__' - else: - os_macro = None - if os_macro is not None: - flags.append(os_macro) - elif fc in ['ifort', 'mpiifort']: - if osname == 'win32': - flags += ['heap-arrays:0', 'fpe:0', 'traceback', 'nologo'] - if debug: - flags += ['debug:full', 'Zi'] - if double: - flags += ['real-size:64', 'double-size:64'] - else: + # get - or / to prepend for compiler switches + prepend = get_prepend(fc, osname) + + # generate standard fortran flags + if fc == 'gfortran': if sharedobject: - flags.append('fpic') + flags.append('fPIC') + flags.append('fbacktrace') + if osname == 'win32': + flags.append('Bstatic') if debug: - flags += ['g'] - flags += ['no-heap-arrays', 'fpe0', 'traceback'] + flags += ['g', 'fcheck=all', 'fbounds-check', 'Wall'] + if flag_available('-ffpe-trap'): + flags.append('ffpe-trap=overflow,zero,invalid,denormal') + else: + if flag_available('-ffpe-summary'): + flags.append('ffpe-summary=overflow') + if flag_available('-ffpe-trap'): + flags.append('ffpe-trap=overflow,zero,invalid') if double: - flags += ['real-size 64', 'double-size 64'] - - # Add passed fortran flags - assume that flags have - or / as the - # first character. fortran flags starting with O are excluded - for flag in fflags: - if flag[1] is not 'O': - if flag[1:] not in flags: - flags.append(flag[1:]) - - # add prepend to compiler flags - for idx, flag in enumerate(flags): - flags[idx] = prepend + flag + flags += ['fdefault-real-8', 'fdefault-double-8'] + # define the OS macro for gfortran + if osname == 'win32': + os_macro = 'D_WIN32' + elif osname == 'darwin': + os_macro = 'D__APPLE__' + elif 'linux' in osname: + os_macro = 'D__linux__' + elif 'bsd' in osname: + os_macro = 'D__unix__' + else: + os_macro = None + if os_macro is not None: + flags.append(os_macro) + elif fc in ['ifort', 'mpiifort']: + if osname == 'win32': + flags += ['heap-arrays:0', 'fpe:0', 'traceback', 'nologo'] + if debug: + flags += ['debug:full', 'Zi'] + if double: + flags += ['real-size:64', 'double-size:64'] + else: + if sharedobject: + flags.append('fPIC') + if debug: + flags += ['g'] + flags += ['no-heap-arrays', 'fpe0', 'traceback'] + if double: + flags += ['real-size 64', 'double-size 64'] + + # Add passed fortran flags - assume that flags have - or / as the + # first character. fortran flags starting with O are excluded + for flag in fflags: + if flag[1] is not 'O': + if flag[1:] not in flags: + flags.append(flag[1:]) + + # add prepend to compiler flags + for idx, flag in enumerate(flags): + flags[idx] = prepend + flag return flags @@ -1047,66 +858,70 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, flags : str c or cpp compiler switches """ - # remove .exe extension of necessary - if '.exe' in cc.lower(): - cc = cc[:-4] + flags = [] - # get lower case OS string - if osname is None: - osname = get_osname() + # define c flags + if cc is not None: + # remove .exe extension of necessary + if '.exe' in cc.lower(): + cc = cc[:-4] - # get - or / to prepend for compiler switches - prepend = get_prepend(cc, osname) + # get lower case OS string + if osname is None: + osname = get_osname() - # generate c flags - flags = [] - if cc in ['gcc', 'g++', 'clang']: - if sharedobject: - flags.append('fPIC') - if osname == 'win32': - flags.append('Bstatic') - if debug: - flags += ['g'] - if flag_available('-Wall'): - flags.append('Wall') - else: - pass - elif cc in ['icc', 'icpc', 'mpiicc', 'mpiicpc']: - if osname == 'win32': - if debug: - flags.append('/debug:full') - else: + # get - or / to prepend for compiler switches + prepend = get_prepend(cc, osname) + + # generate c flags + if cc in ['gcc', 'g++', 'clang']: if sharedobject: - flags.append('fpic') + flags.append('fPIC') + if osname == 'win32': + flags.append('Bstatic') if debug: - flags += ['debug full'] - elif cc in ['cl']: - if osname == 'win32': - if debug: - flags.append('Zi') - - - # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # code that is linked to C/C++ code. Only needed if there are - # any fortran files. -D_UF defines UNIX naming conventions for - # mixed language compilation. - ffiles = fortran_files(srcfiles) - cfiles = c_files(srcfiles) - if ffiles is not None: - use_iso_c = get_iso_c(ffiles) - if not use_iso_c and cfiles is not None: - flags.append('D_UF') - - # add passed c flags - assume that flags have - or / as the - # first character. c flags starting with O are excluded - for flag in cflags: - if flag[1] is not 'O': - if flag[1:] not in flags: - flags.append(flag[1:]) - - # add prepend to compiler flags - for idx, flag in enumerate(flags): - flags[idx] = prepend + flag + flags += ['g'] + if flag_available('-Wall'): + flags.append('Wall') + else: + pass + elif cc in ['icc', 'icpc', 'mpiicc', 'mpiicpc', 'icl']: + if osname == 'win32': + if cc == 'icl': + flags += ['nologo'] + if debug: + flags.append('/debug:full') + else: + if sharedobject: + flags.append('fpic') + if debug: + flags += ['debug full'] + elif cc in ['cl']: + if osname == 'win32': + if debug: + flags.append('Zi') + + # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran + # code that is linked to C/C++ code. Only needed if there are + # any fortran files. -D_UF defines UNIX naming conventions for + # mixed language compilation. + ffiles = fortran_files(srcfiles) + cfiles = c_files(srcfiles) + if ffiles is not None: + use_iso_c = get_iso_c(ffiles) + if not use_iso_c and cfiles is not None: + flags.append('D_UF') + + # add passed c flags - assume that flags have - or / as the + # first character. c flags starting with O are excluded + for flag in cflags: + if flag[1] is not 'O': + if flag[1:] not in flags: + flags.append(flag[1:]) + + # add prepend to compiler flags + for idx, flag in enumerate(flags): + flags[idx] = prepend + flag return flags @@ -1157,354 +972,67 @@ def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, if compiler in ['gfortran', 'ifort', 'mpiifort']: flags = get_fortran_flags(compiler, fflags, debug, double, sharedobject=sharedobject, osname=osname) - elif compiler in ['gcc', 'g++', 'clang', 'icc', 'icpc', + elif compiler in ['gcc', 'g++', 'clang', 'clang++', + 'icc', 'icpc', 'icl', 'cl', 'mpiicc', 'mpiicpc']: flags = get_c_flags(compiler, cflags, debug, double, srcfiles, sharedobject=sharedobject, osname=osname) if sharedobject: - tag = prepend + 'pic' + tag = prepend + 'fPIC' ipos = flags.index(tag) if osname == 'darwin': - copt = '-dynamiclib' + copt = prepend + 'dynamiclib' else: - copt = '-shared' + copt = prepend + 'shared' flags.insert(ipos, copt) - return flags, syslibs - - -def compile_with_macnix_ifort(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, - srcdir, srcdir2, extrafiles, makefile, - sharedobject): - """Make target on Mac OSX.""" - # convert fflags and cflags to lists - if fflags is None: - fflags = [] - elif isinstance(fflags, str): - fflags = fflags.split() - if cflags is None: - cflags = [] - elif isinstance(cflags, str): - cflags = cflags.split() - - # set optimization levels - optlevel = get_optlevel(fc, cc, debug, fflags, cflags) - # if debug: - # optlevel = '-O0' - # else: - # optlevel = '-O2' - # # look for optimization levels in fflags - # for flag in fflags: - # if flag[:2] == '-O' or flag == '-fast': - # if not debug: - # optlevel = flag - # fflags.remove(flag) - # break # after first optimization (O) flag - # # look for optimization levels in cflags - # for flag in cflags: - # if flag[:2] == '-O': - # if not debug: - # optlevel = flag - # cflags.remove(flag) - # break # after first optimization (O) flag - - # get fortran and c compiler switches - tfflags = get_fortran_flags(fc, fflags, debug, double, - sharedobject=sharedobject) - tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, - sharedobject=sharedobject) - - # # add ifort specific compiler switches - # compileflags = [] - # if fc is not None: - # # add shared object switches - # if sharedobject: - # compileflags.append('-fpic') - # - # # Debug flags - # if debug: - # compileflags += ['-debug', 'all', - # '-no-heap-arrays', - # '-fpe0', - # '-traceback'] - # else: - # # production version compile flags - # compileflags += ['-no-heap-arrays', - # '-fpe0', - # '-traceback'] - # - # # add double precision compiler switches - # if double: - # compileflags += ['-real-size', '64'] - # compileflags += ['-double-size', '64'] - # - # # Split all tokens by spaces - # for fflag in ' '.join(fflags).split(): - # if fflag not in compileflags: - # compileflags.append(fflag) - # - # # C/C++ compiler switches -- thanks to mja - # if cflags is None: - # cflags = [] - # else: - # if isinstance(cflags, str): - # cflags = cflags.split() - # - # # look for optimization levels in cflags - # for cflag in cflags: - # if cflag[:2] == '-O': - # if not debug: - # optlevel = cflag - # cflags.remove(cflag) - # break # after first optimization (O) flag - # - # # set additional c flags - # # Debug flags - # if debug: - # cflags += ['-g'] - # - # # Add -D-UF flag for C code if ISO_C_BINDING is not used in Fortran - # # code that is linked to C/C++ code - # # -D_UF defines UNIX naming conventions for mixed language compilation. - # use_iso_c = get_iso_c(srcfiles) - # if not use_iso_c: - # cflags.append('-D_UF') - - # build object files - print('\nCompiling object files for ' + - '{}...'.format(os.path.basename(target))) - objfiles = [] - - # assume that header files may be in other folders, so make a list - searchdir = [] - for f in srcfiles: - dirname = os.path.dirname(f) - if dirname not in searchdir: - searchdir.append(dirname) - - for srcfile in srcfiles: - cmdlist = [] - if srcfile.endswith('.c') or srcfile.endswith('.cpp'): # mja - cmdlist.append(cc) # mja - cmdlist.append(optlevel) - for switch in tcflags: # mja - cmdlist.append(switch) # mja - - # add search path for any header files - for sd in searchdir: - cmdlist.append('-I {}'.format(sd)) - else: # mja - cmdlist.append(fc) - cmdlist.append(optlevel) - for switch in tfflags: - cmdlist.append(switch) - - # put object files in objdir_temp - cmdlist.append('-I {}/'.format(objdir_temp)) - - # put module files in moddir_temp - cmdlist.append('-module') - cmdlist.append(moddir_temp + '/') - - cmdlist.append('-c') - cmdlist.append(srcfile) - - # object file name and location - srcname, srcext = os.path.splitext(srcfile) - srcname = srcname.split(os.path.sep)[-1] - objfile = os.path.join(objdir_temp, srcname + '.o') - cmdlist.append('-o') - cmdlist.append(objfile) - - # If expedited, then check if object file is out of date (if exists). - # No need to compile if object file is newer. - compilefile = True - if expedite: - if not out_of_date(srcfile, objfile): - compilefile = False - - # Compile - if compilefile: - if not dryrun: - # subprocess.check_call(cmdlist) - proc = Popen(cmdlist, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) - - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) - - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode - - # Save the name of the object file so that they can all be linked - # at the end - objfiles.append(objfile) - - # Build the link command and then link - print(('\nLinking object files to make ' + - '{}...'.format(os.path.basename(target)))) - - cmdlist = [] - if fc is None: - cmdlist.append(cc) - cmdlist.append(optlevel) - for switch in cflags: - cmdlist.append(switch) - else: - cmdlist.append(fc) - cmdlist.append(optlevel) - - if sharedobject: - ipos = tfflags.index('-fpic') - if 'darwin' in sys.platform.lower(): - copt = '-dynamiclib' - else: - copt = '-shared' - tfflags.insert(ipos, copt) - - for switch in tfflags: - cmdlist.append(switch) + # set outgoing syslibs + syslibs_out = [] - cmdlist.append('-o') - # cmdlist.append(os.path.join('.', target)) - cmdlist.append(target) - for objfile in objfiles: - cmdlist.append(objfile) - for switch in syslibs: - cmdlist.append(switch) + # add passed syslibs flags - assume that flags have - or / as the + # first character. + for flag in syslibs: + if flag[1:] not in syslibs_out: + syslibs_out.append(flag[1:]) - if not dryrun: - # subprocess.check_call(cmdlist) - proc = Popen(cmdlist, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) + # add prepend to syslibs flags + for idx, flag in enumerate(syslibs_out): + syslibs_out[idx] = prepend + flag - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) + return compiler, flags, syslibs_out - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode - - # create makefile - if makefile: - create_makefile(target, srcdir, srcdir2, extrafiles, - srcfiles, objfiles, - fc, tfflags, cc, tcflags, syslibs, - modules=['-module ']) - - # return - return 0 - -def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, - expedite, dryrun, double, debug, fflags, cflags, - syslibs, arch, srcdir, srcdir2, extrafiles, makefile): +def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, + expedite, dryrun, double, debug, fflags, cflags, + syslibs, arch, srcdir, srcdir2, extrafiles, makefile): """Make target on Windows OS.""" + ext = '.exe' if fc == 'ifort': - fc = 'ifort.exe' + fc += ext elif fc is not None: - fc = '{}.exe'.format(fc) + fc += ext if cc == 'icc': - cc = 'icc.exe' + cc += ext elif cc == 'icl': - cc = 'icl.exe' + cc += ext else: - cc = 'cl.exe' - - # convert fflags and cflags to lists - if fflags is None: - fflags = [] - elif isinstance(fflags, str): - fflags = fflags.split() - if cflags is None: - cflags = [] - elif isinstance(cflags, str): - cflags = cflags.split() + cc = 'cl' + ext # set optimization levels optlevel = get_optlevel(fc, cc, debug, fflags, cflags) - # if debug: - # optlevel = '-O0' - # else: - # optlevel = '-O2' - # # look for optimization levels in fflags - # for flag in fflags: - # if flag[:2] == '-O' or flag == '-fast': - # if not debug: - # optlevel = flag - # fflags.remove(flag) - # break # after first optimization (O) flag - # # look for optimization levels in cflags - # for flag in cflags: - # if flag[:2] == '-O': - # if not debug: - # optlevel = flag - # cflags.remove(flag) - # break # after first optimization (O) flag # get fortran and c compiler switches tfflags = get_fortran_flags(fc, fflags, debug, double) tcflags = get_c_flags(cc, cflags, debug, double, srcfiles) - # - # # C/C++ compiler switches - # cflags = ['/nologo', '/c'] - # # if debug: - # # cflags += ['/O0', '/g'] - # # else: - # # cflags += ['/O3'] - # - # fflags = ['/heap-arrays:0', '/fpe:0', '/traceback', '/nologo'] - # if debug: - # optlevel = '/debug' - # else: - # optlevel = '/O2' - # if fflagsu is None: - # fflagsu = [] - # elif isinstance(fflagsu, str): - # fflagsu = fflagsu.split() - # if cflagsu is None: - # cflagsu = [] - # elif isinstance(cflagsu, str): - # cflagsu = cflagsu.split() - # # look for optimization levels in fflags - # for fflag in fflagsu: - # if fflag[:2] in ('-O', '/O') or fflag in ('-fast', '/fast'): - # if not debug: - # optlevel = fflag - # fflagsu.remove(fflag) - # break # after first optimization (O) flag - # if debug: - # # fflags.append(optlevel) - # cflags.append('/Zi') - # # else: - # # # production version compile flags - # # fflags.append(optlevel) - # # cflags.append('/O2') - # if double: - # fflags.append('/real-size:64') - # fflags.append('/double-size:64') - # # Split all tokens by spaces - # for fflag in ' '.join(fflagsu).split(): - # if fflag not in fflags: - # fflags.append(fflag) - # for cflag in ' '.join(cflagsu).split(): - # if cflag not in cflags: - # cflags.append(cflag) + # get linker flags + tcomp, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, + debug, double, + srcfiles, syslibs) + + # delete the batch file if it exists batchfile = 'compile.bat' if os.path.isfile(batchfile): try: @@ -1512,15 +1040,19 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, except: pass - # Create target + # Create target using a batch file on Windows try: # clean exe prior to build so that test for exe below can return a # non-zero error code if flopy_avail: if flopy_is_exe(target): os.remove(target) - makebatch(batchfile, fc, cc, optlevel, tfflags, tcflags, srcfiles, - target, arch, objdir_temp, moddir_temp) + + makebatch(batchfile, fc, cc, optlevel, + tfflags, tcflags, tlflags, tsyslibs, + srcfiles, target, arch, objdir_temp, moddir_temp) + + # run the batch file proc = Popen([batchfile, ], stdout=PIPE, stderr=STDOUT) while True: line = proc.stdout.readline() @@ -1530,6 +1062,8 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, print('{}'.format(c)) else: break + + # evaluate if the executable is available if flopy_avail: if not flopy_is_exe(target): return 1 @@ -1541,15 +1075,16 @@ def compile_with_ifort(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # create makefile if makefile: - print('makefile not created for Windows with Intel Compiler.') + msg = 'makefile not created for Windows with Intel Compilers.' + print(msg) # return return 0 -def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, - arch, objdir_temp, moddir_temp): - """Make an ifort batch file.""" +def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, lflags, syslibs, + srcfiles, target, arch, objdir_temp, moddir_temp): + """Make an ifort batch file for compiling on windows.""" iflist = ['IFORT_COMPILER{}'.format(i) for i in range(30, 12, -1)] found = False for ift in iflist: @@ -1579,6 +1114,7 @@ def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, cmd = cc + ' ' + optlevel + ' ' for switch in cflags: cmd += switch + ' ' + cmd += '/c' + ' ' # add search path for any header files for sd in searchdir: @@ -1587,27 +1123,34 @@ def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, srcfiles, target, obj = os.path.join(objdir_temp, os.path.splitext(os.path.basename(srcfile))[0] + '.obj') - cmd += '/Fo' + obj + ' ' + cmd += '/Fo:' + obj + ' ' cmd += srcfile else: cmd = fc + ' ' + optlevel + ' ' for switch in fflags: cmd += switch + ' ' - cmd += '-c' + ' ' + cmd += '/c' + ' ' cmd += '/module:{0}\\ '.format(moddir_temp) cmd += '/object:{0}\\ '.format(objdir_temp) cmd += srcfile - f.write('echo ' + os.path.basename(srcfile) + '\n') + f.write("echo compiling '" + os.path.basename(srcfile) + "'\n") f.write(cmd + '\n') # write commands to link + line = "echo Linking oject files to create '" + \ + os.path.basename(target) + "'\n" + f.write(line) if fc is None: - cmd = cc + ' ' + optlevel + ' ' + cmd = cc else: - cmd = fc + ' ' + optlevel + ' ' - for switch in fflags: - cmd += switch + ' ' - cmd += '-o' + ' ' + target + ' ' + objdir_temp + '\\*.obj' + '\n' + cmd = fc + cmd += ' ' + optlevel + for switch in lflags: + cmd += ' ' + switch + cmd += ' ' + '-o' + ' ' + target + ' ' + objdir_temp + '\\*.obj' + for switch in syslibs: + cmd += ' ' + switch + cmd += '\n' f.write(cmd) f.close() return @@ -1672,26 +1215,6 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, if odir not in odirs: odirs.append(odir) - # line = 'VPATH = ' - # for idx, dir in enumerate(odirs): - # #srcdirs.append('SOURCEDIR{}'.format(idx + 1)) - # #line = '{}={}\n'.format(srcdirs[idx], dir) - # #f.write(line) - # line += '{} '.format(dir) - # line += '\n' - # f.write('{}\n'.format(line)) - # # f.write('\n') - # f.write('VPATH = \\\n') - # for idx, sd in enumerate(srcdirs): - # f.write('${' + '{}'.format(sd) + '} ') - # if idx + 1 < len(srcdirs): - # f.write('\\') - # f.write('\n') - # f.write('\n') - - # 'SRCS =$(wildcard $(addsuffix / *.f90, $(SUBDIRS))) - # OBJS =$(filter - out cusg_wrap.o, ${SRCS:.cpp=.o}) - ffiles = ['.f', '.f90', '.F90', '.fpp'] cfiles = ['.c', '.cpp'] @@ -1815,8 +1338,22 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, arch='intel64', makefile=False, srcdir2=None, extrafiles=None, excludefiles=None, cmake=None, sharedobject=False): """Main part of program.""" - # initialize success - success = 0 + # initialize return code + returncode = 0 + + # convert fflags, cflags, and syslibs to lists + if fflags is None: + fflags = [] + elif isinstance(fflags, str): + fflags = fflags.split() + if cflags is None: + cflags = [] + elif isinstance(cflags, str): + cflags = cflags.split() + if syslibs is None: + syslibs = [] + elif isinstance(syslibs, str): + syslibs = syslibs.split() # write summary information print('\nsource files are in:\n {}\n'.format(srcdir)) @@ -1865,53 +1402,42 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, syslibs = syslibs.split() # compile with gfortran or ifort - winifort = False - if fc == 'gfortran' or (fc is None and cc in ['gcc', 'g++', - 'clang', 'clang++']): + intelwin = False + if 'win32' in sys.platform.lower(): + if fc is not None: + if fc in ['ifort', 'mpiifort']: + intelwin = True + if cc is not None: + if cc in ['cl', 'icl']: + intelwin = True + if intelwin: + objext = '.obj' + returncode = compile_intel_win(srcfiles, target, fc, cc, + objdir_temp, moddir_temp, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, arch, + srcdir, srcdir2, + extrafiles, makefile) + else: objext = '.o' if sharedobject: ext = os.path.splitext(target)[-1].lower() if ext != '.so': target += '.so' create_openspec(srcdir_temp) - returncode = compile_with_gnu(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, - expedite, dryrun, - double, debug, - fflags, cflags, syslibs, - srcdir, srcdir2, - extrafiles, makefile, sharedobject) - elif fc == 'ifort' or fc == 'mpiifort' or \ - (fc is None and cc in ['icc', 'icpc', 'cl', 'icl']): - platform = sys.platform - if 'darwin' in platform.lower() or 'linux' in platform.lower(): - objext = '.o' - create_openspec(srcdir_temp) - returncode = compile_with_macnix_ifort(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, - expedite, dryrun, - double, debug, - fflags, cflags, syslibs, - srcdir, srcdir2, - extrafiles, makefile, - sharedobject) - else: - winifort = True - objext = '.obj' - returncode = compile_with_ifort(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, arch, - srcdir, srcdir2, - extrafiles, makefile) - else: - raise Exception('Unsupported compiler') + returncode = compile_std(srcfiles, target, fc, cc, + objdir_temp, moddir_temp, + expedite, dryrun, + double, debug, + fflags, cflags, syslibs, + srcdir, srcdir2, + extrafiles, makefile, sharedobject) # Clean it up if makeclean and returncode == 0: - clean(srcdir_temp, objdir_temp, moddir_temp, objext, winifort) + clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin) - return success + return returncode if __name__ == "__main__": From 5fd518a47847cec4d121c0cd8ebbd5b99c75dead Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 17 Jun 2020 15:32:16 -0400 Subject: [PATCH 4/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to one method for compiling on Windows using Intel compilers and another method for everything else. Simplify call statements to remove temporary directories and use global variables for temporary directories. --- pymake/pymake.py | 116 ++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/pymake/pymake.py b/pymake/pymake.py index 2a4db1b7..fd297b08 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -34,6 +34,11 @@ PY3 = sys.version_info[0] >= 3 +# define temporary directories +srcdir_temp = os.path.join('.', 'src_temp') +objdir_temp = os.path.join('.', 'obj_temp') +moddir_temp = os.path.join('.', 'mod_temp') + def parser(): """Construct the parser and return argument values.""" @@ -115,7 +120,8 @@ def parser(): def process_Popen_command(shellflg, cmdlist): - """Generic function to write Popen command data to the screen. + """ + Generic function to write Popen command data to the screen. Parameters ---------- @@ -134,7 +140,8 @@ def process_Popen_command(shellflg, cmdlist): def process_Popen_communicate(stdout, stderr): - """Generic function to write communication information from Popen to the + """ + Generic function to write communication information from Popen to the screen. Parameters @@ -160,21 +167,19 @@ def process_Popen_communicate(stdout, stderr): def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): - """Remove temp source directory and target, and then copy source into + """ + Remove temp source directory and target, and then copy source into source temp directory. Return temp directory path. """ # remove the target if it already exists - srcdir_temp = os.path.join('.', 'src_temp') - objdir_temp = os.path.join('.', 'obj_temp') - moddir_temp = os.path.join('.', 'mod_temp') - - # remove srcdir_temp and copy in srcdir try: os.remove(target) except: pass + + # remove srcdir_temp and copy in srcdir try: shutil.rmtree(srcdir_temp) except: @@ -230,16 +235,13 @@ def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): os.remove(dst) tail = False - # set srcdir_temp - srcdir_temp = os.path.join(srcdir_temp) - # if they don't exist, create directories for objects and mods if not os.path.exists(objdir_temp): os.makedirs(objdir_temp) if not os.path.exists(moddir_temp): os.makedirs(moddir_temp) - return srcdir_temp, objdir_temp, moddir_temp + return def parse_extrafiles(extrafiles): @@ -264,8 +266,11 @@ def parse_extrafiles(extrafiles): return files -def clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin): - """Remove mod and object files, and remove the temp source directory.""" +def clean(objext, intelwin): + """ + Remove mod and object files, and remove the temp source directory. + + """ # clean things up print('\nCleaning up temporary source, object, and module files...') filelist = os.listdir('.') @@ -274,6 +279,8 @@ def clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin): for ext in delext: if f.endswith(ext): os.remove(f) + + # remove temporary directories shutil.rmtree(srcdir_temp) shutil.rmtree(objdir_temp) shutil.rmtree(moddir_temp) @@ -282,8 +289,9 @@ def clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin): return -def get_ordered_srcfiles(srcdir_temp, include_subdir=False): - """Create a list of ordered source files (both fortran and c). +def get_ordered_srcfiles(include_subdir=False): + """ + Create a list of ordered source files (both fortran and c). Ordering is build using a directed acyclic graph to determine module dependencies. @@ -334,8 +342,9 @@ def get_ordered_srcfiles(srcdir_temp, include_subdir=False): return orderedsourcefiles -def create_openspec(srcdir_temp): - """Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses +def create_openspec(): + """ + Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses STREAM ACCESS. This is specific to MODFLOW and MT3D based targets. @@ -395,7 +404,8 @@ def get_iso_c(srcfiles): def flag_available(flag): - """Determine if a specified flag exists. + """ + Determine if a specified flag exists. Not all flags will be detected, for example -O2 -fbounds-check=on """ @@ -563,7 +573,8 @@ def compile_std(srcfiles, target, fc, cc, objdir_temp, moddir_temp, def get_osname(): - """Return the lower case OS platform name. + """ + Return the lower case OS platform name. Parameters ------- @@ -577,7 +588,8 @@ def get_osname(): def get_prepend(compiler, osname): - """Return the appropriate prepend for a compiler switch for a OS. + """ + Return the appropriate prepend for a compiler switch for a OS. Parameters ------- @@ -602,7 +614,8 @@ def get_prepend(compiler, osname): def fortran_files(srcfiles, extensions=False): - """Return a list of fortran files or unique fortran file extensions. + """ + Return a list of fortran files or unique fortran file extensions. Parameters ------- @@ -633,7 +646,8 @@ def fortran_files(srcfiles, extensions=False): def c_files(srcfiles, extensions=False): - """Return a list of c and cpp files or unique c and cpp file extensions. + """ + Return a list of c and cpp files or unique c and cpp file extensions. Parameters ------- @@ -663,7 +677,8 @@ def c_files(srcfiles, extensions=False): def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): - """Return a compiler optimization switch. + """ + Return a compiler optimization switch. Parameters ------- @@ -730,7 +745,8 @@ def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, osname=None): - """Return a list of standard pymake and user specified fortran compiler + """ + Return a list of standard pymake and user specified fortran compiler switches. Parameters @@ -832,7 +848,8 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, osname=None): - """Return a list of standard pymake and user specified c or cpp compiler + """ + Return a list of standard pymake and user specified c or cpp compiler switches. Parameters @@ -928,7 +945,8 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, syslibs, sharedobject=False, osname=None): - """Return a list of standard pymake and user specified c or cpp compiler + """ + Return a list of standard pymake and user specified c or cpp compiler switches. Parameters @@ -1028,7 +1046,7 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, tcflags = get_c_flags(cc, cflags, debug, double, srcfiles) # get linker flags - tcomp, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, + lc, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, syslibs) @@ -1048,7 +1066,7 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, if flopy_is_exe(target): os.remove(target) - makebatch(batchfile, fc, cc, optlevel, + makebatch(batchfile, fc, cc, lc, optlevel, tfflags, tcflags, tlflags, tsyslibs, srcfiles, target, arch, objdir_temp, moddir_temp) @@ -1082,9 +1100,10 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, return 0 -def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, lflags, syslibs, +def makebatch(batchfile, fc, cc, lc, optlevel, fflags, cflags, lflags, syslibs, srcfiles, target, arch, objdir_temp, moddir_temp): """Make an ifort batch file for compiling on windows.""" + # get path to compilervars batch file iflist = ['IFORT_COMPILER{}'.format(i) for i in range(30, 12, -1)] found = False for ift in iflist: @@ -1140,11 +1159,9 @@ def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, lflags, syslibs, line = "echo Linking oject files to create '" + \ os.path.basename(target) + "'\n" f.write(line) - if fc is None: - cmd = cc - else: - cmd = fc - cmd += ' ' + optlevel + + # assemble the link command + cmd = lc + ' ' + optlevel for switch in lflags: cmd += ' ' + switch cmd += ' ' + '-o' + ' ' + target + ' ' + objdir_temp + '\\*.obj' @@ -1152,7 +1169,10 @@ def makebatch(batchfile, fc, cc, optlevel, fflags, cflags, lflags, syslibs, cmd += ' ' + switch cmd += '\n' f.write(cmd) + + # close the batch file f.close() + return @@ -1370,9 +1390,8 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, os.makedirs(pth) # initialize - srcdir_temp, objdir_temp, moddir_temp = initialize(srcdir, target, - srcdir2, extrafiles, - excludefiles) + initialize(srcdir, target, srcdir2, extrafiles, excludefiles) + if cmake is not None: if excludefiles is not None: efiles = [os.path.basename(fpth) for fpth in @@ -1390,16 +1409,7 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, f.close() # get ordered list of files to compile - srcfiles = get_ordered_srcfiles(srcdir_temp, include_subdirs) - - # add default syslibs - if syslibs is None: - if sys.platform != 'win32': - syslibs = '-lc' - - # convert syslibs to a list - if isinstance(syslibs, str): - syslibs = syslibs.split() + srcfiles = get_ordered_srcfiles(include_subdirs) # compile with gfortran or ifort intelwin = False @@ -1424,7 +1434,11 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, ext = os.path.splitext(target)[-1].lower() if ext != '.so': target += '.so' - create_openspec(srcdir_temp) + + # update openspec files + create_openspec() + + # compile the code returncode = compile_std(srcfiles, target, fc, cc, objdir_temp, moddir_temp, expedite, dryrun, @@ -1433,9 +1447,9 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, srcdir, srcdir2, extrafiles, makefile, sharedobject) - # Clean it up + # clean up temporary files if makeclean and returncode == 0: - clean(srcdir_temp, objdir_temp, moddir_temp, objext, intelwin) + clean(objext, intelwin) return returncode From 84e64f7ae2cdeacf6b52dc4f875f6a96bf32dee3 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Wed, 17 Jun 2020 21:07:33 -0400 Subject: [PATCH 5/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to one method for compiling on Windows using Intel compilers and another method for everything else. Simplify call statements to remove temporary directories and use global variables for temporary directories. Refactor makefile created by pymake. --- autotest/t006_test.py | 28 +- autotest/t008_test.py | 56 +++- pymake/build_program.py | 2 +- pymake/pymake.py | 701 ++++++++++++++++++++++++---------------- 4 files changed, 489 insertions(+), 298 deletions(-) diff --git a/autotest/t006_test.py b/autotest/t006_test.py index f1ea0e38..12525e12 100644 --- a/autotest/t006_test.py +++ b/autotest/t006_test.py @@ -42,13 +42,17 @@ def build_with_makefile(): tepth += '.exe' # remove existing target - print('Removing ' + target) - os.remove(tepth) + if os.path.isfile(tepth): + print('Removing ' + target) + os.remove(tepth) print('Removing temporary build directories') - shutil.rmtree(os.path.join('src_temp')) - shutil.rmtree(os.path.join('obj_temp')) - shutil.rmtree(os.path.join('mod_temp')) + dirs_temp = [os.path.join('src_temp'), + os.path.join('obj_temp'), + os.path.join('mod_temp')] + for d in dirs_temp: + if os.path.isdir(d): + shutil.rmtree(d) # build MODFLOW-NWT with makefile print('build {} with makefile'.format(target)) @@ -62,10 +66,16 @@ def build_with_makefile(): def clean_up(): # clean up make file - if os.path.isfile('makefile'): - print('Removing makefile and temporary build directories') - shutil.rmtree(os.path.join('obj_temp')) - os.remove('makefile') + files = ['makefile', 'makedefaults'] + print('Removing makefile and temporary build directories') + for fpth in files: + if os.path.isfile(fpth): + os.remove(fpth) + dirs_temp = [os.path.join('obj_temp'), + os.path.join('mod_temp')] + for d in dirs_temp: + if os.path.isdir(d): + shutil.rmtree(d) # clean up MODFLOW-NWT download if os.path.isdir(mfnwtpth): diff --git a/autotest/t008_test.py b/autotest/t008_test.py index 15a09297..5fc877c7 100644 --- a/autotest/t008_test.py +++ b/autotest/t008_test.py @@ -36,10 +36,54 @@ def compile_code(): pymake.build_program(target=target, include_subdirs=True, download_dir=dstpth, - replace_function=replace_function) + replace_function=replace_function, + dryrun=False, + makefile=True) + + +def build_with_makefile(): + if os.path.isfile('makefile'): + + tepth = target + if sys.platform == 'win32': + tepth += '.exe' + + # remove existing target + if os.path.isfile(tepth): + print('Removing ' + target) + os.remove(tepth) + + print('Removing temporary build directories') + dirs_temp = [os.path.join('src_temp'), + os.path.join('obj_temp'), + os.path.join('mod_temp')] + for d in dirs_temp: + if os.path.isdir(d): + shutil.rmtree(d) + + # build MODFLOW 6 with makefile + print('build {} with makefile'.format(target)) + os.system('make') + assert os.path.isfile(tepth), \ + '{} created by makefile does not exist.'.format(target) + else: + print('makefile does not exist...skipping build_with_make()') + return def clean_up(): + # clean up makefile + files = ['makefile', 'makedefaults'] + print('Removing makefile and temporary build directories') + for fpth in files: + if os.path.isfile(fpth): + os.remove(fpth) + dirs_temp = [os.path.join('obj_temp'), + os.path.join('mod_temp')] + for d in dirs_temp: + if os.path.isdir(d): + shutil.rmtree(d) + # clean up print('Removing folder ' + mf6pth) shutil.rmtree(mf6pth) @@ -85,6 +129,10 @@ def test_mf6(): yield run_mf6, d +def test_makefile(): + build_with_makefile() + + def test_clean_up(): yield clean_up @@ -92,10 +140,16 @@ def test_clean_up(): if __name__ == "__main__": # compile MODFLOW 6 compile_code() + # get name files and simulation name example_dirs = get_example_dirs() + # run models for d in example_dirs: run_mf6(d) + + # build modflow 6 with a pymake generated makefile + build_with_makefile() + # clean up clean_up() diff --git a/pymake/build_program.py b/pymake/build_program.py index c31a3908..9232a67b 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -787,7 +787,7 @@ def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, msg = 'failure to build {}.'.format(app) assert returncode == 0, msg - if verify: + if verify and not dryrun: msg = '{} build failure.'.format(app) assert os.path.isfile(exe_name), msg diff --git a/pymake/pymake.py b/pymake/pymake.py index fd297b08..350e80df 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -289,7 +289,7 @@ def clean(objext, intelwin): return -def get_ordered_srcfiles(include_subdir=False): +def get_ordered_srcfiles(srcdir, include_subdir=False): """ Create a list of ordered source files (both fortran and c). @@ -299,10 +299,10 @@ def get_ordered_srcfiles(include_subdir=False): # create a list of all c(pp), f and f90 source files templist = [] - for path, subdirs, files in os.walk(srcdir_temp): + for path, subdirs, files in os.walk(srcdir): for name in files: if not include_subdir: - if path != srcdir_temp: + if path != srcdir: continue f = os.path.join(os.path.join(path, name)) templist.append(f) @@ -320,14 +320,14 @@ def get_ordered_srcfiles(include_subdir=False): srcfileswithpath = [] for srcfile in srcfiles: - s = os.path.join(srcdir_temp, srcfile) + s = os.path.join(srcdir, srcfile) s = srcfile srcfileswithpath.append(s) # from mja cfileswithpath = [] for srcfile in cfiles: - s = os.path.join(srcdir_temp, srcfile) + s = os.path.join(srcdir, srcfile) s = srcfile cfileswithpath.append(s) @@ -433,145 +433,6 @@ def flag_available(flag): return avail -def compile_std(srcfiles, target, fc, cc, objdir_temp, moddir_temp, - expedite, dryrun, double, debug, fflags, cflags, syslibs, - srcdir, srcdir2, extrafiles, makefile, sharedobject): - """Standard program compile""" - # set optimization levels - optlevel = get_optlevel(fc, cc, debug, fflags, cflags) - - # get fortran and c compiler switches - tfflags = get_fortran_flags(fc, fflags, debug, double, - sharedobject=sharedobject) - tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, - sharedobject=sharedobject) - - # build object files - print('\nCompiling object files for ' + - '{}...'.format(os.path.basename(target))) - objfiles = [] - - # assume that header files may be in other folders, so make a list - searchdir = [] - for f in srcfiles: - dirname = os.path.dirname(f) - if dirname not in searchdir: - searchdir.append(dirname) - - for srcfile in srcfiles: - cmdlist = [] - iscfile = False - ext = os.path.splitext(srcfile)[1].lower() - if ext in ['.c', '.cpp']: # mja - iscfile = True - cmdlist.append(cc) # mja - cmdlist.append(optlevel) - for switch in tcflags: # mja - cmdlist.append(switch) # mja - else: # mja - cmdlist.append(fc) - cmdlist.append(optlevel) - for switch in tfflags: - cmdlist.append(switch) - - # add search path for any c and c++ header files - if iscfile: - for sd in searchdir: - cmdlist.append('-I{}'.format(sd)) - # put object files and module files in objdir_temp and moddir_temp - else: - cmdlist.append('-I{}'.format(objdir_temp)) - if fc in ['ifort', 'mpiifort']: - cmdlist.append('-module') - cmdlist.append(moddir_temp + '/') - else: - cmdlist.append('-J{}'.format(moddir_temp)) - - cmdlist.append('-c') - cmdlist.append(srcfile) - - # object file name and location - srcname, srcext = os.path.splitext(srcfile) - srcname = srcname.split(os.path.sep)[-1] - objfile = os.path.join(objdir_temp, srcname + '.o') - cmdlist.append('-o') - cmdlist.append(objfile) - - # If expedited, then check if object file is out of date (if exists). - # No need to compile if object file is newer. - compilefile = True - if expedite: - if not out_of_date(srcfile, objfile): - compilefile = False - - # Compile the source code - if compilefile: - if not dryrun: - proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) - - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) - - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode - - # Save the name of the object file so that they can all be linked - # at the end - objfiles.append(objfile) - - # Build the link command and then link to create the executable - msg = '\nLinking object files ' + \ - 'to make {}...'.format(os.path.basename(target)) - print(msg) - - tcomp, tlink_flags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, - debug, double, - srcfiles, syslibs, - sharedobject=sharedobject) - cmdlist = [tcomp, optlevel] - for switch in tlink_flags: - cmdlist.append(switch) - - cmdlist.append('-o') - cmdlist.append(target) - for objfile in objfiles: - cmdlist.append(objfile) - - for switch in tsyslibs: - cmdlist.append(switch) - - if not dryrun: - proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) - - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) - - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode - - # create makefile - if makefile: - create_makefile(target, srcdir, srcdir2, extrafiles, - srcfiles, objfiles, - fc, tfflags, cc, tcflags, syslibs, - modules=['-I', '-J']) - - # return - return 0 - - def get_osname(): """ Return the lower case OS platform name. @@ -835,7 +696,7 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, # Add passed fortran flags - assume that flags have - or / as the # first character. fortran flags starting with O are excluded for flag in fflags: - if flag[1] is not 'O': + if flag[1] != 'O': if flag[1:] not in flags: flags.append(flag[1:]) @@ -932,7 +793,7 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, # add passed c flags - assume that flags have - or / as the # first character. c flags starting with O are excluded for flag in cflags: - if flag[1] is not 'O': + if flag[1] != 'O': if flag[1:] not in flags: flags.append(flag[1:]) @@ -1021,7 +882,142 @@ def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, return compiler, flags, syslibs_out -def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, +def compile_std(srcfiles, target, fc, cc, + expedite, dryrun, double, debug, fflags, cflags, syslibs, + srcdir, srcdir2, extrafiles, makefile, sharedobject): + """ + Standard compile method + + """ + # set optimization levels + optlevel = get_optlevel(fc, cc, debug, fflags, cflags) + + # get fortran and c compiler switches + tfflags = get_fortran_flags(fc, fflags, debug, double, + sharedobject=sharedobject) + tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, + sharedobject=sharedobject) + + # build object files + print('\nCompiling object files for ' + + '{}...'.format(os.path.basename(target))) + objfiles = [] + + # assume that header files may be in other folders, so make a list + searchdir = [] + for f in srcfiles: + dirname = os.path.dirname(f) + if dirname not in searchdir: + searchdir.append(dirname) + + for srcfile in srcfiles: + cmdlist = [] + iscfile = False + ext = os.path.splitext(srcfile)[1].lower() + if ext in ['.c', '.cpp']: # mja + iscfile = True + cmdlist.append(cc) # mja + cmdlist.append(optlevel) + for switch in tcflags: # mja + cmdlist.append(switch) # mja + else: # mja + cmdlist.append(fc) + cmdlist.append(optlevel) + for switch in tfflags: + cmdlist.append(switch) + + # add search path for any c and c++ header files + if iscfile: + for sd in searchdir: + cmdlist.append('-I{}'.format(sd)) + # put object files and module files in objdir_temp and moddir_temp + else: + cmdlist.append('-I{}'.format(objdir_temp)) + if fc in ['ifort', 'mpiifort']: + cmdlist.append('-module') + cmdlist.append(moddir_temp + '/') + else: + cmdlist.append('-J{}'.format(moddir_temp)) + + cmdlist.append('-c') + cmdlist.append(srcfile) + + # object file name and location + srcname, srcext = os.path.splitext(srcfile) + srcname = srcname.split(os.path.sep)[-1] + objfile = os.path.join(objdir_temp, srcname + '.o') + cmdlist.append('-o') + cmdlist.append(objfile) + + # If expedited, then check if object file is out of date (if exists). + # No need to compile if object file is newer. + compilefile = True + if expedite: + if not out_of_date(srcfile, objfile): + compilefile = False + + # Compile the source code + if compilefile: + if not dryrun: + proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) + process_Popen_command(False, cmdlist) + + # establish communicator + stdout, stderr = proc.communicate() + process_Popen_communicate(stdout, stderr) + + # catch non-zero return code + if proc.returncode != 0: + msg = '{} failed, status code {}\n' \ + .format(' '.join(cmdlist), proc.returncode) + print(msg) + return proc.returncode + + # Save the name of the object file so that they can all be linked + # at the end + objfiles.append(objfile) + + # Build the link command and then link to create the executable + msg = '\nLinking object files ' + \ + 'to make {}...'.format(os.path.basename(target)) + print(msg) + + tcomp, tlink_flags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, + debug, double, + srcfiles, syslibs, + sharedobject=sharedobject) + cmdlist = [tcomp, optlevel] + for switch in tlink_flags: + cmdlist.append(switch) + + cmdlist.append('-o') + cmdlist.append(target) + for objfile in objfiles: + cmdlist.append(objfile) + + for switch in tsyslibs: + cmdlist.append(switch) + + if not dryrun: + proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) + process_Popen_command(False, cmdlist) + + # establish communicator + stdout, stderr = proc.communicate() + process_Popen_communicate(stdout, stderr) + + # catch non-zero return code + if proc.returncode != 0: + msg = '{} failed, status code {}\n' \ + .format(' '.join(cmdlist), proc.returncode) + print(msg) + return proc.returncode + + # return + return 0 + + +def compile_intel_win(srcfiles, target, fc, cc, expedite, dryrun, double, debug, fflags, cflags, syslibs, arch, srcdir, srcdir2, extrafiles, makefile): """Make target on Windows OS.""" @@ -1047,8 +1043,8 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, # get linker flags lc, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, - debug, double, - srcfiles, syslibs) + debug, double, + srcfiles, syslibs) # delete the batch file if it exists batchfile = 'compile.bat' @@ -1068,7 +1064,7 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, makebatch(batchfile, fc, cc, lc, optlevel, tfflags, tcflags, tlflags, tsyslibs, - srcfiles, target, arch, objdir_temp, moddir_temp) + srcfiles, target, arch) # run the batch file proc = Popen([batchfile, ], stdout=PIPE, stderr=STDOUT) @@ -1091,18 +1087,16 @@ def compile_intel_win(srcfiles, target, fc, cc, objdir_temp, moddir_temp, print('Could not make x64 target: ', target) print(traceback.print_exc()) - # create makefile - if makefile: - msg = 'makefile not created for Windows with Intel Compilers.' - print(msg) - # return return 0 def makebatch(batchfile, fc, cc, lc, optlevel, fflags, cflags, lflags, syslibs, - srcfiles, target, arch, objdir_temp, moddir_temp): - """Make an ifort batch file for compiling on windows.""" + srcfiles, target, arch): + """ + Make an ifort batch file for compiling on windows. + + """ # get path to compilervars batch file iflist = ['IFORT_COMPILER{}'.format(i) for i in range(30, 12, -1)] found = False @@ -1177,35 +1171,44 @@ def makebatch(batchfile, fc, cc, lc, optlevel, fflags, cflags, lflags, syslibs, def create_makefile(target, srcdir, srcdir2, extrafiles, - srcfiles, objfiles, - fc, fflags, cc, cflags, syslibs, - objext='.o', modules=['-I', '-J']): + srcfiles, debug, double, + fc, cc, fflags, cflags, syslibs, + objext='.o', + makedefaults='makedefaults'): + # get list of unique fortran and c/c++ file extensions + fext = fortran_files(srcfiles, extensions=True) + cext = c_files(srcfiles, extensions=True) + + # build heading + heading = '# makefile created on {}\n'.format(datetime.datetime.now()) + \ + '# by pymake (version {})\n'.format(__version__) + heading += '# using the' + if fext is not None: + heading += " '{}' fortran".format(fc) + if cext is not None: + heading += ' and' + if cext is not None: + heading += " '{}' c/c++".format(cc) + heading += ' compiler(s).\n' + # open makefile f = open('makefile', 'w') - # write header for the make file - f.write('# makefile created on {}\n'.format(datetime.datetime.now()) + - '# by pymake (version {})\n'.format(__version__)) - f.write('# using the {} fortran and {} c/c++ compilers.\n'.format(fc, cc)) - f.write('\n') + # write header + f.write(heading + '\n') - # specify directory for the executable - f.write('# Define the directories for the object and module files,\n' + - '# the executable, and the executable name and path.\n') - opth = os.path.dirname(objfiles[0]).replace('\\', '/') - f.write('OBJDIR = {}\n'.format(opth)) - pth = os.path.dirname(target).replace('\\', '/') - if len(pth) < 1: - pth = '.' - f.write('BINDIR = {}\n'.format(pth)) - pth = target.replace('\\', '/') - f.write('PROGRAM = {}\n'.format(pth)) - f.write('\n') + # write include file + line = '\ninclude ./{}\n\n'.format(makedefaults) + f.write(line) + + # determine the directories with source files + # source files in sdir and sdir2 dirs = [d[0].replace('\\', '/') for d in os.walk(srcdir)] if srcdir2 is not None: dirs2 = [d[0].replace('\\', '/') for d in os.walk(srcdir2)] dirs = dirs + dirs2 - # add extrafiles + + # source files in extrafiles files = parse_extrafiles(extrafiles) if files is not None: for ef in files: @@ -1215,142 +1218,261 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, if rdir not in dirs: dirs.append(rdir) - srcdirs = [] + # write directories with source files and create vpath data + line = '# Define the source file directories\n' + f.write(line) + vpaths = [] for idx, dir in enumerate(dirs): - srcdirs.append('SOURCEDIR{}'.format(idx + 1)) - line = '{}={}\n'.format(srcdirs[idx], dir) + vpaths.append('SOURCEDIR{}'.format(idx + 1)) + line = '{}={}\n'.format(vpaths[idx], dir) f.write(line) f.write('\n') + + # write vpath f.write('VPATH = \\\n') - for idx, sd in enumerate(srcdirs): + for idx, sd in enumerate(vpaths): f.write('${' + '{}'.format(sd) + '} ') - if idx + 1 < len(srcdirs): + if idx + 1 < len(vpaths): f.write('\\') f.write('\n') f.write('\n') - odirs = [] - for idx, objfile in enumerate(srcfiles): - odir = os.path.dirname(objfile) - if odir not in odirs: - odirs.append(odir) - - ffiles = ['.f', '.f90', '.F90', '.fpp'] - cfiles = ['.c', '.cpp'] - - # for tf in ffiles + cfiles: - # line = 'SRCS =$(wildcard $(addsuffix / *{}, $(SUBDIRS))'.format(tf) - # f.write('{}\n'.format(line)) - # for tf in ffiles + cfiles: - # line = 'OBJS =$(${{SRCS:{}=.o}})'.format(tf) - # f.write('{}\n'.format(line)) - + # write file extensions line = '.SUFFIXES: ' - for tc in cfiles: - line += '{} '.format(tc) - for tf in ffiles: - line += '{} '.format(tf) + if fext is not None: + for ext in fext: + line += '{} '.format(ext) + if cext is not None: + for ext in cext: + line += '{} '.format(ext) line += objext f.write('{}\n'.format(line)) f.write('\n') - f.write('# Define the Fortran compile flags\n') - f.write('FC = {}\n'.format(fc)) - line = 'FFLAGS = ' - if '-MMD' not in fflags: - fflags += ['-MMD', '-cpp'] - for ff in fflags: - line += '{} '.format(ff) - f.write('{}\n'.format(line)) - f.write('\n') - - f.write('# Define the C compile flags\n') - f.write('CC = {}\n'.format(cc)) - line = 'CFLAGS = ' - if '-MMD' not in cflags: - cflags += ['-MMD'] - if cc not in ['g++']: - cflags += ['-cpp'] - for cf in cflags: - line += '{} '.format(cf) - f.write('{}\n'.format(line)) - f.write('\n') - - f.write('# Define the libraries\n') - line = 'SYSLIBS = ' - for sl in syslibs: - line += '{} '.format(sl) - f.write('{}\n'.format(line)) - f.write('\n') - f.write('OBJECTS = \\\n') - for idx, objfile in enumerate(objfiles): - f.write('$(OBJDIR)/{} '.format(os.path.basename(objfile))) - if idx + 1 < len(objfiles): + for idx, srcfile in enumerate(srcfiles): + objpth = os.path.splitext(os.path.basename(srcfile))[0] + objext + f.write('$(OBJDIR)/{} '.format(objpth)) + if idx + 1 < len(srcfiles): f.write('\\') f.write('\n') f.write('\n') - f.write('# Define task functions\n') - f.write('\n') - - f.write('# Create the bin directory and compile and link the program\n') - f.write('all: makebin | $(PROGRAM)\n') - f.write('\n') - - f.write('# Make the bin directory for the executable\n') - f.write('makebin :\n') - f.write('\tmkdir -p $(BINDIR)\n') - f.write('\n') - f.write('# Define the objects that make up the program\n') f.write('$(PROGRAM) : $(OBJECTS)\n') - line = '\t-$(FC) $(FFLAGS) -o $@ $(OBJECTS) $(SYSLIBS) ' - for m in modules: - line += '{}$(OBJDIR) '.format(m) + if fc is None: + line = '\t-$(CC) $(OPTLEVEL) $(CFLAGS) -o $@ $(OBJECTS)\n' + else: + line = '\t-$(FC) $(OPTLEVEL) $(FFLAGS) -o $@ $(OBJECTS) $(SYSLIBS)\n' f.write('{}\n'.format(line)) - f.write('\n') - for tf in ffiles: - f.write('$(OBJDIR)/%{} : %{}\n'.format(objext, tf)) - f.write('\t@mkdir -p $(@D)\n') - line = '\t$(FC) $(FFLAGS) -c $< -o $@ ' - for m in modules: - line += '{}$(OBJDIR) '.format(m) - f.write('{}\n'.format(line)) - f.write('\tcat {}/$*.d >> Dependencies\n\trm -f $*.d\n'.format(opth)) - f.write('\n') + if fext is not None: + for ext in fext: + f.write('$(OBJDIR)/%{} : %{}\n'.format(objext, ext)) + f.write('\t@mkdir -p $(@D)\n') + line = '\t$(FC) $(OPTLEVEL) $(FFLAGS) -c $< -o $@ ' + \ + '$(INCSWITCH) $(MODSWITCH)\n' + f.write('{}\n'.format(line)) + + if cext is not None: + for ext in cext: + f.write('$(OBJDIR)/%{} : %{}\n'.format(objext, ext)) + f.write('\t@mkdir -p $(@D)\n') + line = '\t$(CC) $(OPTLEVEL) $(CFLAGS) -c $< -o $@ ' + \ + '$(INCSWITCH)\n' + f.write('{}\n'.format(line)) + + # close the makefile + f.close() - for tc in cfiles: - f.write('$(OBJDIR)/%.o : %{}\n'.format(tc)) - f.write('\t@mkdir -p $(@D)\n') - line = '\t$(CC) $(CFLAGS) -c $< -o $@' - f.write('{}\n'.format(line)) - f.write('\tcat {}/$*.d >> Dependencies\n\trm -f $*.d\n'.format(opth)) - f.write('\n') + # open makedefaults + f = open(makedefaults, 'w') + + # replace makefile in heading with makedefaults + heading = heading.replace('makefile', makedefaults) + + # write header + f.write(heading + '\n') + + # write OS evaluation + line = '# determine OS\n' + line += 'ifeq ($(OS), Windows_NT)\n' + line += '\tdetected_OS = Windows\n' + line += '\tOS_macro = -D_WIN32\n' + line += 'else\n' + line += "\tdetected_OS = $(shell sh -c 'uname 2>/dev/null " + \ + "|| echo Unknown')\n" + line += '\tifeq ($(detected_OS), Darwin)\n' + line += '\t\tOS_macro = -D__APPLE__\n' + line += '\telse\n' + line += '\t\tOS_macro = -D__LINUX__\n' + line += '\tendif\n' + line += 'endif\n\n' + f.write(line) - f.write('# Clean the object and module files and the executable\n') - f.write('.PHONY : clean\n' + - 'clean : \n' + - '\t-rm -r Dependencies\n' + - '\t-rm -rf $(OBJDIR)\n' + - '\t-rm -rf $(PROGRAM)\n') - f.write('\n') + # get path to executable + dpth = os.path.dirname(target) + if len(dpth) > 0: + dpth = os.path.relpath(dpth) + else: + dpth = '.' + + # write + line = '# Define the directories for the object and module files\n' + \ + '# and the executable and its path.\n' + line += 'BINDIR = {}\n'.format(dpth) + line += 'OBJDIR = {}\n'.format(objdir_temp) + line += 'MODDIR = {}\n'.format(moddir_temp) + line += 'INCSWITCH = -I $(OBJDIR)\n' + line += 'MODSWITCH = -J $(MODDIR)\n\n' + f.write(line) - f.write('# Clean the object and module files\n') - f.write('.PHONY : cleanobj\n' + - 'cleanobj : \n' + - '\t-rm -rf $(OBJDIR)\n') - f.write('\n') + exe_name = os.path.splitext(os.path.basename(target))[0] + line = '# define os dependent executable name\n' + line += 'ifeq ($(detected_OS), Windows)\n' + line += '\tPROGRAM = {}.exe\n'.format(exe_name) + line += 'else\n' + line += '\tPROGRAM = {}\n'.format(exe_name) + line += 'endif\n\n' + f.write(line) - f.write('# Touch dependencies\n') - f.write('Dependencies : \n' + - '\ttouch Dependencies\n') - f.write('\n') + # set gfortran as compiler if it is f77 + line = '# set fortran compiler to gfortran if it is f77\n' + line += 'ifeq ($(FC), f77)\n' + line += '\tFC = gfortran\n' + line += '\t# set c compiler to gcc if not passed on the command line\n' + line += '\tifneq ($(origin CC), "command line")\n' + line += '\t\tifneq ($(CC), gcc)\n' + line += '\t\t\tCC = gcc\n' + line += '\t\tendif\n' + line += '\tendif\n' + line += 'endif\n\n' + f.write(line) - # close the make file + # optimization level + optlevel = get_optlevel(fc, cc, debug, fflags, cflags) + line = '# set the optimization level (OPTLEVEL) if not defined\n' + line += 'OPTLEVEL ?= {}\n\n'.format(optlevel) + f.write(line) + + # fortran flags + if fext is not None: + line = '# set the fortran flags\n' + line += 'ifeq ($(detected_OS), Windows)\n' + line += '\tifeq ($(FC), gfortran)\n' + tfflags = get_fortran_flags('gfortran', fflags, debug, double, + osname='win32') + line += '\t\tFFLAGS ?= {}\n'.format(' '.join(tfflags)) + line += '\tendif\n' + line += 'else\n' + line += '\tifeq ($(FC), gfortran)\n' + tfflags = get_fortran_flags('gfortran', fflags, debug, double, + osname='linux') + for idx, flag in enumerate(tfflags): + if '-D__' in flag: + tfflags[idx] = '$(OS_macro)' + line += '\t\tFFLAGS ?= {}\n'.format(' '.join(tfflags)) + line += '\tendif\n' + line += '\tifeq ($(FC), ifort mpiifort)\n' + tfflags = get_fortran_flags('ifort', fflags, debug, double, + osname='linux') + line += '\t\tFFLAGS ?= {}\n'.format(' '.join(tfflags)) + line += '\t\tMODSWITCH = -module $(MODDIR)\n' + line += '\tendif\n' + line += 'endif\n\n' + f.write(line) + + # c/c++ flags + if cext is not None: + line = '# set the c/c++ flags\n' + line += 'ifeq ($(detected_OS), Windows)\n' + line += '\tifeq ($(FC), gcc g++ clang clang++)\n' + tcflags = get_c_flags('gcc', fflags, debug, double, + osname='win32') + line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) + line += '\tendif\n' + line += 'else\n' + line += '\tifeq ($(FC), gcc g++ clang clang++)\n' + tcflags = get_c_flags('gcc', fflags, debug, double, + osname='linux') + line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) + line += '\tendif\n' + line += '\tifeq ($(FC), icc mpiicc icpc)\n' + tcflags = get_c_flags('icc', fflags, debug, double, + osname='linux') + line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) + line += '\tendif\n' + line += 'endif\n\n' + f.write(line) + + # syslibs + line = '# set the syslibs\n' + line += 'ifeq ($(detected_OS), Windows)\n' + line += '\tifeq ($(FC), gfortran gcc g++)\n' + tcomp, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='win32') + line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) + line += '\tendif\n' + line += 'else\n' + line += '\tifeq ($(FC), gfortran gcc g++ clang clang++)\n' + tcomp, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') + line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) + line += '\tendif\n' + line += '\tifeq ($(FC), ifort mpiifort icc icpc mpiicc)\n' + tcomp, tlink_flags, tsyslibs = get_linker_flags('ifort', 'icc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') + line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) + line += '\tendif\n' + line += 'endif\n\n' + f.write(line) + + # task functions + line = '# Define task functions\n' + line += '# Create the bin directory and compile and link the program\n' + line += 'all: makedirs | $(PROGRAM)\n\n' + line += '# Make the bin directory for the executable\n' + line += 'makedirs:\n' + line += '\tmkdir -p $(BINDIR)\n' + line += '\tmkdir -p $(MODDIR)\n\n' + line += '# Write selected compiler settings\n' + line += '.PHONY: settings\n' + line += 'settings:\n' + line += '\t@echo "Optimization level: $(OPTLEVEL)"\n' + if fext is not None: + line += '\t@echo "Fortran compiler: $(FC)"\n' + line += '\t@echo "Fortran flags: $(FFLAGS)"\n' + if cext is not None: + line += '\t@echo "C compiler: $(CC)"\n' + line += '\t@echo "C flags: $(CFLAGS)"\n' + line += '\t@echo "SYSLIBS: $(SYSLIBS)"\n\n' + line += '# Clean the object and module files and the executable\n' + line += '.PHONY: clean\n' + line += 'clean:\n' + line += '\t-rm -rf $(OBJDIR)\n' + line += '\t-rm -rf $(MODDIR)\n' + line += '\t-rm -rf $(PROGRAM)\n\n' + line += '# Clean the object and module files\n' + line += '.PHONY: cleanobj\n' + line += 'cleanobj:\n' + line += '\t-rm -rf $(OBJDIR)\n' + line += '\t-rm -rf $(MODDIR)\n\n' + f.write(line) + + # close the makedefaults f.close() + return + def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, expedite=False, dryrun=False, double=False, debug=False, @@ -1409,7 +1531,7 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, f.close() # get ordered list of files to compile - srcfiles = get_ordered_srcfiles(include_subdirs) + srcfiles = get_ordered_srcfiles(srcdir_temp, include_subdirs) # compile with gfortran or ifort intelwin = False @@ -1423,7 +1545,6 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, if intelwin: objext = '.obj' returncode = compile_intel_win(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, expedite, dryrun, double, debug, fflags, cflags, syslibs, arch, srcdir, srcdir2, @@ -1440,13 +1561,19 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, # compile the code returncode = compile_std(srcfiles, target, fc, cc, - objdir_temp, moddir_temp, expedite, dryrun, double, debug, fflags, cflags, syslibs, srcdir, srcdir2, extrafiles, makefile, sharedobject) + # create makefile + if makefile: + create_makefile(target, srcdir, srcdir2, extrafiles, srcfiles, + debug, double, + fc, cc, fflags, cflags, syslibs, + objext=objext) + # clean up temporary files if makeclean and returncode == 0: clean(objext, intelwin) From 1cb5e77e02f74d354e5a1859fd14887907b2adf4 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Thu, 18 Jun 2020 00:19:02 -0400 Subject: [PATCH 6/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to one method for compiling on Windows using Intel compilers and another method for everything else. Simplify call statements to remove temporary directories and use global variables for temporary directories. Refactor makefile created by pymake. --- autotest/t006_test.py | 15 +++++++------ autotest/t008_test.py | 18 ++++++++++------ pymake/pymake.py | 50 +++++++++++++++++++++++-------------------- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/autotest/t006_test.py b/autotest/t006_test.py index 12525e12..f3e1f243 100644 --- a/autotest/t006_test.py +++ b/autotest/t006_test.py @@ -31,6 +31,7 @@ def compile_code(): makeclean=False, makefile=True, exe_dir=dstpth, + dryrun=False, replace_function=replace_function) @@ -38,13 +39,13 @@ def build_with_makefile(): if os.path.isfile('makefile'): tepth = epth - if sys.platform == 'win32': + if sys.platform.lower() == 'win32': tepth += '.exe' # remove existing target - if os.path.isfile(tepth): + if os.path.isfile(target): print('Removing ' + target) - os.remove(tepth) + os.remove(target) print('Removing temporary build directories') dirs_temp = [os.path.join('src_temp'), @@ -57,8 +58,10 @@ def build_with_makefile(): # build MODFLOW-NWT with makefile print('build {} with makefile'.format(target)) os.system('make') - assert os.path.isfile(tepth), \ - '{} created by makefile does not exist.'.format(target) + + # verify that MODFLOW-NWT was made + errmsg = '{} created by makefile does not exist.'.format(target) + assert os.path.isfile(target), errmsg else: print('makefile does not exist...skipping build_with_make()') return @@ -82,7 +85,7 @@ def clean_up(): print('Removing folder ' + mfnwtpth) shutil.rmtree(mfnwtpth) - tepth = epth + tepth = target if sys.platform == 'win32': tepth += '.exe' diff --git a/autotest/t008_test.py b/autotest/t008_test.py index 5fc877c7..33cc7f7d 100644 --- a/autotest/t008_test.py +++ b/autotest/t008_test.py @@ -45,7 +45,7 @@ def build_with_makefile(): if os.path.isfile('makefile'): tepth = target - if sys.platform == 'win32': + if sys.platform.lower() == 'win32': tepth += '.exe' # remove existing target @@ -64,8 +64,11 @@ def build_with_makefile(): # build MODFLOW 6 with makefile print('build {} with makefile'.format(target)) os.system('make') - assert os.path.isfile(tepth), \ - '{} created by makefile does not exist.'.format(target) + + # verify that MODFLOW 6 was made + errmsg = '{} created by makefile does not exist.'.format(target) + assert os.path.isfile(tepth), errmsg + else: print('makefile does not exist...skipping build_with_make()') return @@ -88,12 +91,13 @@ def clean_up(): print('Removing folder ' + mf6pth) shutil.rmtree(mf6pth) - ext = '' + tepth = target if sys.platform == 'win32': - ext = '.exe' + tepth += '.exe' - print('Removing ' + target) - os.remove(target + ext) + if os.path.isfile(tepth): + print('Removing ' + target) + os.remove(tepth) return diff --git a/pymake/pymake.py b/pymake/pymake.py index 350e80df..e692e7e6 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -1387,18 +1387,18 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, line = '# set the c/c++ flags\n' line += 'ifeq ($(detected_OS), Windows)\n' line += '\tifeq ($(FC), gcc g++ clang clang++)\n' - tcflags = get_c_flags('gcc', fflags, debug, double, + tcflags = get_c_flags('gcc', fflags, debug, double, srcfiles, osname='win32') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' line += 'else\n' line += '\tifeq ($(FC), gcc g++ clang clang++)\n' - tcflags = get_c_flags('gcc', fflags, debug, double, + tcflags = get_c_flags('gcc', fflags, debug, double, srcfiles, osname='linux') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' line += '\tifeq ($(FC), icc mpiicc icpc)\n' - tcflags = get_c_flags('icc', fflags, debug, double, + tcflags = get_c_flags('icc', fflags, debug, double, srcfiles, osname='linux') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' @@ -1408,29 +1408,33 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, # syslibs line = '# set the syslibs\n' line += 'ifeq ($(detected_OS), Windows)\n' - line += '\tifeq ($(FC), gfortran gcc g++)\n' - tcomp, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='win32') - line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) - line += '\tendif\n' + _, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='win32') + line += '\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += 'else\n' - line += '\tifeq ($(FC), gfortran gcc g++ clang clang++)\n' - tcomp, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='linux') + if fc is None: + line += '\tifeq ($(CC), gcc g++ clang clang++)\n' + else: + line += '\tifeq ($(FC), gfortran)\n' + _, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += '\tendif\n' - line += '\tifeq ($(FC), ifort mpiifort icc icpc mpiicc)\n' - tcomp, tlink_flags, tsyslibs = get_linker_flags('ifort', 'icc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='linux') + if fc is None: + line += '\tifeq ($(CC), icc icpc mpiicc)\n' + else: + line += '\tifeq ($(FC), ifort mpiifort)\n' + _, tlink_flags, tsyslibs = get_linker_flags('ifort', 'icc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += '\tendif\n' line += 'endif\n\n' From a9ce8418133773c634c3c91fa931b89a557db02e Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Thu, 18 Jun 2020 14:00:44 -0400 Subject: [PATCH 7/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to a single compile method for Windows using Intel compilers and everything else. Simplify call statements to remove temporary directories and use global variables for temporary directories. Refactor makefile created by pymake. --- pymake/pymake.py | 437 +++++++++++++++++++++++++---------------------- 1 file changed, 230 insertions(+), 207 deletions(-) diff --git a/pymake/pymake.py b/pymake/pymake.py index e692e7e6..7d842e7a 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -119,6 +119,34 @@ def parser(): return args +def process_Popen_stdout(proc): + """ + Generic function to write Popen stdout data to the terminal. + + Parameters + ---------- + proc : Popen + Popen instance + + Returns + ------- + """ + # write stdout to the terminal + while True: + line = proc.stdout.readline() + c = line.decode('utf-8') + if c != '': + c = c.rstrip('\r\n') + print('{}'.format(c)) + else: + break + + # setup a communicator so that the Popen return code is set + proc.communicate() + + return + + def process_Popen_command(shellflg, cmdlist): """ Generic function to write Popen command data to the screen. @@ -135,26 +163,31 @@ def process_Popen_command(shellflg, cmdlist): ------- """ if not shellflg: - print(' '.join(cmdlist)) + if isinstance(cmdlist, str): + print(cmdlist) + elif isinstance(cmdlist, list): + print(' '.join(cmdlist)) return -def process_Popen_communicate(stdout, stderr): +def process_Popen_communicate(proc): """ Generic function to write communication information from Popen to the screen. Parameters ---------- - stdout : str - string with standard output from Popen - - stderr : str - string with standard error output from Popen + proc : Popen + Popen instance Returns ------- + returncode : int + proc.returncode + """ + stdout, stderr = proc.communicate() + if stdout: if PY3: stdout = stdout.decode() @@ -163,6 +196,13 @@ def process_Popen_communicate(stdout, stderr): if PY3: stderr = stderr.decode() print(stderr) + + # catch non-zero return code + if proc.returncode != 0: + msg = '{} failed\n'.format(' '.join(proc.args)) + \ + '\tstatus code {}\n'.format(proc.returncode) + print(msg) + return @@ -882,13 +922,20 @@ def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, return compiler, flags, syslibs_out -def compile_std(srcfiles, target, fc, cc, - expedite, dryrun, double, debug, fflags, cflags, syslibs, - srcdir, srcdir2, extrafiles, makefile, sharedobject): +def compile(srcfiles, target, fc, cc, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, arch, intelwin, + sharedobject): """ Standard compile method """ + # initialize returncode + returncode = 0 + + # initialize ilink + ilink = 0 + # set optimization levels optlevel = get_optlevel(fc, cc, debug, fflags, cflags) @@ -898,203 +945,189 @@ def compile_std(srcfiles, target, fc, cc, tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=sharedobject) - # build object files - print('\nCompiling object files for ' + - '{}...'.format(os.path.basename(target))) - objfiles = [] - - # assume that header files may be in other folders, so make a list - searchdir = [] - for f in srcfiles: - dirname = os.path.dirname(f) - if dirname not in searchdir: - searchdir.append(dirname) - - for srcfile in srcfiles: - cmdlist = [] - iscfile = False - ext = os.path.splitext(srcfile)[1].lower() - if ext in ['.c', '.cpp']: # mja - iscfile = True - cmdlist.append(cc) # mja - cmdlist.append(optlevel) - for switch in tcflags: # mja - cmdlist.append(switch) # mja - else: # mja - cmdlist.append(fc) - cmdlist.append(optlevel) - for switch in tfflags: - cmdlist.append(switch) - - # add search path for any c and c++ header files - if iscfile: - for sd in searchdir: - cmdlist.append('-I{}'.format(sd)) - # put object files and module files in objdir_temp and moddir_temp - else: - cmdlist.append('-I{}'.format(objdir_temp)) - if fc in ['ifort', 'mpiifort']: - cmdlist.append('-module') - cmdlist.append(moddir_temp + '/') - else: - cmdlist.append('-J{}'.format(moddir_temp)) - - cmdlist.append('-c') - cmdlist.append(srcfile) - - # object file name and location - srcname, srcext = os.path.splitext(srcfile) - srcname = srcname.split(os.path.sep)[-1] - objfile = os.path.join(objdir_temp, srcname + '.o') - cmdlist.append('-o') - cmdlist.append(objfile) - - # If expedited, then check if object file is out of date (if exists). - # No need to compile if object file is newer. - compilefile = True - if expedite: - if not out_of_date(srcfile, objfile): - compilefile = False - - # Compile the source code - if compilefile: - if not dryrun: - proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) - - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) - - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode - - # Save the name of the object file so that they can all be linked - # at the end - objfiles.append(objfile) - - # Build the link command and then link to create the executable - msg = '\nLinking object files ' + \ - 'to make {}...'.format(os.path.basename(target)) - print(msg) - - tcomp, tlink_flags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, - debug, double, - srcfiles, syslibs, - sharedobject=sharedobject) - cmdlist = [tcomp, optlevel] - for switch in tlink_flags: - cmdlist.append(switch) - - cmdlist.append('-o') - cmdlist.append(target) - for objfile in objfiles: - cmdlist.append(objfile) - - for switch in tsyslibs: - cmdlist.append(switch) + # get linker flags and syslibs + lc, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, + debug, double, + srcfiles, syslibs, + sharedobject=sharedobject) - if not dryrun: - proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) - process_Popen_command(False, cmdlist) + # clean exe prior to build so that test for exe below can return a + # non-zero error code + if flopy_avail: + if flopy_is_exe(target): + os.remove(target) - # establish communicator - stdout, stderr = proc.communicate() - process_Popen_communicate(stdout, stderr) + if intelwin: + # update compiler names if necessary + ext = '.exe' + if fc is not None: + if ext not in fc: + fc += ext + if cc is not None: + if ext not in cc: + cc += ext + if ext not in lc: + lc += ext + if ext not in target: + target += ext + + # update target extension if shared object + if sharedobject: + ttarget, ext = os.path.splitext(target) + if ext.lower() != '.dll': + target = ttarget + '.dll' + + # delete the batch file if it exists + batchfile = 'compile.bat' + if os.path.isfile(batchfile): + try: + os.remove(batchfile) + except: + print("could not remove '{}'".format(batchfile)) + + # Create target using a batch file on Windows + try: + create_win_batch(batchfile, fc, cc, lc, optlevel, + tfflags, tcflags, tlflags, tsyslibs, + srcfiles, target, arch) - # catch non-zero return code - if proc.returncode != 0: - msg = '{} failed, status code {}\n' \ - .format(' '.join(cmdlist), proc.returncode) - print(msg) - return proc.returncode + # build the command list for the Windows batch file + cmdlists = [batchfile, ] + except: + errmsg = 'Could not make x64 target: {}\n'.format(target) + errmsg += traceback.print_exc() + print(errmsg) - # return - return 0 - - -def compile_intel_win(srcfiles, target, fc, cc, - expedite, dryrun, double, debug, fflags, cflags, - syslibs, arch, srcdir, srcdir2, extrafiles, makefile): - """Make target on Windows OS.""" - - ext = '.exe' - if fc == 'ifort': - fc += ext - elif fc is not None: - fc += ext - if cc == 'icc': - cc += ext - elif cc == 'icl': - cc += ext else: - cc = 'cl' + ext + if sharedobject: + ext = os.path.splitext(target)[-1].lower() + if ext != '.so': + target += '.so' - # set optimization levels - optlevel = get_optlevel(fc, cc, debug, fflags, cflags) + # initialize the commands and object files list + cmdlists = [] + objfiles = [] + + # assume that header files may be in other folders, so make a list + searchdir = [] + for f in srcfiles: + dirname = os.path.dirname(f) + if dirname not in searchdir: + searchdir.append(dirname) + + # build the command for each source file and add to the + # list of commands + for srcfile in srcfiles: + cmdlist = [] + iscfile = False + ext = os.path.splitext(srcfile)[1].lower() + if ext in ['.c', '.cpp']: # mja + iscfile = True + cmdlist.append(cc) # mja + cmdlist.append(optlevel) + for switch in tcflags: # mja + cmdlist.append(switch) # mja + else: # mja + cmdlist.append(fc) + cmdlist.append(optlevel) + for switch in tfflags: + cmdlist.append(switch) + + # add search path for any c and c++ header files + if iscfile: + for sd in searchdir: + cmdlist.append('-I{}'.format(sd)) + # put object files and module files in objdir_temp and moddir_temp + else: + cmdlist.append('-I{}'.format(objdir_temp)) + if fc in ['ifort', 'mpiifort']: + cmdlist.append('-module') + cmdlist.append(moddir_temp + '/') + else: + cmdlist.append('-J{}'.format(moddir_temp)) + + cmdlist.append('-c') + cmdlist.append(srcfile) + + # object file name and location + srcname, srcext = os.path.splitext(srcfile) + srcname = srcname.split(os.path.sep)[-1] + objfile = os.path.join(objdir_temp, srcname + '.o') + cmdlist.append('-o') + cmdlist.append(objfile) + + # Save the name of the object file for linker + objfiles.append(objfile) + + # If expedited, then check if object file is out of date, if it + # exists. No need to compile if object file is newer. + compilefile = True + if expedite: + if not out_of_date(srcfile, objfile): + compilefile = False + + if compilefile: + cmdlists.append(cmdlist) + + # Build the link command and then link to create the executable + ilink = len(cmdlists) + if ilink > 0: + cmdlist = [lc, optlevel] + for switch in tlflags: + cmdlist.append(switch) - # get fortran and c compiler switches - tfflags = get_fortran_flags(fc, fflags, debug, double) - tcflags = get_c_flags(cc, cflags, debug, double, srcfiles) + cmdlist.append('-o') + cmdlist.append(target) + for objfile in objfiles: + cmdlist.append(objfile) - # get linker flags - lc, tlflags, tsyslibs = get_linker_flags(fc, cc, fflags, cflags, - debug, double, - srcfiles, syslibs) + for switch in tsyslibs: + cmdlist.append(switch) - # delete the batch file if it exists - batchfile = 'compile.bat' - if os.path.isfile(batchfile): - try: - os.remove(batchfile) - except: - pass + # add linker + cmdlists.append(cmdlist) - # Create target using a batch file on Windows - try: - # clean exe prior to build so that test for exe below can return a - # non-zero error code - if flopy_avail: - if flopy_is_exe(target): - os.remove(target) - - makebatch(batchfile, fc, cc, lc, optlevel, - tfflags, tcflags, tlflags, tsyslibs, - srcfiles, target, arch) - - # run the batch file - proc = Popen([batchfile, ], stdout=PIPE, stderr=STDOUT) - while True: - line = proc.stdout.readline() - c = line.decode('utf-8') - if c != '': - c = c.rstrip('\r\n') - print('{}'.format(c)) + # execute each command in cmdlists + if not dryrun: + for idx, cmdlist in enumerate(cmdlists): + if idx == 0: + if intelwin: + msg = "\nCompiling '{}' ".format(os.path.basename(target)) + \ + 'for Windows using Intel compilers...' + else: + msg = "\nCompiling object files for " + \ + "'{}'...".format(os.path.basename(target)) + print(msg) + if idx > 0 and idx == ilink: + msg = "\nLinking object files " + \ + "to make '{}'...".format(os.path.basename(target)) + print(msg) + + # write the command to the terminal + process_Popen_command(False, cmdlist) + + # run the command using Popen + proc = Popen(cmdlist, shell=False, stdout=PIPE, stderr=PIPE) + + # write batch file execution to terminal + if intelwin: + process_Popen_stdout(proc) + # establish communicator to report errors else: - break + process_Popen_communicate(proc) - # evaluate if the executable is available - if flopy_avail: - if not flopy_is_exe(target): - return 1 - else: - return 0 - except: - print('Could not make x64 target: ', target) - print(traceback.print_exc()) + # set return code + returncode = proc.returncode # return - return 0 + return returncode -def makebatch(batchfile, fc, cc, lc, optlevel, fflags, cflags, lflags, syslibs, - srcfiles, target, arch): +def create_win_batch(batchfile, fc, cc, lc, optlevel, + fflags, cflags, lflags, syslibs, + srcfiles, target, arch): """ - Make an ifort batch file for compiling on windows. + Make an intel compiler batch file for compiling on windows. """ # get path to compilervars batch file @@ -1109,7 +1142,7 @@ def makebatch(batchfile, fc, cc, lc, optlevel, fflags, cflags, lflags, syslibs, raise Exception('Pymake could not find IFORT compiler.') cpvars += os.path.join('bin', 'compilervars.bat') if not os.path.isfile(cpvars): - raise Exception('Could not find cpvars: {0}'.format(cpvars)) + raise Exception('Could not find cpvars: {}'.format(cpvars)) f = open(batchfile, 'w') line = 'call ' + '"' + os.path.normpath(cpvars) + '" ' + arch + '\n' f.write(line) @@ -1546,30 +1579,20 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, if cc is not None: if cc in ['cl', 'icl']: intelwin = True + if intelwin: objext = '.obj' - returncode = compile_intel_win(srcfiles, target, fc, cc, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, arch, - srcdir, srcdir2, - extrafiles, makefile) else: objext = '.o' - if sharedobject: - ext = os.path.splitext(target)[-1].lower() - if ext != '.so': - target += '.so' # update openspec files create_openspec() - # compile the code - returncode = compile_std(srcfiles, target, fc, cc, - expedite, dryrun, - double, debug, - fflags, cflags, syslibs, - srcdir, srcdir2, - extrafiles, makefile, sharedobject) + # compile the executable + returncode = compile(srcfiles, target, fc, cc, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, arch, intelwin, + sharedobject) # create makefile if makefile: From d50d22d1dd5e7e94c605847aa42d9e68a14e0666 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Thu, 18 Jun 2020 17:07:18 -0400 Subject: [PATCH 8/8] refactor(pymake): refactor makefile generation Add generic functions for creating OPTLEVEL, FFLAGS, CFLAGS, and SYSLIBS. Simplify to a single compile method for Windows using Intel compilers and everything else. Simplify call statements to remove temporary directories and use global variables for temporary directories. Refactor makefile created by pymake. --- autotest/t011_test.py | 29 +++ pymake/build_program.py | 36 ++- pymake/pymake.py | 548 ++++++++++++++++++++++++++++++---------- 3 files changed, 475 insertions(+), 138 deletions(-) diff --git a/autotest/t011_test.py b/autotest/t011_test.py index 67a724ed..6a3742ac 100644 --- a/autotest/t011_test.py +++ b/autotest/t011_test.py @@ -1,5 +1,6 @@ from __future__ import print_function import os +import sys import shutil import json @@ -107,6 +108,33 @@ def test_usgsprograms_list_json(): return +def test_make(): + # add --makefile to command line list + sys.argv.append('--makefile') + + # get current directory and change to working directory + cwd = os.getcwd() + os.chdir(cpth) + + # build triangle and makefile + success = True + try: + pymake.build_apps('triangle') + except: + success = False + + # change to starting directory + os.chdir(cwd) + + # remove --makefile from command line list + sys.argv.remove('--makefile') + + # evaluate success + assert success, 'could not build triangle' + + return + + def test_clean_up(): shutil.rmtree(cpth) @@ -120,4 +148,5 @@ def test_clean_up(): test_usgsprograms_load_json() test_usgsprograms_list_json_error() test_usgsprograms_list_json() + test_make() test_clean_up() diff --git a/pymake/build_program.py b/pymake/build_program.py index 9232a67b..b95fd186 100644 --- a/pymake/build_program.py +++ b/pymake/build_program.py @@ -593,6 +593,33 @@ def set_zip(): return zip_pth +def set_makefile(): + """ + Set boolean for whether a makefile should be created + + Parameters + ---------- + + Returns + ------- + makefile : bool + boolean indicating if a makefile should be created + + """ + makefile = False + for idx, arg in enumerate(sys.argv): + if arg.lower() == '--makefile': + makefile = True + break + + # write c/c++ flags + if makefile: + msg = 'a GNU make makefile will be created:\n' + print(msg) + + return makefile + + def build_program(target='mf2005', fc='gfortran', cc='gcc', makeclean=True, expedite=False, dryrun=False, double=False, debug=False, include_subdirs=False, @@ -1023,10 +1050,16 @@ def build_apps(targets=None): download = download_now clean = download_clean + # set makefile, only done for first target and precision + makefile = False + if idt == 0 and idx == 0: + makefile = set_makefile() + # print download information msg = 'downloading file: {}\n'.format(download) msg += 'verified download: {}\n'.format(download_verify) msg += 'download timeout: {} sec.\n'.format(timeout) + msg += 'create GNU make makefile: {}\n'.format(makefile) msg += 'cleaning extracted files: {}\n'.format(clean) print(msg) @@ -1049,7 +1082,8 @@ def build_apps(targets=None): download_dir=download_dir, download_clean=clean, download_verify=download_verify, - timeout=timeout) + timeout=timeout, + makefile=makefile) # calculate download and compile time end_downcomp = datetime.now() diff --git a/pymake/pymake.py b/pymake/pymake.py index 7d842e7a..01352640 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -19,7 +19,7 @@ import sys import traceback import shutil -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE import argparse import datetime @@ -27,7 +27,6 @@ try: from flopy import is_exe as flopy_is_exe - flopy_avail = True except: flopy_avail = False @@ -41,7 +40,18 @@ def parser(): - """Construct the parser and return argument values.""" + """ + Construct the parser and return argument values. + + Parameters + ---------- + + Returns + ------- + args : list + command line argument list + + """ description = __description__ parser = argparse.ArgumentParser(description=description, epilog='''Note that the source directory @@ -95,9 +105,6 @@ def parser(): parser.add_argument('-mf', '--makefile', help='''Create a standard makefile.''', action='store_true') - parser.add_argument('-cm', '--cmake', - help='''File with DAG sorted source files for CMAKE.''', - default=None) parser.add_argument('-cs', '--commonsrc', help='''Additional directory with common source files.''', default=None) @@ -130,6 +137,8 @@ def process_Popen_stdout(proc): Returns ------- + None + """ # write stdout to the terminal while True: @@ -161,6 +170,8 @@ def process_Popen_command(shellflg, cmdlist): Returns ------- + None + """ if not shellflg: if isinstance(cmdlist, str): @@ -182,8 +193,7 @@ def process_Popen_communicate(proc): Returns ------- - returncode : int - proc.returncode + None """ stdout, stderr = proc.communicate() @@ -206,12 +216,30 @@ def process_Popen_communicate(proc): return -def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): +def pymake_initialize(srcdir, target, commonsrc, extrafiles, excludefiles): """ Remove temp source directory and target, and then copy source into source temp directory. - Return temp directory path. + Parameters + ---------- + srcdir : str + path for directory containing source files + target : str + path for executable to create + commonsrc : str + additional directory with common source files. + extrafiles : str + path for extrafiles file that contains paths to additional source + files to include + excludefiles : str + path for excludefiles file that contains filename of source files + to exclude from the build + + Returns + ------- + None + """ # remove the target if it already exists try: @@ -236,7 +264,7 @@ def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): # if extrafiles is not none, then it is a text file with a list of # additional source files that need to be copied into srctemp and # compiled. - files = parse_extrafiles(extrafiles) + files = get_extrafiles(extrafiles) if files is None: files = [] for fname in files: @@ -253,7 +281,7 @@ def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): # if exclude is not None, then it is a text file with a list of # source files that need to be excluded from srctemp. - files = parse_extrafiles(excludefiles) + files = get_extrafiles(excludefiles) if files is None: files = [] for fname in files: @@ -284,7 +312,22 @@ def initialize(srcdir, target, commonsrc, extrafiles, excludefiles): return -def parse_extrafiles(extrafiles): +def get_extrafiles(extrafiles): + """ + Get + + Parameters + ---------- + extrafiles : str + path for extrafiles file that contains paths to additional source + files to include + + Returns + ------- + files : list + list of files in the extra files input file + + """ if extrafiles is None: files = None else: @@ -308,7 +351,20 @@ def parse_extrafiles(extrafiles): def clean(objext, intelwin): """ - Remove mod and object files, and remove the temp source directory. + Cleanup intermediate files. Remove mod and object files, and remove + the temporary source directory. + + Parameters + ---------- + objext : str + object file extension + intelwin : bool + boolean indicating if pymake was used to compile source code on + Windows using Intel compilers + + Returns + ------- + None """ # clean things up @@ -331,15 +387,25 @@ def clean(objext, intelwin): def get_ordered_srcfiles(srcdir, include_subdir=False): """ - Create a list of ordered source files (both fortran and c). + Create a list of ordered source files (both fortran and c). Ordering + is build using a directed acyclic graph to determine module dependencies. + + Parameters + ---------- + srcdir : str + path for directory containing source files + include_subdir : bool + flag indicating if source files are in subdirectories in srcdir + + Returns + ------- + orderedsourcefiles : list + list of ordered source files - Ordering is build using a directed acyclic graph to determine module - dependencies. """ # create a list of all c(pp), f and f90 source files - templist = [] - for path, subdirs, files in os.walk(srcdir): + for path, _, files in os.walk(srcdir): for name in files: if not include_subdir: if path != srcdir: @@ -355,19 +421,16 @@ def get_ordered_srcfiles(srcdir, include_subdir=False): elif f.lower().endswith('.c') or f.lower().endswith('.cpp'): # mja cfiles.append(f) # mja - # orderedsourcefiles = order_source_files(srcfiles) + \ - # order_c_source_files(cfiles) - srcfileswithpath = [] for srcfile in srcfiles: - s = os.path.join(srcdir, srcfile) + # s = os.path.join(srcdir, srcfile) s = srcfile srcfileswithpath.append(s) # from mja cfileswithpath = [] for srcfile in cfiles: - s = os.path.join(srcdir, srcfile) + # s = os.path.join(srcdir, srcfile) s = srcfile cfileswithpath.append(s) @@ -385,9 +448,17 @@ def get_ordered_srcfiles(srcdir, include_subdir=False): def create_openspec(): """ Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses - STREAM ACCESS. + STREAM ACCESS. This is specific to MODFLOW and MT3D based targets. + Source directories are scanned and files defining file access are + replaced. + + Parameters + ---------- + + Returns + ------- + None - This is specific to MODFLOW and MT3D based targets. """ files = ['openspec.inc', 'filespec.inc'] dirs = [d[0] for d in os.walk(srcdir_temp)] @@ -408,19 +479,49 @@ def create_openspec(): return -def out_of_date(srcfile, objfile): - ood = True +def check_out_of_date(srcfile, objfile): + """ + Check if existing object files are current with the existing source files. + + Parameters + ---------- + srcfile : str + source file path + objfile : str + object file path + + Returns + ------- + stale : bool + boolean indicating if the object file is current + + """ + stale = True if os.path.exists(objfile): t1 = os.path.getmtime(objfile) t2 = os.path.getmtime(srcfile) if t1 > t2: - ood = False - return ood + stale = False + return stale -# determine if iso_c_binding is used so that correct -# gcc and clang compiler flags can be set def get_iso_c(srcfiles): + """ + Determine if iso_c_binding is used so that the correct c/c++ + compiler flags can be set. All fortran files are scanned + + Parameters + ---------- + srcfiles : list + list of fortran source files + + Returns + ------- + iso_c : bool + flag indicating if iso_c_binding is used in any fortran file + + """ + iso_c = False for srcfile in srcfiles: try: f = open(srcfile, 'rb') @@ -439,35 +540,44 @@ def get_iso_c(srcfiles): if linelist[0].upper() == 'USE': modulename = linelist[1].split(',')[0].upper() if 'ISO_C_BINDING' == modulename: - return True - return False - + iso_c = True + break + # terminate is iso_c is True + if iso_c: + break + return iso_c -def flag_available(flag): - """ - Determine if a specified flag exists. - Not all flags will be detected, for example -O2 -fbounds-check=on +def check_switch_available(switch): """ + Determine if a specified gfortran switch exists. Not all switches will be + detected, for example '-O2' adn '-fbounds-check=on' - # set shell flag based on os - shellflg = False - if sys.platform == 'win32': - shellflg = True + Parameters + ---------- + switch : str + compiler switch + Returns + ------- + avail : bool + boolean indicating if the compiler switch is available + """ # determine the gfortran command line flags available cmdlist = ['gfortran', '--help', '-v'] - proc = Popen(cmdlist, stdout=PIPE, stderr=PIPE, shell=shellflg) - process_Popen_command(shellflg, cmdlist) + proc = Popen(cmdlist, stdout=PIPE, stderr=PIPE, shell=False) + process_Popen_command(False, cmdlist) # establish communicator stdout, stderr = proc.communicate() - if PY3: stdout = stdout.decode() - avail = flag in stdout - msg = ' {} flag available: {}'.format(flag, avail) + # determine if flag exists + avail = switch in stdout + + # write a message + msg = ' {} switch available: {}'.format(switch, avail) print(msg) return avail @@ -514,14 +624,14 @@ def get_prepend(compiler, osname): return prepend -def fortran_files(srcfiles, extensions=False): +def get_fortran_files(srcfiles, extensions=False): """ Return a list of fortran files or unique fortran file extensions. Parameters ------- srcfiles : list - list of sourcefile names + list of source file names extensions : bool flag controls return of either a list of fortran files or a list of unique fortran file extensions @@ -546,14 +656,14 @@ def fortran_files(srcfiles, extensions=False): return l -def c_files(srcfiles, extensions=False): +def get_c_files(srcfiles, extensions=False): """ Return a list of c and cpp files or unique c and cpp file extensions. Parameters ------- srcfiles : list - list of sourcefile names + list of source file names extensions : bool flag controls return of either a list of c and cpp files or a list of unique c and cpp file extensions @@ -644,11 +754,10 @@ def get_optlevel(fc, cc, debug, fflags, cflags, osname=None): return optlevel -def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, - osname=None): +def get_fortran_flags(fc, fflags, debug, double, + sharedobject=False, osname=None): """ - Return a list of standard pymake and user specified fortran compiler - switches. + Return a list of pymake and user specified fortran compiler switches. Parameters ------- @@ -657,11 +766,12 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, fflags : list user provided list of fortran compiler flags debug : bool - flag indicating a debug executable will be built + boolean indicating a debug executable will be built double : bool - flag indicating a double precision executable will be built + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. sharedobject : bool - flag indicating a shared object (.so or .dll) will be built + boolean indicating a shared object (.so or .dll) will be built osname : str optional lower case OS name. If not passed it will be determined using sys.platform @@ -695,12 +805,12 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, flags.append('Bstatic') if debug: flags += ['g', 'fcheck=all', 'fbounds-check', 'Wall'] - if flag_available('-ffpe-trap'): + if check_switch_available('-ffpe-trap'): flags.append('ffpe-trap=overflow,zero,invalid,denormal') else: - if flag_available('-ffpe-summary'): + if check_switch_available('-ffpe-summary'): flags.append('ffpe-summary=overflow') - if flag_available('-ffpe-trap'): + if check_switch_available('-ffpe-trap'): flags.append('ffpe-trap=overflow,zero,invalid') if double: flags += ['fdefault-real-8', 'fdefault-double-8'] @@ -747,11 +857,10 @@ def get_fortran_flags(fc, fflags, debug, double, sharedobject=False, return flags -def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, - osname=None): +def get_c_flags(cc, cflags, debug, srcfiles, + sharedobject=False, osname=None): """ - Return a list of standard pymake and user specified c or cpp compiler - switches. + Return a list of standard and user specified c/c++ compiler switches. Parameters ------- @@ -761,12 +870,10 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, user provided list of c or cpp compiler flags debug : bool flag indicating a debug executable will be built - double : bool - flag indicating a double precision executable will be built srcfiles : list - list of sourcefile names + list of source file names sharedobject : bool - flag indicating a shared object (.so or .dll) will be built + boolean indicating a shared object (.so or .dll) will be built osname : str optional lower case OS name. If not passed it will be determined using sys.platform @@ -799,7 +906,7 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, flags.append('Bstatic') if debug: flags += ['g'] - if flag_available('-Wall'): + if check_switch_available('-Wall'): flags.append('Wall') else: pass @@ -823,8 +930,8 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, # code that is linked to C/C++ code. Only needed if there are # any fortran files. -D_UF defines UNIX naming conventions for # mixed language compilation. - ffiles = fortran_files(srcfiles) - cfiles = c_files(srcfiles) + ffiles = get_fortran_files(srcfiles) + cfiles = get_c_files(srcfiles) if ffiles is not None: use_iso_c = get_iso_c(ffiles) if not use_iso_c and cfiles is not None: @@ -847,31 +954,41 @@ def get_c_flags(cc, cflags, debug, double, srcfiles, sharedobject=False, def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, syslibs, sharedobject=False, osname=None): """ - Return a list of standard pymake and user specified c or cpp compiler - switches. + Return a list of pymake and user specified linker switches, including + the linker compiler and syslibs. Parameters ------- + fc : str + fortran compiler cc : str c or cpp compiler + fflags : list + user provided list of fortran compiler flags cflags : list user provided list of c or cpp compiler flags debug : bool flag indicating a debug executable will be built double : bool - flag indicating a double precision executable will be built + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. srcfiles : list - list of sourcefile names + list of source file names sharedobject : bool - flag indicating a shared object (.so or .dll) will be built + boolean indicating a shared object (.so or .dll) will be built osname : str optional lower case OS name. If not passed it will be determined using sys.platform Returns ------- - flags : str - c or cpp compiler switches + compiler : str + linker compiler + flags : list + list of linker switches + syslibs : list + list of syslibs for the linker + """ compiler = fc if compiler is None: @@ -894,7 +1011,7 @@ def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, elif compiler in ['gcc', 'g++', 'clang', 'clang++', 'icc', 'icpc', 'icl', 'cl', 'mpiicc', 'mpiicpc']: - flags = get_c_flags(compiler, cflags, debug, double, srcfiles, + flags = get_c_flags(compiler, cflags, debug, srcfiles, sharedobject=sharedobject, osname=osname) if sharedobject: @@ -922,13 +1039,51 @@ def get_linker_flags(fc, cc, fflags, cflags, debug, double, srcfiles, return compiler, flags, syslibs_out -def compile(srcfiles, target, fc, cc, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, arch, intelwin, - sharedobject): +def pymake_compile(srcfiles, target, fc, cc, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, arch, intelwin, + sharedobject): """ Standard compile method + Parameters + ------- + srcfiles : list + list of source file names + target : str + path for executable to create + fc : str + fortran compiler + cc : str + c or cpp compiler + expedite : bool + boolean indicating if only out of date source files will be compiled. + Clean must not have been used on previous build. + dryrun : bool + boolean indicating if source files should be compiled. Files will be + deleted, if makeclean is True. + double : bool + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. + debug : bool + boolean indicating is a debug executable will be built + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + syslibs : list + user provided syslibs + arch : str + architecture to use for Intel Compilers on Windows (default is intel64) + intelwin : bool + boolean indicating if pymake was used to compile source code on + Windows using Intel compilers + + Returns + ------- + returncode : int + returncode + """ # initialize returncode returncode = 0 @@ -942,7 +1097,7 @@ def compile(srcfiles, target, fc, cc, # get fortran and c compiler switches tfflags = get_fortran_flags(fc, fflags, debug, double, sharedobject=sharedobject) - tcflags = get_c_flags(cc, cflags, debug, double, srcfiles, + tcflags = get_c_flags(cc, cflags, debug, srcfiles, sharedobject=sharedobject) # get linker flags and syslibs @@ -1063,7 +1218,7 @@ def compile(srcfiles, target, fc, cc, # exists. No need to compile if object file is newer. compilefile = True if expedite: - if not out_of_date(srcfile, objfile): + if not check_out_of_date(srcfile, objfile): compilefile = False if compilefile: @@ -1092,7 +1247,8 @@ def compile(srcfiles, target, fc, cc, for idx, cmdlist in enumerate(cmdlists): if idx == 0: if intelwin: - msg = "\nCompiling '{}' ".format(os.path.basename(target)) + \ + msg = "\nCompiling '{}' ".format( + os.path.basename(target)) + \ 'for Windows using Intel compilers...' else: msg = "\nCompiling object files for " + \ @@ -1129,6 +1285,37 @@ def create_win_batch(batchfile, fc, cc, lc, optlevel, """ Make an intel compiler batch file for compiling on windows. + Parameters + ------- + batchfile : str + batch file name to create + fc : str + fortran compiler + cc : str + c or cpp compiler + lc : str + compiler to use for linking + optlevel : str + compiler optimization switch + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + lflags : list + linker compiler flags, which are a combination of user provided list + of compiler flags for thw compiler to used for linking + syslibs : list + user provided syslibs + srcfiles : list + list of source file names + target : str + path for executable to create + arch : str + architecture to use for Intel Compilers on Windows (default is intel64) + + Returns + ------- + """ # get path to compilervars batch file iflist = ['IFORT_COMPILER{}'.format(i) for i in range(30, 12, -1)] @@ -1143,7 +1330,11 @@ def create_win_batch(batchfile, fc, cc, lc, optlevel, cpvars += os.path.join('bin', 'compilervars.bat') if not os.path.isfile(cpvars): raise Exception('Could not find cpvars: {}'.format(cpvars)) + + # open the batch file f = open(batchfile, 'w') + + # write the compilervars batch command to batchfile line = 'call ' + '"' + os.path.normpath(cpvars) + '" ' + arch + '\n' f.write(line) @@ -1208,13 +1399,57 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, fc, cc, fflags, cflags, syslibs, objext='.o', makedefaults='makedefaults'): + """ + + Parameters + ---------- + target : str + path for executable to create + srcdir : str + path for directory containing source files + srcdir2 : str + additional directory with common source files. + extrafiles : str + path for extrafiles file that contains paths to additional source + files to include + srcfiles : list + ordered list of source files to include in the makefile + debug : bool + boolean indicating is a debug executable will be built + double : bool + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. + fc : str + fortran compiler + cc : str + c or cpp compiler + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + syslibs : list + user provided syslibs + objext : str + object file extension + makedefaults : str + name of the makedefaults file to create with makefile (default is + makedefaults) + + Returns + ------- + + """ # get list of unique fortran and c/c++ file extensions - fext = fortran_files(srcfiles, extensions=True) - cext = c_files(srcfiles, extensions=True) + fext = get_fortran_files(srcfiles, extensions=True) + cext = get_c_files(srcfiles, extensions=True) + + # set exe_name + exe_name = os.path.splitext(os.path.basename(target))[0] # build heading - heading = '# makefile created on {}\n'.format(datetime.datetime.now()) + \ - '# by pymake (version {})\n'.format(__version__) + heading = "# makefile created on {}\n".format(datetime.datetime.now()) + \ + "# by pymake (version {}) ".format(__version__) + \ + "for the '{}' executable \n".format(exe_name) heading += '# using the' if fext is not None: heading += " '{}' fortran".format(fc) @@ -1242,7 +1477,7 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, dirs = dirs + dirs2 # source files in extrafiles - files = parse_extrafiles(extrafiles) + files = get_extrafiles(extrafiles) if files is not None: for ef in files: fdir = os.path.dirname(ef) @@ -1360,7 +1595,6 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, line += 'MODSWITCH = -J $(MODDIR)\n\n' f.write(line) - exe_name = os.path.splitext(os.path.basename(target))[0] line = '# define os dependent executable name\n' line += 'ifeq ($(detected_OS), Windows)\n' line += '\tPROGRAM = {}.exe\n'.format(exe_name) @@ -1420,18 +1654,18 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, line = '# set the c/c++ flags\n' line += 'ifeq ($(detected_OS), Windows)\n' line += '\tifeq ($(FC), gcc g++ clang clang++)\n' - tcflags = get_c_flags('gcc', fflags, debug, double, srcfiles, + tcflags = get_c_flags('gcc', fflags, debug, srcfiles, osname='win32') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' line += 'else\n' line += '\tifeq ($(FC), gcc g++ clang clang++)\n' - tcflags = get_c_flags('gcc', fflags, debug, double, srcfiles, + tcflags = get_c_flags('gcc', fflags, debug, srcfiles, osname='linux') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' line += '\tifeq ($(FC), icc mpiicc icpc)\n' - tcflags = get_c_flags('icc', fflags, debug, double, srcfiles, + tcflags = get_c_flags('icc', fflags, debug, srcfiles, osname='linux') line += '\t\tCFLAGS ?= {}\n'.format(' '.join(tcflags)) line += '\tendif\n' @@ -1441,33 +1675,33 @@ def create_makefile(target, srcdir, srcdir2, extrafiles, # syslibs line = '# set the syslibs\n' line += 'ifeq ($(detected_OS), Windows)\n' - _, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='win32') + _, _, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='win32') line += '\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += 'else\n' if fc is None: line += '\tifeq ($(CC), gcc g++ clang clang++)\n' else: line += '\tifeq ($(FC), gfortran)\n' - _, tlink_flags, tsyslibs = get_linker_flags('gfortran', 'gcc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='linux') + _, _, tsyslibs = get_linker_flags('gfortran', 'gcc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += '\tendif\n' if fc is None: line += '\tifeq ($(CC), icc icpc mpiicc)\n' else: line += '\tifeq ($(FC), ifort mpiifort)\n' - _, tlink_flags, tsyslibs = get_linker_flags('ifort', 'icc', - fflags, cflags, - debug, double, - srcfiles, syslibs, - osname='linux') + _, _, tsyslibs = get_linker_flags('ifort', 'icc', + fflags, cflags, + debug, double, + srcfiles, syslibs, + osname='linux') line += '\t\tSYSLIBS ?= {}\n'.format(' '.join(tsyslibs)) line += '\tendif\n' line += 'endif\n\n' @@ -1515,8 +1749,65 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, expedite=False, dryrun=False, double=False, debug=False, include_subdirs=False, fflags=None, cflags=None, syslibs=None, arch='intel64', makefile=False, srcdir2=None, extrafiles=None, - excludefiles=None, cmake=None, sharedobject=False): - """Main part of program.""" + excludefiles=None, sharedobject=False): + """ + Main pymake function. + + Parameters + ---------- + srcdir : str + path for directory containing source files + target : str + path for executable to create + fc : str + fortran compiler + cc : str + c or cpp compiler + makeclean : bool + boolean indicating if intermediate files should be cleaned up + after successful build + expedite : bool + boolean indicating if only out of date source files will be compiled. + Clean must not have been used on previous build. + dryrun : bool + boolean indicating if source files should be compiled. Files will be + deleted, if makeclean is True. + double : bool + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. + debug : bool + boolean indicating is a debug executable will be built + include_subdirs : bool + boolean indicating source files in srcdir subdirectories should be + included in the build + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + syslibs : list + user provided syslibs + arch : str + Architecture to use for Intel Compilers on Windows (default is intel64) + makefile : bool + boolean indicating if a GNU make makefile should be created + srcdir2 : str + additional directory with common source files. + extrafiles : str + path for extrafiles file that contains paths to additional source + files to include + excludefiles : str + path for excludefiles file that contains filename of source files + to exclude from the build + sharedobject : bool + boolean indicating a shared object (.so or .dll) will be built + + + Returns + ------- + returncode : int + return code + + """ # initialize return code returncode = 0 @@ -1549,23 +1840,7 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, os.makedirs(pth) # initialize - initialize(srcdir, target, srcdir2, extrafiles, excludefiles) - - if cmake is not None: - if excludefiles is not None: - efiles = [os.path.basename(fpth) for fpth in - parse_extrafiles(excludefiles)] - else: - efiles = [] - csrcfiles = get_ordered_srcfiles(srcdir, include_subdirs) - f = open(cmake, 'w') - fstart = os.path.dirname(cmake) - for fpth in csrcfiles: - fname = os.path.basename(fpth) - if fname not in efiles: - fpthr = os.path.relpath(fpth, fstart) - f.write('{}\n'.format(os.path.join(fpthr))) - f.close() + pymake_initialize(srcdir, target, srcdir2, extrafiles, excludefiles) # get ordered list of files to compile srcfiles = get_ordered_srcfiles(srcdir_temp, include_subdirs) @@ -1589,10 +1864,10 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, create_openspec() # compile the executable - returncode = compile(srcfiles, target, fc, cc, - expedite, dryrun, double, debug, - fflags, cflags, syslibs, arch, intelwin, - sharedobject) + returncode = pymake_compile(srcfiles, target, fc, cc, + expedite, dryrun, double, debug, + fflags, cflags, syslibs, arch, intelwin, + sharedobject) # create makefile if makefile: @@ -1619,5 +1894,4 @@ def main(srcdir, target, fc='gfortran', cc='gcc', makeclean=True, dryrun=args.dryrun, double=args.double, debug=args.debug, include_subdirs=args.subdirs, fflags=args.fflags, cflags=args.cflags, arch=args.arch, makefile=args.makefile, - srcdir2=args.commonsrc, extrafiles=args.extrafiles, - cmake=args.cmake) + srcdir2=args.commonsrc, extrafiles=args.extrafiles)