diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60b9b42a..43825470 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,35 @@ on: pull_request: jobs: + + pymake_lint: + name: pymake linting + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v1 + - name: Install packages + run: | + pip install requests pydotplus appdirs numpy matplotlib + pip install https://github.com/modflowpy/flopy/zipball/develop + pip install pylint flake8 black + - name: Run black + run: black --check --line-length 79 ./pymake + - name: Run flake8 + run: flake8 --count --show-source ./pymake + - name: Run pylint + run: pylint --jobs=0 --errors-only ./pymake + pymakeCI: + name: Run pymake CI on different python versions and different OSs + needs: pymake_lint runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -62,32 +90,6 @@ jobs: g++ --version - name: Run nosetests + shell: bash run: | nosetests -v --with-id --with-timer -w ./autotest - env: - repo-token: ${{secrets.GITHUB_TOKEN}} - - - pymake_lint: - runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@v1 - - name: Install packages - run: | - pip install requests pydotplus appdirs numpy matplotlib - pip install https://github.com/modflowpy/flopy/zipball/develop - pip install pylint flake8 black - - name: Run black - run: black --check --line-length 79 ./pymake - - name: Run flake8 - run: flake8 --count --show-source ./pymake - - name: Run pylint - run: pylint --jobs=0 --errors-only ./pymake \ No newline at end of file diff --git a/README.md b/README.md index ec67668b..eba145d6 100644 --- a/README.md +++ b/README.md @@ -2,28 +2,32 @@ Python package for compiling MODFLOW-based programs. -### Version 1.1 +### Version 1.2 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/ff198bf587524161ad2bc60b3ab15979)](https://app.codacy.com/manual/jdhughes-usgs/pymake?utm_source=github.com&utm_medium=referral&utm_content=modflowpy/pymake&utm_campaign=Badge_Grade_Settings) [![Build Status](https://travis-ci.org/modflowpy/pymake.svg?branch=master)](https://travis-ci.org/modflowpy/pymake) [![Coverage Status](https://coveralls.io/repos/github/modflowpy/pymake/badge.svg?branch=master)](https://coveralls.io/github/modflowpy/pymake?branch=master) -This is a relatively simple python package for compiling MODFLOW-based programs. -The package determines the build order using a directed acyclic graph and then -compiles the source files using gfortran or intel Fortran (ifort). +This is a python package for compiling MODFLOW-based and other Fortran, C, and +C++ programs. The package determines the build order using a directed acyclic +graph and then compiles the source files using GNU compilers (`gcc`, `g++`, +`gfortran`) or Intel compilers (`ifort`, `icc`). pymake can be run from the command line or it can be called from within python. +By default, pymake sets the optimization level, Fortran flags, C/C++ flags, and +linker flags that are consistent with those used to compile MODFLOW-based +programs released by the USGS. -pymake includes example scripts for building MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, -MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, MODPATH 6, MODPATH 7, MT3DMS, MT3D-USGS, -and SEAWAT using gfortran on Mac or Linux. The scripts download the distribution -file from the USGS and compile the source into a binary executable. +pymake includes example scripts for building MODFLOW 6, MODFLOW-2005, +MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, MODPATH 6, MODPATH 7, +GSFLOW, VS2DT, MT3DMS, MT3D-USGS, SEAWAT, and SUTRA. Example scripts for +creating the utility programs CRT, Triangle, and GRIDGEN are also included. +The scripts download the distribution file from the USGS (and other +organizations) and compile the source into a binary executable. -pymake includes code for compiling with ifort on Windows and OSX. - -Note that if gfortran is used, the `openspec.f` and `FILESPEC.inc` (MT3DMS) -file will automatically be changed to the following so that binary files are -created properly using standard Fortran: +Note that if gfortran is used to compile MODFLOW-based codes, the `openspec.f` +and `FILESPEC.inc` (MT3DMS) files will automatically be changed to the +following so that binary files are created properly using standard Fortran: ``` c -- created by pymake.py @@ -36,88 +40,202 @@ c -- end of include file ## Command Line Usage +pymake can be used to compile MODFLOW 6 directly from the command line using +the Intel Fortran compiler `ifort` from a subdirectory at the same level as +the `src` subdirectory by specifying: + +``` +python -m pymake ../src/ ../bin/mf6 -mc --subdirs -fc ifort +``` + To see help for running from command line, use the following statement. -```python -m pymake.pymake -h``` +``` +python -m pymake -h +``` -usage: ```pymake.py [-h] [-fc {ifort,gfortran}] [-cc {gcc,clang}] -[-ar {ia32,ia32_intel64,intel64}] [-mc] [-dbl] [-dbg] [-e] -[-dr] [-sd] [-ff] -srcdir target``` +The help message identifies required positional arguments and optional +arguments that can be provided to overide default values. -This is the pymake program for compiling fortran source files, such as the -source files that come with MODFLOW. The program works by building a directed -acyclic graph of the module dependencies and then compiling the source files -in the proper order. +``` +usage: __main__.py [-h] [-fc {ifort,mpiifort,gfortran,none}] + [-cc {gcc,clang,clang++,icc,icl,mpiicc,g++,cl,none}] + [-ar {ia32,ia32_intel64,intel64}] [-mc] [-dbl] [-dbg] [-e] + [-dr] [-sd] [-ff FFLAGS] [-cf CFLAGS] [-sl {-lc,-lm}] [-mf] + [-cs COMMONSRC] [-ef EXTRAFILES] [-exf EXCLUDEFILES] [-so] + [-ad APPDIR] [-v] [--keep] [--zip ZIP] [--inplace] + srcdir target + +This is the pymake program for compiling fortran, c, and c++ source files, +such as the source files that come with MODFLOW. The program works by building +a directed acyclic graph of the module dependencies and then compiling the +source files in the proper order. positional arguments: -srcdir Location of source directory -target Name of target to create + srcdir Path source directory. + target Name of target to create. (can include path) optional arguments: --h, --help show this help message and exit --fc {ifort,gfortran} Fortran compiler to use (default is gfortran) --cc {gcc,clang} C compiler to use (default is gcc) --ar {ia32,ia32_intel64,intel64} -Architecture to use for ifort (default is intel64) --mc, --makeclean Clean files when done --dbl, --double Force double precision --dbg, --debug Create debug version --e, --expedite Only compile out of date source files. Clean must not -have been used on previous build. Does not work yet -for ifort. --dr, --dryrun Do not actually compile. Files will be deleted, if ---makeclean is used. Does not work yet for ifort. --sd, --subdirs Include source files in srcdir subdirectories. --ff, --fflags Additional fortran compiler flags. --mf, --makefile Create a standard makefile. Does not work for -ifort for Windows yet. + -h, --help show this help message and exit + -fc {ifort,mpiifort,gfortran,none} + Fortran compiler to use. (default is gfortran) + -cc {gcc,clang,clang++,icc,icl,mpiicc,g++,cl,none} + C/C++ compiler to use. (default is gcc) + -ar {ia32,ia32_intel64,intel64}, --arch {ia32,ia32_intel64,intel64} + Architecture to use for Intel and Microsoft compilers + on Windows. (default is intel64) + -mc, --makeclean Clean temporary object, module, and source files when + done. (default is False) + -dbl, --double Force double precision. (default is False) + -dbg, --debug Create debug version. (default is False) + -e, --expedite Only compile out of date source files. Clean must not + have been used on previous build. (default is False) + -dr, --dryrun Do not actually compile. Files will be deleted, if + --makeclean is used. Does not work yet for ifort. + (default is False) + -sd, --subdirs Include source files in srcdir subdirectories. + (default is None) + -ff FFLAGS, --fflags FFLAGS + Additional Fortran compiler flags. Fortran compiler + flags should be enclosed in quotes and start with a + blank space or separated from the name (-ff or + --fflags) with a equal sign (-ff='-O3'). (default is + None) + -cf CFLAGS, --cflags CFLAGS + Additional C/C++ compiler flags. C/C++ compiler flags + should be enclosed in quotes and start with a blank + space or separated from the name (-cf or --cflags) + with a equal sign (-cf='-O3'). (default is None) + -sl {-lc,-lm}, --syslibs {-lc,-lm} + Linker system libraries. Linker libraries should be + enclosed in quotes and start with a blank space or + separated from the name (-sl or --syslibs) with a + equal sign (-sl='-libgcc'). (default is None) + -mf, --makefile Create a GNU make makefile. (default is False) + -cs COMMONSRC, --commonsrc COMMONSRC + Additional directory with common source files. + (default is None) + -ef EXTRAFILES, --extrafiles EXTRAFILES + List of extra source files to include in the + compilation. extrafiles can be either a list of files + or the name of a text file that contains a list of + files. (default is None) + -exf EXCLUDEFILES, --excludefiles EXCLUDEFILES + List of extra source files to exclude from the + compilation. excludefiles can be either a list of + files or the name of a text file that contains a list + of files. (default is None) + -so, --sharedobject Create shared object or dll on Windows. (default is + False) + -ad APPDIR, --appdir APPDIR + Target path that overides path defined target path + (default is None) + -v, --verbose Verbose output to terminal. (default is False) + --keep Keep existing executable. (default is False) + --zip ZIP Zip built executable. (default is False) + --inplace Source files in srcdir are used directly. (default is + False) Note that the source directory should not contain any bad or duplicate source -files as all source files in the source directory will be built and linked. +files as all source files in the source directory, the common source file +directory (srcdir2), and the extra files (extrafiles) will be built and +linked. Files can be excluded by using the excludefiles command line switch. +``` + +Note that command line arguments for Fortran flags, C/C++ flags, and syslib +libraries should be enclosed in quotes and start with a space prior to the +first value (`-ff ' -O3'`) or use an equal sign separating the command line +argument and the values (`-ff='-O3'`). The command line argument to use an +`-O3` optimization level when compiling MODFLOW 6 with the `ifort` compiler +would be: +``` +python -m pymake ../src/ ../bin/mf6 -mc --subdirs -fc ifort -ff='-O3' +``` + ## From Python -### Script to compile mfnwt +### Script to compile MODFLOW 6 + +When using the pymake object (`Pymake()`) only the positional arguments +(`srcdir`, `target`) need to be specified in the script. +```python import pymake -srcdir = '../mfnwt/src' -target = 'mfnwt' -pymake.main(srcdir, target, 'gfortran', 'gcc', makeclean=True, expedite=False, -dryrun=False, double=False, debug=False, include_subdirs=False) +pm = pymake.Pymake() +pm.srcdir = '../mfnwt/src' +pm.target = 'mf6' +pm.include_subdirs = True +pm.build() +``` + +It is suggested that optional variables required for successful compiling and +linking be manually specified in the script to mininimize the potential for +unsuccessful builds. For MODFLOW 6, subdirectories in the `src` subdirectory +need to be included and '`pm.include_subdirs = True`' has been specified in +the script. Custom optimization levels and compiler flags could be specified +to get consistent builds. -*or see make_mfnwt.py in examples directory* +Non-default values for the optional arguments can specified as command line +arguments. For example, MODFLOW 6 could be compiled using Intel compilers +instead of the default GNU compilers with the script listed above by +specifying: +``` +python mymf6script.py -fc ifort -cc icc +``` ## Automatic Download and Build The following scripts can be run directly from the command line to build -MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, -MODFLOW-2000, MODPATH 6, MODPATH 7, MT3DMS, MT3D-USGS, and SEAWAT -binaries on Mac and Linux. The scripts will download the distribution -file from the USGS (requires internet connection), unzip the file, and -compile the source. MT3DMS will be downloaded from the University of -Alabama. - -python make_modflow6.py -python make_mf2005.py -python make_mfnwt.py -python make_mfusg.py -python make_mflgr.py -python make_mf2000.py -python make_modpath6.py -python make_modpath7.py -python make_mt3d.py -python make_mt3dusgs -python make_swtv4.py +MODFLOW 6, MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, MODFLOW-LGR, MODFLOW-2000, +MODPATH 6, MODPATH 7, MT3DMS, MT3D-USGS, and SEAWAT binaries on Linux, Mac, +and Windows. The scripts will download the distribution file from the USGS +(requires internet connection), unzip the file, and compile the source. +MT3DMS will be downloaded from the University of Alabama and Triangle will be +downloaded from +[netlib.org](http://www.netlib.org/voronoi/triangle.zip). The scripts use the +`pymake.build_apps()` method which download and unzip the distribution files +and set all of the pymake settings required to build the program. Available +example scripts include: + +* make_modflow6.py +* make_mf2005.py +* make_mfnwt.py +* make_mfusg.py +* make_mflgr.py +* make_mf2000.py +* make_modpath6.py +* make_modpath7.py +* make_gsflow.py +* make_vs2dt.py +* make_mt3d.py +* make_mt3dusgs.py +* make_swtv4.py +* make_crt.py +* make_gridgen.py +* make_triangle.py + +Optional command line arguments can be used to customize the build (`-fc`, +`-cc`, `--fflags`, etc.). MODFLOW 6 could be built using intel compilers and +an `O3` optimation level by specifying: + +``` +python make_mf6.py -fc=ifort --fflags='-O3' +``` + ## Installation To install pymake directly from the git repository type: +``` pip install https://github.com/modflowpy/pymake/zipball/master +``` To update your version of pymake with the latest from the git repository type: +``` pip install https://github.com/modflowpy/pymake/zipball/master --upgrade +``` diff --git a/autotest/t001_test.py b/autotest/t001_test.py index 95adb4d5..77677a75 100644 --- a/autotest/t001_test.py +++ b/autotest/t001_test.py @@ -150,8 +150,8 @@ def cleanup(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(epth): print("Removing " + target) diff --git a/autotest/t002_test.py b/autotest/t002_test.py index bd41c012..c7b8ab30 100644 --- a/autotest/t002_test.py +++ b/autotest/t002_test.py @@ -72,8 +72,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(epth): print("Removing " + target) diff --git a/autotest/t003_test.py b/autotest/t003_test.py index 84fc7663..808b0560 100644 --- a/autotest/t003_test.py +++ b/autotest/t003_test.py @@ -79,8 +79,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(epth): print("Removing " + target) diff --git a/autotest/t004_test.py b/autotest/t004_test.py index a131e2dd..499360aa 100644 --- a/autotest/t004_test.py +++ b/autotest/t004_test.py @@ -93,8 +93,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(epth): print("Removing " + target) diff --git a/autotest/t006_test.py b/autotest/t006_test.py index f0b8caed..014c86bf 100644 --- a/autotest/t006_test.py +++ b/autotest/t006_test.py @@ -25,6 +25,7 @@ pm.target = target pm.appdir = dstpth pm.makefile = True +pm.inplace = True def download_src(): @@ -37,6 +38,7 @@ def download_src(): def build_with_makefile(): + success = True if os.path.isfile("makefile"): # remove existing target if os.path.isfile(epth): @@ -61,7 +63,7 @@ def build_with_makefile(): errmsg = "{} created by makefile does not exist.".format(target) success = os.path.isfile(epth) else: - errmsg = "makefile does not exist...skipping build_with_make()" + errmsg = "makefile does not exist" assert success, errmsg @@ -82,8 +84,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() # clean up MODFLOW-NWT if os.path.isfile(epth): diff --git a/autotest/t007_test.py b/autotest/t007_test.py index 2df2c7d1..9c5c738f 100644 --- a/autotest/t007_test.py +++ b/autotest/t007_test.py @@ -211,8 +211,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() # clean up compiled executables for epth in epths: diff --git a/autotest/t008_test.py b/autotest/t008_test.py index c5166fd7..6ac7fc79 100644 --- a/autotest/t008_test.py +++ b/autotest/t008_test.py @@ -28,6 +28,7 @@ pm.appdir = dstpth pm.dryrun = False pm.makefile = True +pm.inplace = True def get_example_dirs(): @@ -108,8 +109,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(epth): print("Removing " + target) @@ -154,10 +155,7 @@ def test_compile(): def test_mf6(): - # get name files and simulation name - example_dirs = get_example_dirs() - # run models - for ws in example_dirs: + for ws in get_example_dirs(): yield run_mf6, ws @@ -165,6 +163,18 @@ def test_makefile(): build_with_makefile() +def test_sharedobject(): + pm.target = "libmf6" + prog_dict = pymake.usgs_program_data.get_target(pm.target) + pm.srcdir = os.path.join(mf6pth, prog_dict.srcdir) + pm.srcdir2 = os.path.join(mf6pth, "src") + pm.excludefiles = [os.path.join(pm.srcdir2, "mf6.f90")] + pm.makefile = False + pm.sharedobject = True + pm.inplace = False + pm.build() + + def test_clean_up(): clean_up() @@ -174,5 +184,6 @@ def test_clean_up(): test_compile() for ws in get_example_dirs(): run_mf6(ws) - build_with_makefile() + test_makefile() + test_sharedobject() test_clean_up() diff --git a/autotest/t009_test.py b/autotest/t009_test.py index 168584a1..9674591e 100644 --- a/autotest/t009_test.py +++ b/autotest/t009_test.py @@ -153,8 +153,8 @@ def clean_up(): if os.path.isdir(d): shutil.rmtree(d) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() for epth in epths: if os.path.isfile(epth): diff --git a/autotest/t010_test.py b/autotest/t010_test.py index d25b47ad..d5cbf467 100644 --- a/autotest/t010_test.py +++ b/autotest/t010_test.py @@ -27,6 +27,7 @@ pm.appdir = dstpth pm.cc = "g++" pm.fc = None +pm.inplace = True def download_src(): @@ -57,8 +58,8 @@ def clean_up(): if os.path.isdir(pth): shutil.rmtree(pth) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() if os.path.isfile(exe_name): print("Removing " + target) diff --git a/autotest/t012_test.py b/autotest/t012_test.py index c8c0a03e..36fdc538 100644 --- a/autotest/t012_test.py +++ b/autotest/t012_test.py @@ -113,8 +113,8 @@ def clean_up(): print("Removing example folder " + example) shutil.rmtree(pth) - # remove download directory - pm.download_cleanup() + # finalize pymake object + pm.finalize() # clean up compiled executables if os.path.isfile(egsflow): diff --git a/examples/buildall.py b/examples/buildall.py index 37fba0bd..d0862e0b 100644 --- a/examples/buildall.py +++ b/examples/buildall.py @@ -3,7 +3,7 @@ try: import pymake except: - print('pymake is not installed...will not build executables') + print("pymake is not installed...will not build executables") pymake = None @@ -13,8 +13,8 @@ def build_all(): pymake.usgs_program_data.export_json(current=True) # build all of the applications - pymake.build_apps() + pymake.build_apps(release_precision=False) -if __name__ == '__main__': +if __name__ == "__main__": build_all() diff --git a/examples/make_crt.py b/examples/make_crt.py index fa266512..c65b6060 100644 --- a/examples/make_crt.py +++ b/examples/make_crt.py @@ -3,7 +3,7 @@ # Download and compile the CRT distribution def make_app(): - pymake.build_apps(['crt']) + pymake.build_apps(["crt"]) if __name__ == "__main__": diff --git a/examples/make_gridgen.py b/examples/make_gridgen.py index 67e3a418..71004227 100644 --- a/examples/make_gridgen.py +++ b/examples/make_gridgen.py @@ -2,7 +2,7 @@ def test_build_gridgen(): - pymake.build_apps('gridgen') + pymake.build_apps("gridgen") return diff --git a/examples/make_gsflow.py b/examples/make_gsflow.py index e353691b..8a1f268a 100644 --- a/examples/make_gsflow.py +++ b/examples/make_gsflow.py @@ -3,7 +3,7 @@ # Download and compile the GSFLOW distribution def make_app(): - pymake.build_apps(['gsflow']) + pymake.build_apps(["gsflow"]) if __name__ == "__main__": diff --git a/examples/make_mf2000.py b/examples/make_mf2000.py index 524c6947..75fb398a 100644 --- a/examples/make_mf2000.py +++ b/examples/make_mf2000.py @@ -2,7 +2,7 @@ def make_mf2000(): - pymake.build_apps('mf2000') + pymake.build_apps("mf2000") if __name__ == "__main__": diff --git a/examples/make_mf2005.py b/examples/make_mf2005.py index 88ac85c2..e1b5cf97 100644 --- a/examples/make_mf2005.py +++ b/examples/make_mf2005.py @@ -2,7 +2,7 @@ def make_mf2005(): - pymake.build_apps('mf2005') + pymake.build_apps("mf2005") if __name__ == "__main__": diff --git a/examples/make_mf6.py b/examples/make_mf6.py index 552f3673..bf699e92 100644 --- a/examples/make_mf6.py +++ b/examples/make_mf6.py @@ -3,7 +3,7 @@ # Download and compile the MODFLOW 6 distribution def make_mf6(): - build_apps(["mf6", "zbud6"]) + build_apps(("mf6",)) return diff --git a/examples/make_mf6beta.py b/examples/make_mf6beta.py index b194c620..2dad97e0 100644 --- a/examples/make_mf6beta.py +++ b/examples/make_mf6beta.py @@ -3,7 +3,7 @@ # Download and compile the MODFLOW 6 distribution def make_mf6beta(): - pymake.build_apps('mf6beta') + pymake.build_apps("mf6beta") if __name__ == "__main__": diff --git a/examples/make_mflgr.py b/examples/make_mflgr.py index 2154916b..9e7bc46e 100644 --- a/examples/make_mflgr.py +++ b/examples/make_mflgr.py @@ -2,7 +2,7 @@ def make_mflgr(): - pymake.build_apps('mflgr') + pymake.build_apps("mflgr") if __name__ == "__main__": diff --git a/examples/make_mfnwt.py b/examples/make_mfnwt.py index a3d28efa..ab462a70 100644 --- a/examples/make_mfnwt.py +++ b/examples/make_mfnwt.py @@ -2,7 +2,7 @@ def make_mfnwt(): - pymake.build_apps('mfnwt') + pymake.build_apps("mfnwt") if __name__ == "__main__": diff --git a/examples/make_mfusg.py b/examples/make_mfusg.py index 5c0ee145..1de26319 100644 --- a/examples/make_mfusg.py +++ b/examples/make_mfusg.py @@ -2,7 +2,7 @@ def make_mfusg(): - pymake.build_apps(['mfusg', 'zonbudusg']) + pymake.build_apps(["mfusg"]) if __name__ == "__main__": diff --git a/examples/make_mp6.py b/examples/make_mp6.py index 713672fa..281f87f1 100644 --- a/examples/make_mp6.py +++ b/examples/make_mp6.py @@ -2,7 +2,7 @@ def make_mp6(): - pymake.build_apps('mp6') + pymake.build_apps("mp6") if __name__ == "__main__": diff --git a/examples/make_mp7.py b/examples/make_mp7.py index 0e525fcd..ba43937e 100644 --- a/examples/make_mp7.py +++ b/examples/make_mp7.py @@ -2,7 +2,7 @@ def make_mp7(): - pymake.build_apps('mp7') + pymake.build_apps("mp7") if __name__ == "__main__": diff --git a/examples/make_mt3dms.py b/examples/make_mt3dms.py index 84d2eca8..06d81e88 100644 --- a/examples/make_mt3dms.py +++ b/examples/make_mt3dms.py @@ -2,7 +2,7 @@ def make_mt3dms(): - pymake.build_apps('mt3dms') + pymake.build_apps("mt3dms") if __name__ == "__main__": diff --git a/examples/make_mt3dusgs.py b/examples/make_mt3dusgs.py index abbc2d9b..0235a2bd 100644 --- a/examples/make_mt3dusgs.py +++ b/examples/make_mt3dusgs.py @@ -2,7 +2,7 @@ def make_mt3dusgs(): - pymake.build_apps('mt3dusgs') + pymake.build_apps("mt3dusgs") if __name__ == "__main__": diff --git a/examples/make_swtv4.py b/examples/make_swtv4.py index 073f3b9c..17c25879 100644 --- a/examples/make_swtv4.py +++ b/examples/make_swtv4.py @@ -2,7 +2,7 @@ def make_swtv4(): - pymake.build_apps('swtv4') + pymake.build_apps("swtv4") return diff --git a/examples/make_triangle.py b/examples/make_triangle.py index 8012fffb..dd688d38 100644 --- a/examples/make_triangle.py +++ b/examples/make_triangle.py @@ -2,7 +2,7 @@ def make_triangle(): - pymake.build_apps('triangle') + pymake.build_apps("triangle") if __name__ == "__main__": diff --git a/examples/make_vs2dt.py b/examples/make_vs2dt.py index da3bba13..87702a48 100644 --- a/examples/make_vs2dt.py +++ b/examples/make_vs2dt.py @@ -2,7 +2,7 @@ def make_vs2dt(): - pymake.build_apps('vs2dt') + pymake.build_apps("vs2dt") if __name__ == "__main__": diff --git a/pymake/__init__.py b/pymake/__init__.py index 9b69d992..30a4b9c6 100644 --- a/pymake/__init__.py +++ b/pymake/__init__.py @@ -1,8 +1,8 @@ # __init__.py from .pymake import Pymake, __version__ from .usgsprograms import usgs_program_data -from .pymake_base import main, parser -from .compiler_switches import set_compiler +from .pymake_base import main +from .pymake_parser import parser from .build_apps import build_apps from .download import ( download_and_unzip, diff --git a/pymake/__main__.py b/pymake/__main__.py new file mode 100644 index 00000000..f24ff201 --- /dev/null +++ b/pymake/__main__.py @@ -0,0 +1,32 @@ +# __main__.py +from .pymake_base import main +from .pymake_parser import parser + +# get the arguments +args = parser() + +# call main -- note that this form allows main to be called +# from python as a function. +main( + args.srcdir, + args.target, + fc=args.fc, + cc=args.cc, + makeclean=args.makeclean, + expedite=args.expedite, + dryrun=args.dryrun, + double=args.double, + debug=args.debug, + include_subdirs=args.subdirs, + fflags=args.fflags, + cflags=args.cflags, + arch=args.arch, + makefile=args.makefile, + srcdir2=args.commonsrc, + extrafiles=args.extrafiles, + excludefiles=args.excludefiles, + sharedobject=args.sharedobject, + appdir=args.appdir, + verbose=args.verbose, + inplace=args.inplace, +) diff --git a/pymake/build_apps.py b/pymake/build_apps.py index 59595c5e..b3dc0ba5 100644 --- a/pymake/build_apps.py +++ b/pymake/build_apps.py @@ -12,6 +12,7 @@ def build_apps( download_dir=None, appdir=None, verbose=None, + release_precision=True, ): """Build all of the current targets or a subset of targets. @@ -26,6 +27,12 @@ def build_apps( download directory path appdir : str target path + release_precision : bool + boolean indicating if only the release precision version should be + build. If release_precision is False, then the release precision + version will be compiled along with a double precision version of + the program for programs where the standard_switch and double_switch + in usgsprograms.txt is True. default is True. Returns ------- @@ -53,6 +60,9 @@ def build_apps( ) raise TypeError(msg) + # set object to clean after each build + pmobj.makeclean = True + # reset variables based on passed args if download_dir is not None: pmobj.download_dir = download_dir @@ -88,12 +98,22 @@ def build_apps( pmobj.cc = "clang++" elif target in ("triangle",): pmobj.fc = "none" - elif target in ("mf6",): + elif target in ("mf6", "libmf6"): pmobj.cc = "none" else: pmobj.fc = fc0 pmobj.cc = cc0 + # set sharedobject + if target in ("libmf6",): + pmobj.sharedobject = True + else: + pmobj.sharedobject = False + + # reset srcdir2 - TODO make more robust + if target not in ("libmf6",): + pmobj.srcdir2 = None + # reset extrafiles for instances with more than one target if idt > 0: pmobj.extrafiles = None @@ -106,31 +126,9 @@ def build_apps( update_target_name = True # set download information - download_now = True - download_clean = True download_dir = "temp" - # modify download if mf6 and also building zonbud6 - if target == "mf6": - if idt + 1 < len(targets): - if targets[idt + 1] == "zbud6": - download_clean = False - elif target == "zbud6": - if idt > 0: - if targets[idt - 1] == "mf6": - download_now = False - - # modify download if mfusg and also building zonbudusg - if target == "mfusg": - if idt + 1 < len(targets): - if targets[idt + 1] == "zonbudusg": - download_clean = False - elif target == "zonbudusg": - if idt > 0: - if targets[idt - 1] == "mfusg": - download_now = False - - if target in ["mt3dms", "triangle", "mf6beta"]: + if target in ("mt3dms", "triangle"): download_verify = False timeout = 10 else: @@ -146,7 +144,12 @@ def build_apps( # determine if single, double, or both should be built precision = usgs_program_data.get_precision(target) - for idx, double in enumerate(precision): + # just build the first precision in precision list if + # standard_precision is True + if release_precision: + precision = precision[0:1] + + for double in precision: # set double flag pmobj.double = double @@ -157,38 +160,18 @@ def build_apps( ) ) - # set download boolean - if idx == 0: - download = download_now - else: - download = False - - # set clean boolean - if len(precision) > 1: - if idx < len(precision) - 1: - clean = False - else: - clean = download_clean - else: - clean = download_clean - # setup download for target - if download: - pmobj.download_setup( - target, - download_path=download_dir, - verify=download_verify, - timeout=timeout, - ) + pmobj.download_setup( + target, + download_path=download_dir, + verify=download_verify, + timeout=timeout, + ) # build the code if build_target: pmobj.build(modify_exe_name=update_target_name) - # clean up the download - if clean: - pmobj.download_cleanup() - # calculate download and compile time end_downcomp = datetime.now() elapsed = end_downcomp - start_downcomp @@ -207,4 +190,7 @@ def build_apps( if pmobj.returncode == 0: pmobj.compress_targets() + # execute final Pymake object operations + pmobj.finalize() + return pmobj.returncode diff --git a/pymake/compiler_language_files.py b/pymake/compiler_language_files.py index 06d32938..00f7d72a 100644 --- a/pymake/compiler_language_files.py +++ b/pymake/compiler_language_files.py @@ -116,64 +116,80 @@ def get_iso_c(srcfiles): return iso_c -def get_ordered_srcfiles(srcdir, include_subdir=False): - """Create a list of ordered source files (both fortran and c). Ordering is - build using a directed acyclic graph to determine module dependencies. +def get_srcfiles(srcdir, include_subdir): + """Get a list of source files in source file directory srcdir Parameters ---------- srcdir : str path for directory containing source files - include_subdir : bool - flag indicating if source files are in subdirectories in srcdir + include_subdirs : bool + boolean indicating source files in srcdir subdirectories should be + included in the build Returns ------- - orderedsourcefiles : list - list of ordered source files + srcfiles : list + list of fortran and c/c++ file in srcdir """ # create a list of all c(pp), f and f90 source files templist = [] for path, _, files in os.walk(srcdir): - for name in files: + for file in files: if not include_subdir: if path != srcdir: continue - f = os.path.join(os.path.join(path, name)) - templist.append(f) - cfiles = [] # mja + file = os.path.join(os.path.join(path, file)) + templist.append(file) srcfiles = [] - for f in templist: + for file in templist: if ( - f.lower().endswith(".f") - or f.lower().endswith(".f90") - or f.lower().endswith(".for") - or f.lower().endswith(".fpp") + file.lower().endswith(".f") + or file.lower().endswith(".f90") + or file.lower().endswith(".for") + or file.lower().endswith(".fpp") + or file.lower().endswith(".c") + or file.lower().endswith(".cpp") ): - srcfiles.append(f) - elif f.lower().endswith(".c") or f.lower().endswith(".cpp"): # mja - cfiles.append(f) # mja + srcfiles.append(os.path.relpath(file, os.getcwd())) + return srcfiles - srcfileswithpath = [] - for srcfile in srcfiles: - # s = os.path.join(srcdir, srcfile) - s = srcfile - srcfileswithpath.append(s) - # from mja - cfileswithpath = [] - for srcfile in cfiles: - # s = os.path.join(srcdir, srcfile) - s = srcfile - cfileswithpath.append(s) +def get_ordered_srcfiles(all_srcfiles): + """Create a list of ordered source files (both fortran and c). Ordering is + build using a directed acyclic graph to determine module dependencies. + + Parameters + ---------- + all_srcfiles : list + list of all fortran and c/c++ source files + + Returns + ------- + ordered_srcfiles : list + list of ordered source files + + """ + cfiles = [] + ffiles = [] + for file in all_srcfiles: + if ( + file.lower().endswith(".f") + or file.lower().endswith(".f90") + or file.lower().endswith(".for") + or file.lower().endswith(".fpp") + ): + ffiles.append(file) + elif file.lower().endswith(".c") or file.lower().endswith(".cpp"): + cfiles.append(file) # order the source files using the directed acyclic graph in dag.py - orderedsourcefiles = [] - if len(srcfileswithpath) > 0: - orderedsourcefiles += order_source_files(srcfileswithpath) + ordered_srcfiles = [] + if ffiles: + ordered_srcfiles += order_source_files(ffiles) - if len(cfileswithpath) > 0: - orderedsourcefiles += order_c_source_files(cfileswithpath) + if cfiles: + ordered_srcfiles += order_c_source_files(cfiles) - return orderedsourcefiles + return ordered_srcfiles diff --git a/pymake/compiler_switches.py b/pymake/compiler_switches.py index 91ec13f7..59cebc3b 100644 --- a/pymake/compiler_switches.py +++ b/pymake/compiler_switches.py @@ -71,6 +71,31 @@ def get_osname(): return osname +def get_base_app_name(value): + """Remove path and extension from an application name. + + Parameters + ---------- + value : str + application name that may include a directory path and extension + + Returns + ------- + value : str + application name base name with out directory path and extension + + """ + value = os.path.basename(value) + if ( + value.endswith(".exe") + or value.endswith(".dll") + or value.endswith(".so") + ): + value = os.path.splitext(value)[0] + + return value + + def get_prepend(compiler, osname): """Return the appropriate prepend for a compiler switch for a OS. @@ -128,10 +153,8 @@ def get_optlevel( compiler optimization switch """ - # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + # remove target extension, if necessary + target = get_base_app_name(target) # get lower case OS string if osname is None: @@ -139,11 +162,9 @@ def get_optlevel( # remove .exe extension from compiler if necessary if fc is not None: - if ".exe" in fc.lower(): - fc = fc[:-4] + fc = get_base_app_name(fc) if cc is not None: - if ".exe" in cc.lower(): - cc = cc[:-4] + cc = get_base_app_name(cc) compiler = None if fc is not None: @@ -241,13 +262,10 @@ def get_fortran_flags( # define fortran flags if fc is not None: # remove .exe extension of necessary - if ".exe" in fc.lower(): - fc = fc[:-4] + fc = get_base_app_name(fc) # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # get lower case OS string if osname is None: @@ -264,6 +282,8 @@ def get_fortran_flags( else: if osname == "win32": flags.append("static") + if "fPIC" in flags: + flags.remove("fPIC") flags.append("fbacktrace") if debug: flags += ["g", "fcheck=all", "fbounds-check", "Wall"] @@ -299,6 +319,9 @@ def get_fortran_flags( else: if sharedobject: flags.append("fPIC") + else: + if "fPIC" in flags: + flags.remove("fPIC") if debug: flags += ["g"] flags += ["no-heap-arrays", "fpe0", "traceback"] @@ -370,13 +393,10 @@ def get_c_flags( # define c flags if cc is not None: # remove .exe extension of necessary - if ".exe" in cc.lower(): - cc = cc[:-4] + cc = get_base_app_name(cc) # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # get lower case OS string if osname is None: @@ -393,6 +413,8 @@ def get_c_flags( else: if osname == "win32": flags.append("static") + if "fPIC" in flags: + flags.remove("fPIC") if debug: flags += ["g"] if check_gnu_switch_available( @@ -422,6 +444,10 @@ def get_c_flags( else: if sharedobject: flags.append("fpic") + else: + if "fpic" in flags: + flags.remove("fpic") + if debug: flags += ["debug full"] elif cc in ["cl"]: @@ -517,11 +543,9 @@ def get_linker_flags( # remove .exe extension of necessary if fc is not None: - if ".exe" in fc.lower(): - fc = fc[:-4] + fc = get_base_app_name(fc) if cc is not None: - if ".exe" in cc.lower(): - cc = fc[:-4] + cc = get_base_app_name(cc) # set linker compiler compiler = None @@ -531,9 +555,7 @@ def get_linker_flags( compiler = cc # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # get lower case OS string if osname is None: @@ -567,6 +589,12 @@ def get_linker_flags( syslibs_out.append(copt) # add static link flags for GNU compilers else: + if "shared" in syslibs_out: + syslibs_out.remove("shared") + if "dynamiclib" in syslibs_out: + syslibs_out.remove("dynamiclib") + if "dll" in syslibs_out: + syslibs_out.remove("dll") isstatic = False isgfortran = False if osname == "win32": @@ -616,66 +644,6 @@ def get_linker_flags( return compiler, syslibs_out -def set_compiler(target, verbose=False): - """Set fortran and c compilers based on --ifort, --mpiifort, --icc, --cl, - clang++, and --clang command line arguments. - - Parameters - ---------- - target : str - target to build - verbose : bool - boolean for verbose output to terminal - - Returns - ------- - fc : str - string denoting the fortran compiler to use. Default is gfortran. - cc : str - string denoting the c compiler to use. Default is gcc. - - """ - fc = "gfortran" - if target in ["triangle", "gridgen"]: - fc = None - - cc = "gcc" - if target in ["gridgen"]: - cc = "g++" - - # parse command line arguments to see if user specified options - # relative to building the target - for arg in sys.argv: - if arg.lower() == "--ifort" and fc is not None: - fc = "ifort" - elif arg.lower() == "--icc": - cc = "icc" - elif arg.lower() == "--icpc": - cc = "icpc" - elif arg.lower() == "--cl": - cc = "cl" - elif arg.lower() == "--icl": - cc = "icl" - elif arg.lower() == "--clang": - cc = "clang" - elif arg.lower() == "--clang++": - cc = "clang++" - - # reset cc for gridgen if it is specified as 'clang' - if target == "gridgen": - if cc == "clang": - cc = "clang++" - elif cc == "icc": - cc = "icpc" - - if verbose: - msg = '{} fortran code will be built with "{}".\n'.format(target, fc) - msg += '{} c/c++ code will be built with "{}".\n'.format(target, cc) - print(msg) - - return fc, cc - - def set_fflags(target, fc="gfortran", argv=True, osname=None, verbose=False): """Set appropriate fortran compiler flags based on target. @@ -709,13 +677,10 @@ def set_fflags(target, fc="gfortran", argv=True, osname=None, verbose=False): osname = get_osname() # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # remove .exe extension if necessary - if ".exe" in fc.lower(): - fc = fc[:-4] + fc = get_base_app_name(fc) if target == "mp7": if fc == "gfortran": @@ -792,13 +757,10 @@ def set_cflags(target, cc="gcc", argv=True, osname=None, verbose=False): osname = get_osname() # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # remove .exe extension of necessary - if ".exe" in cc.lower(): - cc = cc[:-4] + cc = get_base_app_name(cc) if target == "triangle": if osname in ["linux", "darwin"]: @@ -873,17 +835,13 @@ def set_syslibs( osname = get_osname() # remove target .exe extension, if necessary - target = os.path.basename(target) - if ".exe" in target.lower(): - target = target[:-4] + target = get_base_app_name(target) # remove .exe extension of necessary if fc is not None: - if ".exe" in fc.lower(): - fc = fc[:-4] + fc = get_base_app_name(fc) if cc is not None: - if ".exe" in cc.lower(): - cc = cc[:-4] + cc = get_base_app_name(cc) # initialize syslibs syslibs = [] @@ -943,73 +901,3 @@ def set_syslibs( print(msg) return syslibs - - -def set_debug(target, verbose=False): - """Set boolean that defines if the target should be compiled with debug - compiler options based on -dbg or --debug command line arguments. - - Parameters - ---------- - target : str - target to build - verbose : bool - boolean for verbose output to terminal - - Returns - ------- - debug : bool - - """ - debug = False - for arg in sys.argv: - if arg.lower() == "-dbg" or arg.lower() == "--debug": - debug = True - break - - # write a message - if verbose: - if debug: - comptype = "debug" - else: - comptype = "release" - msg = '{} will be built as a "{}" application.\n'.format( - target, comptype - ) - print(msg) - - return debug - - -def set_arch(target, verbose=False): - """Set architecture to compile target for based on --ia32 command line - argument. Default architecture is intel64 (64-bit). - - Parameters - ---------- - target : str - target to build - verbose : bool - boolean for verbose output to terminal - - Returns - ------- - arch : str - - """ - arch = "intel64" - for arg in sys.argv: - if arg.lower() == "--ia32": - arch = "ia32" - - # set arch to ia32 if building on windows - if target == "triangle": - if get_osname() == "win32": - arch = "ia32" - - # write a message - if verbose: - msg = '{} will be built for "{}" architecture.\n'.format(target, arch) - print(msg) - - return arch diff --git a/pymake/download.py b/pymake/download.py index 8520e169..59332834 100644 --- a/pymake/download.py +++ b/pymake/download.py @@ -1,7 +1,9 @@ import os import sys import shutil +import time import timeit +import requests from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED import tarfile @@ -163,13 +165,60 @@ def compressall(path, file_pths=None, dir_pths=None, patterns=None): return success +def request_get(url, verify=True, timeout=1, max_requests=10, verbose=False): + """Make a url request + + Parameters + ---------- + url : str + url address for the zip file + verify : bool + boolean indicating if the url request should be verified + (default is True) + timeout : int + url request time out length (default is 1 seconds) + max_requests : int + number of url download request attempts (default is 10) + verbose : bool + boolean indicating if output will be printed to the terminal + (default is False) + + Returns + ------- + req : request object + request object for url + + """ + for idx in range(max_requests): + if verbose: + msg = "open request attempt {} of {}".format(idx + 1, max_requests) + print(msg) + try: + req = requests.get( + url, stream=True, verify=verify, timeout=timeout + ) + except: + if idx < max_requests - 1: + time.sleep(13) + continue + else: + msg = "Cannot open request from:\n" + " {}\n\n".format(url) + print(msg) + req.raise_for_status() + + # successful request + break + + return req + + def download_and_unzip( url, pth="./", delete_zip=True, verify=True, timeout=30, - nattempts=10, + max_requests=10, chunk_size=2048000, verbose=False, ): @@ -188,7 +237,7 @@ def download_and_unzip( boolean indicating if the url request should be verified timeout : int url request time out length (default is 30 seconds) - nattempts : int + max_requests : int number of url download request attempts (default is 10) chunk_size : int maximum url download request chunk size (default is 2048000 bytes) @@ -199,11 +248,7 @@ def download_and_unzip( ------- """ - try: - import requests - except Exception as e: - msg = "pymake.download_and_unzip() error import requests: " + str(e) - raise Exception(msg) + # create download directory if not os.path.exists(pth): if verbose: print("Creating the directory:\n {}".format(pth)) @@ -218,70 +263,87 @@ def download_and_unzip( # download the file success = False tic = timeit.default_timer() - for idx in range(nattempts): - # open request - req = requests.get(url, stream=True, verify=verify) - if req.status_code != 200: - if idx < nattempts - 1: - continue - else: - msg = "Cannot download file:\n {}\n\n".format(url) - print(msg) - req.raise_for_status() + # open request + req = request_get( + url, + verify=verify, + timeout=timeout, + max_requests=max_requests, + verbose=verbose, + ) + + # get content length + tag = "Content-length" + if tag in req.headers: + file_size = req.headers[tag] + len_file_size = len(file_size) + file_size = int(file_size) + + bfmt = "{:" + "{}".format(len_file_size) + ",d}" + sbfmt = ( + "{:>" + "{}".format(len(bfmt.format(int(file_size)))) + "s} bytes" + ) + msg = " file size: {}".format( + sbfmt.format(bfmt.format(int(file_size))) + ) + if verbose: + print(msg) + else: + msg = "'{}' header not available from '{}'".format(tag, url) + raise Exception(msg) + + if file_size <= 0: + msg = "invalid request file size ({}) from '{}'".format(file_size, url) + raise Exception(msg) + + # download data from url + for idx in range(max_requests): # print download attempt message if verbose: print(" download attempt: {}".format(idx + 1)) # connection established - download the file - fs = 0 - lenfs = 0 - if "Content-length" in req.headers: - fs = req.headers["Content-length"] - lenfs = len(fs) - fs = int(fs) - if fs > 0: - bfmt = "{:" + "{}".format(lenfs) + ",d}" - sbfmt = "{:>" + "{}".format(len(bfmt.format(int(fs)))) + "s} bytes" - msg = " file size: {}".format(sbfmt.format(bfmt.format(int(fs)))) - if verbose: - print(msg) - ds = 0 + download_size = 0 try: - req = requests.get( - url, verify=verify, timeout=timeout, stream=True - ) - if req.status_code == 200: - with open(file_name, "wb") as f: - for chunk in req.iter_content(chunk_size=chunk_size): - if chunk: - ds += len(chunk) - if fs > 0: - msg = ( - " downloaded " - + sbfmt.format(bfmt.format(ds)) - + " of " - + bfmt.format(int(fs)) - + " bytes" - + " ({:10.4%})".format( - float(ds) / float(fs) - ) + with open(file_name, "wb") as f: + for chunk in req.iter_content(chunk_size=chunk_size): + if chunk: + download_size += len(chunk) + if file_size > 0: + msg = ( + " downloaded " + + sbfmt.format(bfmt.format(download_size)) + + " of " + + bfmt.format(int(file_size)) + + " bytes" + + " ({:10.4%})".format( + float(download_size) / float(file_size) ) - if verbose: - print(msg) - else: - sys.stdout.write(".") - sys.stdout.flush() - f.write(chunk) + ) + if verbose: + print(msg) + else: + sys.stdout.write(".") + sys.stdout.flush() + f.write(chunk) # check that the entire file has been downloaded - if ds == fs: + if download_size == file_size: success = True else: continue - else: - success = False except: + # reestablish request + req = request_get( + url, + verify=verify, + timeout=timeout, + max_requests=max_requests, + verbose=verbose, + ) + + # try to download the data again continue # terminate the download attempt loop @@ -295,9 +357,13 @@ def download_and_unzip( print("\ntotal download time: {} seconds".format(tsec)) if success: - if fs > 0: + if file_size > 0: if verbose: - print("download speed: {} MB/s".format(fs / (1e6 * tsec))) + print( + "download speed: {} MB/s".format( + file_size / (1e6 * tsec) + ) + ) else: msg = "could not download...{}".format(url) raise ConnectionError(msg) @@ -344,14 +410,11 @@ def zip_all(path, file_pths=None, dir_pths=None, patterns=None): ---------- path : str path of the zip file that will be created - file_pths : str or list file path or list of file paths to be compressed - dir_pths : str or list directory path or list of directory paths to search for files that will be compressed - patterns : str or list file pattern or list of file patterns s to match to when creating a list of files that will be compressed @@ -412,42 +475,43 @@ def get_default_json(tag_name=None): return json_obj -def get_request_json(request_url): +def get_request_json(request_url, verbose=False): """Process a url request and return a json if successful. Parameters ---------- request_url : str url for request + verbose : bool + boolean indicating if output will be printed to the terminal + default is false Returns ------- success : bool boolean indicating if the requat failed - status_code: integer request status code - json_obj : dict json object """ - import requests import json + max_requests = 10 json_obj = None success = True # open request - r = requests.get(request_url) + req = request_get(request_url, max_requests=max_requests, verbose=verbose) # connection established - retrieve the json - if r.ok: - json_obj = json.loads(r.text or r.content) + if req.ok: + json_obj = json.loads(req.text or req.content) else: - success = r.status_code == requests.codes.ok + success = req.status_code == requests.codes.ok - return success, r, json_obj + return success, req, json_obj def repo_json(github_repo, tag_name=None, error_return=False, verbose=False): @@ -479,7 +543,7 @@ def repo_json(github_repo, tag_name=None, error_return=False, verbose=False): request_url = "{}/releases/latest".format(repo_url) else: request_url = "{}/releases".format(repo_url) - success, r, json_cat = get_request_json(request_url) + success, _, json_cat = get_request_json(request_url, verbose=verbose) if success: request_url = None for release in json_cat: @@ -513,7 +577,7 @@ def repo_json(github_repo, tag_name=None, error_return=False, verbose=False): print(msg) # process the request - success, r, json_obj = get_request_json(request_url) + success, req, json_obj = get_request_json(request_url, verbose=verbose) # evaluate request errors if not success: @@ -529,7 +593,7 @@ def repo_json(github_repo, tag_name=None, error_return=False, verbose=False): if error_return: json_obj = None else: - r.raise_for_status() + req.raise_for_status() # return json object return json_obj diff --git a/pymake/pymake.py b/pymake/pymake.py index ff4ea7c7..158b7a3c 100644 --- a/pymake/pymake.py +++ b/pymake/pymake.py @@ -1,15 +1,17 @@ -"""Make a binary executable for a FORTRAN program, such as MODFLOW.""" +"""Make a binary executable for a FORTRAN, C, or C++ program, such as +MODFLOW 6. +""" __author__ = "Christian D. Langevin" __date__ = "October 26, 2014" -__version__ = "1.1.0" -__maintainer__ = "Christian D. Langevin" +__version__ = "1.2.0" +__maintainer__ = "Joseph D. Hughes" __email__ = "langevin@usgs.gov" __status__ = "Production" __description__ = """ -This is the pymake program for compiling fortran source files, such as -the source files that come with MODFLOW. The program works by building -a directed acyclic graph of the module dependencies and then compiling -the source files in the proper order. +This is the pymake program for compiling fortran, c, and c++ source files, +such as the source files that come with MODFLOW. The program works by building +a directed acyclic graph of the module dependencies and then compiling the +source files in the proper order. """ import os @@ -26,7 +28,11 @@ get_linker_flags, ) from .download import download_and_unzip, zip_all -from .pymake_base import main, parser, get_arg_dict, parser_setup +from .pymake_base import main +from .pymake_parser import ( + get_arg_dict, + parser_setup, +) from .usgsprograms import usgs_program_data from .usgs_src_update import build_replace @@ -71,20 +77,30 @@ def __init__(self, name="pymake", verbose=None): self.appdir = None self.keep = None self.zip = None + self.inplace = None # set class variables with default values from arg_dict for key, value in get_arg_dict().items(): setattr(self, key, value["default"]) - # do not parse command line arguments if program running script is - # nosetests, pytest, etc. - if "test" not in sys.argv[0].lower(): + # do not parse command line arguments if python is running script + if sys.argv[0].lower().endswith(".py"): self.arg_parser() # reset select variables using passed variables if verbose is not None: self.verbose = verbose + def finalize(self): + """Finalize Pymake class + + Returns + ------- + + """ + if self.download: + self._download_cleanup() + def print_settings(self): """Print settings defined by command line arguments @@ -94,7 +110,10 @@ def print_settings(self): """ print("\nPymake settings\n" + 30 * "-") for key, value in get_arg_dict().items(): - print(" {}={}".format(key, getattr(self, key, value["default"]))) + print_value = getattr(self, key, value["default"]) + if isinstance(print_value, list): + print_value = ", ".join(print_value) + print(" {}={}".format(key, print_value)) print("\n") def argv_reset_settings(self, args): @@ -237,16 +256,24 @@ def download_setup( # set url if url is None: url = prog_dict.url - self.url = url - # set download_dir - self.download_path = download_path - self.download_dir = os.path.join(download_path, prog_dict.dirname) + # determine if the download url has changed + new_url = False + if self.url != url: + new_url = True + + if new_url: + # automatic clean up + if self.download: + self._download_cleanup() - # set download parameters - self.download = False - self.verify = verify - self.timeout = timeout + # setup new + self.url = url + self.download = False + self.verify = verify + self.timeout = timeout + self.download_path = download_path + self.download_dir = os.path.join(download_path, prog_dict.dirname) return @@ -309,7 +336,7 @@ def download_url(self): ) return self.download - def download_cleanup(self): + def _download_cleanup(self): """ Returns @@ -376,7 +403,7 @@ def set_include_subdirs(self): target = target[:-4] # determine if source subdirectories should be included - if target in ["mf6", "gridgen", "mf6beta", "gsflow"]: + if target in ["mf6", "libmf6", "gridgen", "mf6beta", "gsflow"]: self.include_subdirs = True return @@ -408,15 +435,26 @@ def set_build_target_bool(self, target=None): return build_target + def set_srcdir2(self): + """Set srcdir2 to compile target. Default is None. + + Parameters + ---------- + + Returns + ------- + + """ + if self.srcdir2 is None: + if self.target in ("libmf6",): + self.srcdir2 = os.path.join(self.download_dir, "src") + return + def set_extrafiles(self): """Set extrafiles to compile target. Default is None. Parameters ---------- - target : str - target to build - download_dir : str - path downloaded files will be placed in Returns ------- @@ -466,6 +504,23 @@ def set_extrafiles(self): return + def set_excludefiles(self): + """Set excludefiles to compile target. Default is None. + + Parameters + ---------- + + Returns + ------- + + """ + if self.excludefiles is None: + if self.target in ("libmf6",): + self.excludefiles = [ + os.path.join(self.download_dir, "src", "mf6.f90") + ] + return + def build(self, target=None, srcdir=None, modify_exe_name=False): """Build the target @@ -495,9 +550,15 @@ def build(self, target=None, srcdir=None, modify_exe_name=False): # set include_subdirs for known targets self.set_include_subdirs() + # set srcdir2 for known targets + self.set_srcdir2() + # set extrafiles for known targets self.set_extrafiles() + # set excludefiles for known targets + self.set_excludefiles() + # set compiler flags if self.fc != "none": if self.fflags is None: @@ -602,6 +663,8 @@ def build(self, target=None, srcdir=None, modify_exe_name=False): excludefiles=self.excludefiles, sharedobject=self.sharedobject, appdir=self.appdir, + verbose=self.verbose, + inplace=self.inplace, ) # issue error if target was not built @@ -632,11 +695,21 @@ def update_target(self, target, modify_target=False): updated target name """ - # add exe extension to target on windows + # add extension to target on windows or if shared object if sys.platform.lower() == "win32": + if self.sharedobject: + ext = ".dll" + else: + ext = ".exe" + else: + if self.sharedobject: + ext = ".so" + else: + ext = None + if ext is not None: filename, file_extension = os.path.splitext(target) - if file_extension.lower() != ".exe": - target += ".exe" + if file_extension.lower() != ext: + target += ext # add double and debug to target name if modify_target: @@ -649,33 +722,3 @@ def update_target(self, target, modify_target=False): if filename.lower()[-1] != "d": target = filename + "d" + file_extension return target - - -if __name__ == "__main__": - # get the arguments - args = parser() - - # call main -- note that this form allows main to be called - # from python as a function. - main( - args.srcdir, - args.target, - fc=args.fc, - cc=args.cc, - makeclean=args.makeclean, - expedite=args.expedite, - dryrun=args.dryrun, - double=args.double, - debug=args.debug, - include_subdirs=args.subdirs, - fflags=args.fflags, - cflags=args.cflags, - arch=args.arch, - makefile=args.makefile, - srcdir2=args.commonsrc, - extrafiles=args.extrafiles, - excludefiles=args.excludefiles, - sharedobject=args.sharedobject, - appdir=args.appdir, - verbose=args.verbose, - ) diff --git a/pymake/pymake_base.py b/pymake/pymake_base.py index 399d7fed..b9a94ff9 100644 --- a/pymake/pymake_base.py +++ b/pymake/pymake_base.py @@ -1,12 +1,10 @@ -#! /usr/bin/env python import os import traceback import shutil -import argparse import datetime import inspect -from .pymake import __description__, __version__ +from .pymake import __version__ from .Popen_wrapper import ( process_Popen_initialize, @@ -22,286 +20,256 @@ get_linker_flags, ) from .compiler_language_files import ( + get_srcfiles, get_ordered_srcfiles, get_c_files, get_fortran_files, ) -# if sys.version_info >= (3, 3): -# from shutil import which -# else: -# from distutils.spawn import find_executable as which - # define temporary directories -srcdir_temp = os.path.join(".", "src_temp") objdir_temp = os.path.join(".", "obj_temp") moddir_temp = os.path.join(".", "mod_temp") -def get_arg_dict(): - """Get command line argument dictionary - - Returns - ------- - return : dict - Dictionary of command line argument options - - """ - return { - "srcdir": { - "tag": ("srcdir",), - "help": "Location of source directory", - "default": None, - "choices": None, - "action": None, - }, - "target": { - "tag": ("target",), - "help": "Name of target to create (can include path)", - "default": None, - "choices": None, - "action": None, - }, - "fc": { - "tag": ("-fc",), - "help": "Fortran compiler to use (default is gfortran)", - "default": "gfortran", - "choices": ["ifort", "mpiifort", "gfortran", "none"], - "action": None, - }, - "cc": { - "tag": ("-cc",), - "help": "C/C++ compiler to use (default is gcc)", - "default": "gcc", - "choices": [ - "gcc", - "clang", - "clang++", - "icc", - "icl", - "mpiicc", - "g++", - "cl", - "none", - ], - "action": None, - }, - "arch": { - "tag": ("-ar", "--arch"), - "help": "Architecture to use for ifort (default is intel64)", - "default": "intel64", - "choices": ["ia32", "ia32_intel64", "intel64"], - "action": None, - }, - "makeclean": { - "tag": ("-mc", "--makeclean"), - "help": "Clean files when done (default is True)", - "default": True, - "choices": None, - "action": "store_true", - }, - "double": { - "tag": ("-dbl", "--double"), - "help": "Force double precision (default is False)", - "default": False, - "choices": None, - "action": "store_false", - }, - "debug": { - "tag": ("-dbg", "--debug"), - "help": "Create debug version (default is False)", - "default": False, - "choices": None, - "action": "store_false", - }, - "expedite": { - "tag": ("-e", "--expedite"), - "help": """Only compile out of date source files. - Clean must not have been used on previous build. - Does not work yet for ifort. (default is False)""", - "default": False, - "choices": None, - "action": "store_true", - }, - "dryrun": { - "tag": ("-dr", "--dryrun"), - "help": """Do not actually compile. Files will be - deleted, if --makeclean is used. - Does not work yet for ifort. (default is False)""", - "default": False, - "choices": None, - "action": "store_true", - }, - "include_subdirs": { - "tag": ("-sd", "--subdirs"), - "help": """Include source files in srcdir subdirectories. - (default is None)""", - "default": None, - "choices": None, - "action": "store_true", - }, - "fflags": { - "tag": ("-ff", "--fflags"), - "help": "Additional fortran compiler flags (default is None)", - "default": None, - "choices": None, - "action": None, - }, - "cflags": { - "tag": ("-cf", "--cflags"), - "help": "Additional C/C++ compiler flags (default is None)", - "default": None, - "choices": None, - "action": None, - }, - "syslibs": { - "tag": ("-sl", "--syslibs"), - "help": "Linker system libraries (default is None)", - "default": None, - "choices": ["-lc", "-lm"], - "action": None, - }, - "makefile": { - "tag": ("-mf", "--makefile"), - "help": "Create a standard makefile (default is False)", - "default": False, - "choices": None, - "action": "store_true", - }, - "srcdir2": { - "tag": ("-cs", "--commonsrc"), - "help": """Additional directory with common source files. - (default is None)""", - "default": None, - "choices": None, - "action": None, - }, - "extrafiles": { - "tag": ("-ef", "--extrafiles"), - "help": """List of extra source files to include in the - compilation. extrafiles can be either a list of files - or the name of a text file that contains a list of - files. (default is None)""", - "default": None, - "choices": None, - "action": None, - }, - "excludefiles": { - "tag": ("-exf", "--excludefiles"), - "help": """List of extra source files to exclude from the - compilation. excludefiles can be either a list of - files or the name of a text file that contains a list - of files. (default is None)""", - "default": None, - "choices": None, - "action": None, - }, - "sharedobject": { - "tag": ("-so", "--sharedobject"), - "help": "Create shared object (default is False)", - "default": False, - "choices": None, - "action": "store_true", - }, - "appdir": { - "tag": ("-ad", "--appdir"), - "help": "Target path (default is None)", - "default": None, - "choices": None, - "action": None, - }, - "verbose": { - "tag": ("-v", "--verbose"), - "help": "Verbose output to terminal (default is False)", - "default": False, - "choices": None, - "action": "store_true", - }, - "keep": { - "tag": ("--keep",), - "help": "Keep existing executable (default is False)", - "default": False, - "choices": None, - "action": "store_true", - }, - "zip": { - "tag": ("--zip",), - "help": "Zip built executable (default is False)", - "default": None, - "choices": None, - "action": None, - }, - } - - -def parser_setup(p, value, reset_default=False): - """Add argument to argparse object +def main( + srcdir=None, + target=None, + fc="gfortran", + cc="gcc", + makeclean=True, + expedite=False, + dryrun=False, + double=False, + debug=False, + include_subdirs=False, + fflags=None, + cflags=None, + syslibs=None, + arch="intel64", + makefile=False, + srcdir2=None, + extrafiles=None, + excludefiles=None, + sharedobject=False, + appdir=None, + verbose=False, + inplace=False, +): + """Main pymake function. Parameters ---------- - p : object - argparse object - value : dict - argparse settings - reset_default : bool - boolean that defines if default values should be used + srcdir : str + path for directory containing source files + target : str + executable name or path for executable to create + fc : str + fortran compiler + cc : str + c or cpp compiler + makeclean : bool + boolean indicating if intermediate files should be cleaned up + after successful build + expedite : bool + boolean indicating if only out of date source files will be compiled. + Clean must not have been used on previous build. + dryrun : bool + boolean indicating if source files should be compiled. Files will be + deleted, if makeclean is True. + double : bool + boolean indicating a compiler switch will be used to create an + executable with double precision real variables. + debug : bool + boolean indicating is a debug executable will be built + include_subdirs : bool + boolean indicating source files in srcdir subdirectories should be + included in the build + fflags : list + user provided list of fortran compiler flags + cflags : list + user provided list of c or cpp compiler flags + syslibs : list + user provided syslibs + arch : str + Architecture to use for Intel Compilers on Windows (default is intel64) + makefile : bool + boolean indicating if a GNU make makefile should be created + srcdir2 : str + additional directory with common source files. + extrafiles : str + path for extrafiles file that contains paths to additional source + files to include + excludefiles : str + path for excludefiles file that contains filename of source files + to exclude from the build + sharedobject : bool + boolean indicating a shared object (.so or .dll) will be built + appdir : str + path for executable + verbose : bool + boolean indicating if output will be printed to the terminal + inplace : bool + boolean indicating that the source files in srcdir, srcdir2, and + defined in extrafiles will be used directly. If inplace is True, + source files will be copied to a directory named srcdir_temp. + (default is False) Returns ------- - p : object - updated argparse object + returncode : int + return code """ - if reset_default: - default = None - else: - default = value["default"] - if value["action"] is None: - p.add_argument( - *value["tag"], - help=value["help"], - default=default, - choices=value["choices"], - ) - else: - p.add_argument( - *value["tag"], - help=value["help"], - default=default, - action=value["action"], + + if srcdir is not None and target is not None: + + if inplace: + srcdir_temp = srcdir + else: + srcdir_temp = os.path.join(".", "src_temp") + + # process appdir + if appdir is not None: + target = os.path.join(appdir, target) + + # make appdir if it does not exist + if not os.path.isdir(appdir): + os.makedirs(appdir) + else: + target = os.path.join(".", target) + + # set fc and cc to None if they are passed as 'none' + if fc == "none": + fc = None + if cc == "none": + cc = None + if fc is None and cc is None: + msg = ( + "Nothing to do the fortran (-fc) and c/c++ compilers (-cc)" + + "are both 'none'." + ) + raise ValueError(msg) + + # convert fflags, cflags, and syslibs to lists + if fflags is None: + fflags = [] + elif isinstance(fflags, str): + fflags = fflags.split() + if cflags is None: + cflags = [] + elif isinstance(cflags, str): + cflags = cflags.split() + if syslibs is None: + syslibs = [] + elif isinstance(syslibs, str): + syslibs = syslibs.split() + + # write summary information + if verbose: + print("\nsource files are in:\n {}\n".format(srcdir)) + print("executable name to be created:\n {}\n".format(target)) + if srcdir2 is not None: + msg = "additional source files are in:\n" + " {}\n".format( + srcdir2 + ) + print(msg) + + # make sure the path for the target exists + pth = os.path.dirname(target) + if pth == "": + pth = "." + if not os.path.exists(pth): + print("creating target path - {}\n".format(pth)) + os.makedirs(pth) + + # initialize + srcfiles = pymake_initialize( + srcdir, + target, + srcdir2, + extrafiles, + excludefiles, + include_subdirs, + srcdir_temp, ) - return p + # get ordered list of files to compile + srcfiles = get_ordered_srcfiles(srcfiles) -def parser(): - """Construct the parser and return argument values. + # set intelwin flag to True in compiling on windows with Intel compilers + intelwin = False + if get_osname() == "win32": + if fc is not None: + if fc in ["ifort", "mpiifort"]: + intelwin = True + if cc is not None: + if cc in ["cl", "icl"]: + intelwin = True - Parameters - ---------- + # update openspec files based on intelwin + if not intelwin: + create_openspec(srcfiles, verbose) - Returns - ------- - args : list - command line argument list + # compile the executable + returncode = pymake_compile( + srcfiles, + target, + fc, + cc, + expedite, + dryrun, + double, + debug, + fflags, + cflags, + syslibs, + arch, + intelwin, + sharedobject, + verbose, + ) - """ - description = __description__ - parser = argparse.ArgumentParser( - description=description, - epilog="""Note that the source directory - should not contain any bad or duplicate - source files as all source files in the - source directory will be built and - linked.""", - ) + # create makefile + if makefile: + create_makefile( + target, + srcdir, + srcdir2, + extrafiles, + srcfiles, + debug, + double, + fc, + cc, + fflags, + cflags, + syslibs, + verbose, + ) - for _, value in get_arg_dict().items(): - parser = parser_setup(parser, value) - args = parser.parse_args() - return args + # clean up temporary files + if makeclean and returncode == 0: + clean(target, intelwin, inplace, srcdir_temp, verbose) + else: + msg = ( + "Nothing to do, the srcdir ({}) ".format(srcdir) + + "and/or target ({}) ".format(target) + + "are not specified." + ) + raise ValueError(msg) + + return returncode -def pymake_initialize(srcdir, target, commonsrc, extrafiles, excludefiles): +def pymake_initialize( + srcdir, + target, + commonsrc, + extrafiles, + excludefiles, + include_subdirs, + srcdir_temp, +): """Remove temp source directory and target, and then copy source into source temp directory. @@ -319,73 +287,105 @@ def pymake_initialize(srcdir, target, commonsrc, extrafiles, excludefiles): excludefiles : str path for excludefiles file that contains filename of source files to exclude from the build + include_subdirs : bool + boolean indicating source files in srcdir subdirectories should be + included in the build + srcdir_temp : str + path for directory that will contain the source files. If + srcdir_temp is the same as srcdir then the original source files + will be used. Returns ------- - None + srcfiles : list + list of source files for build """ # remove the target if it already exists - try: + if os.path.isfile(target): os.remove(target) - except: - pass + + inplace = False + if srcdir == srcdir_temp: + inplace = True + + # if exclude is not None, then it is a text file with a list of + # source files that need to be excluded from srctemp. + excludefiles = get_extra_exclude_files(excludefiles) + if excludefiles: + for idx, exclude_file in enumerate(excludefiles): + excludefiles[idx] = os.path.basename(exclude_file) # remove srcdir_temp and copy in srcdir - if os.path.isdir(srcdir_temp): - shutil.rmtree(srcdir_temp) - shutil.copytree(srcdir, srcdir_temp) + if not inplace: + if os.path.isdir(srcdir_temp): + shutil.rmtree(srcdir_temp) + if excludefiles: + shutil.copytree( + srcdir, + srcdir_temp, + ignore=shutil.ignore_patterns(*excludefiles), + ) + else: + shutil.copytree(srcdir, srcdir_temp) + + # get a list of source files in srcdir_temp to include + srcfiles = get_srcfiles(srcdir_temp, include_subdirs) # copy files from a specified common source directory if # commonsrc is not None if commonsrc is not None: - pth = os.path.basename(os.path.normpath(commonsrc)) - pth = os.path.join(srcdir_temp, pth) - shutil.copytree(commonsrc, pth) + if not inplace: + src = os.path.relpath(commonsrc, os.getcwd()) + dst = os.path.join( + srcdir_temp, os.path.basename(os.path.normpath(commonsrc)) + ) + if excludefiles: + shutil.copytree( + src, dst, ignore=shutil.ignore_patterns(*excludefiles), + ) + else: + shutil.copytree(src, dst) + else: + dst = os.path.normpath(os.path.relpath(commonsrc, srcdir_temp)) - # if extrafiles is not none, then it is a text file with a list of + srcfiles += get_srcfiles(dst, include_subdirs) + + # if extrafiles is not None, then it is a text file with a list of # additional source files that need to be copied into srctemp and # compiled. - files = get_extrafiles(extrafiles) + files = get_extra_exclude_files(extrafiles) if files is None: files = [] - for fname in files: - if not os.path.isfile(fname): - print("Current working directory: {}".format(os.getcwd())) - print("Error in extrafiles: {}".format(extrafiles)) - print("Could not find file: {}".format(fname)) - raise Exception() - dst = os.path.join(srcdir_temp, os.path.basename(fname)) - if os.path.isfile(dst): - raise Exception( - "Error with extrafile. Name conflicts with " - "an existing source file: {}".format(dst) - ) - shutil.copy(fname, dst) - - # if exclude is not None, then it is a text file with a list of - # source files that need to be excluded from srctemp. - files = get_extrafiles(excludefiles) - if files is None: - files = [] - for fname in files: - if not os.path.isfile(fname): - print("Current working directory: {}".format(os.getcwd())) - print("Warning in excludefiles: {}".format(excludefiles)) - print("Could not find file: {}".format(fname)) + for fpth in files: + if not os.path.isfile(fpth): + msg = "Current working directory: {}\n".format(os.getcwd()) + msg += "Error in extrafiles: {}\n".format(extrafiles) + msg += "Could not find file: {}".format(fpth) + raise FileNotFoundError(msg) + if inplace: + dst = os.path.normpath(os.path.relpath(fpth, os.getcwd())) else: - base = None - tail = True - while tail: - fname, tail = os.path.split(fname) - if base is None: - base = tail - else: - base = os.path.join(tail, base) - dst = os.path.join(srcdir_temp, base) - if os.path.isfile(dst): - os.remove(dst) - tail = False + dst = os.path.join(srcdir_temp, os.path.basename(fpth)) + if os.path.isfile(dst): + raise ValueError( + "Error with extrafile. Name conflicts with " + "an existing source file: {}".format(dst) + ) + if not inplace: + shutil.copy(fpth, dst) + + # add extrafiles to srcfiles + srcfiles.append(dst) + + # remove exclude files from srcfiles list + if excludefiles: + remove_list = [] + for fpth in srcfiles: + if os.path.basename(fpth) in excludefiles: + remove_list.append(fpth) + for fpth in remove_list: + srcfiles.remove(fpth) # if they don't exist, create directories for objects and mods if not os.path.exists(objdir_temp): @@ -393,10 +393,10 @@ def pymake_initialize(srcdir, target, commonsrc, extrafiles, excludefiles): if not os.path.exists(moddir_temp): os.makedirs(moddir_temp) - return + return srcfiles -def get_extrafiles(extrafiles): +def get_extra_exclude_files(extrafiles): """Get extrafiles to include in compilation from a file or a list. Parameters @@ -434,7 +434,7 @@ def get_extrafiles(extrafiles): return files -def clean(target, intelwin, verbose=False): +def clean(target, intelwin, inplace, srcdir_temp, verbose=False): """Cleanup intermediate files. Remove mod and object files, and remove the temporary source directory. @@ -445,6 +445,17 @@ def clean(target, intelwin, verbose=False): intelwin : bool boolean indicating if pymake was used to compile source code on Windows using Intel compilers + inplace : bool + boolean indicating that the source files in srcdir, srcdir2, and + defined in extrafiles will be used directly. If inplace is True, + source files will be copied to a directory named srcdir_temp. + (default is False) + srcdir_temp : str + path for directory that will contain the source files. If + srcdir_temp is the same as srcdir then the original source files + will be used. + verbose : bool + boolean indicating if output will be printed to the terminal Returns ------- @@ -489,11 +500,18 @@ def clean(target, intelwin, verbose=False): + "and module directories..." ) print(msg) - if os.path.isdir(srcdir_temp): - shutil.rmtree(srcdir_temp) + if not inplace: + if os.path.isdir(srcdir_temp): + if verbose: + print("removing...'{}'".format(srcdir_temp)) + shutil.rmtree(srcdir_temp) if os.path.isdir(objdir_temp): + if verbose: + print("removing...'{}'".format(objdir_temp)) shutil.rmtree(objdir_temp) if os.path.isdir(moddir_temp): + if verbose: + print("removing...'{}'".format(moddir_temp)) shutil.rmtree(moddir_temp) # remove the windows batchfile @@ -502,7 +520,7 @@ def clean(target, intelwin, verbose=False): return -def create_openspec(verbose): +def create_openspec(srcfiles, verbose): """Create new openspec.inc, FILESPEC.INC, and filespec.inc files that uses STREAM ACCESS. This is specific to MODFLOW and MT3D based targets. Source directories are scanned and files defining file access are replaced. @@ -515,11 +533,20 @@ def create_openspec(verbose): None """ + # list of files to replace files = ["openspec.inc", "filespec.inc"] - dirs = [d[0] for d in os.walk(srcdir_temp)] - for d in dirs: + + # build list of directory paths from srcfiles + dpths = [] + for fpth in srcfiles: + dpth = os.path.dirname(fpth) + if dpth not in dpths: + dpths.append(dpth) + + # replace files in directory paths if they exist + for dpth in dpths: for file in files: - fpth = os.path.join(d, file) + fpth = os.path.join(dpth, file) if os.path.isfile(fpth): if verbose: print('replacing..."{}"'.format(fpth)) @@ -618,6 +645,11 @@ def pymake_compile( boolean indicating a shared object (.so or .dll) will be built verbose : bool boolean indicating if output will be printed to the terminal + inplace : bool + boolean indicating that the source files in srcdir, srcdir2, and + defined in extrafiles will be used directly. If inplace is True, + source files will be copied to a directory named srcdir_temp. + (default is False) Returns ------- @@ -625,6 +657,7 @@ def pymake_compile( returncode """ + # write pymake setting if verbose: msg = ( "\nPymake settings in {}\n".format(pymake_compile.__name__) @@ -632,11 +665,13 @@ def pymake_compile( ) print(msg) frame = inspect.currentframe() - args, _, _, values = inspect.getargvalues(frame) - for arg in args: + fnargs, _, _, values = inspect.getargvalues(frame) + for arg in fnargs: value = values[arg] - if isinstance(value, list): - value = " ".join(value) + if not value: + value = "None" + elif isinstance(value, list): + value = ", ".join(value) print(" {}={}".format(arg, value)) # initialize returncode @@ -1117,7 +1152,7 @@ def create_makefile( dirs = dirs + dirs2 # source files in extrafiles - files = get_extrafiles(extrafiles) + files = get_extra_exclude_files(extrafiles) if files is not None: for ef in files: fdir = os.path.dirname(ef) @@ -1554,214 +1589,3 @@ def create_makefile( f.close() return - - -def main( - srcdir=None, - target=None, - fc="gfortran", - cc="gcc", - makeclean=True, - expedite=False, - dryrun=False, - double=False, - debug=False, - include_subdirs=False, - fflags=None, - cflags=None, - syslibs=None, - arch="intel64", - makefile=False, - srcdir2=None, - extrafiles=None, - excludefiles=None, - sharedobject=False, - appdir=None, - verbose=False, -): - """Main pymake function. - - Parameters - ---------- - srcdir : str - path for directory containing source files - target : str - executable name or path for executable to create - fc : str - fortran compiler - cc : str - c or cpp compiler - makeclean : bool - boolean indicating if intermediate files should be cleaned up - after successful build - expedite : bool - boolean indicating if only out of date source files will be compiled. - Clean must not have been used on previous build. - dryrun : bool - boolean indicating if source files should be compiled. Files will be - deleted, if makeclean is True. - double : bool - boolean indicating a compiler switch will be used to create an - executable with double precision real variables. - debug : bool - boolean indicating is a debug executable will be built - include_subdirs : bool - boolean indicating source files in srcdir subdirectories should be - included in the build - fflags : list - user provided list of fortran compiler flags - cflags : list - user provided list of c or cpp compiler flags - syslibs : list - user provided syslibs - arch : str - Architecture to use for Intel Compilers on Windows (default is intel64) - makefile : bool - boolean indicating if a GNU make makefile should be created - srcdir2 : str - additional directory with common source files. - extrafiles : str - path for extrafiles file that contains paths to additional source - files to include - excludefiles : str - path for excludefiles file that contains filename of source files - to exclude from the build - sharedobject : bool - boolean indicating a shared object (.so or .dll) will be built - appdir : str - path for executable - verbose : bool - boolean indicating if output will be printed to the terminal - - Returns - ------- - returncode : int - return code - - """ - - if srcdir is not None and target is not None: - - # process appdir - if appdir is not None: - target = os.path.join(appdir, target) - - # make appdir if it does not exist - if not os.path.isdir(appdir): - os.makedirs(appdir) - else: - target = os.path.join(".", target) - - # set fc and cc to None if they are passed as 'none' - if fc == "none": - fc = None - if cc == "none": - cc = None - if fc is None and cc is None: - msg = ( - "Nothing to do the fortran (-fc) and c/c++ compilers (-cc)" - + "are both 'none'." - ) - raise ValueError(msg) - - # convert fflags, cflags, and syslibs to lists - if fflags is None: - fflags = [] - elif isinstance(fflags, str): - fflags = fflags.split() - if cflags is None: - cflags = [] - elif isinstance(cflags, str): - cflags = cflags.split() - if syslibs is None: - syslibs = [] - elif isinstance(syslibs, str): - syslibs = syslibs.split() - - # write summary information - if verbose: - print("\nsource files are in:\n {}\n".format(srcdir)) - print("executable name to be created:\n {}\n".format(target)) - if srcdir2 is not None: - msg = "additional source files are in:\n" + " {}\n".format( - srcdir2 - ) - print(msg) - - # make sure the path for the target exists - pth = os.path.dirname(target) - if pth == "": - pth = "." - if not os.path.exists(pth): - print("creating target path - {}\n".format(pth)) - os.makedirs(pth) - - # initialize - pymake_initialize(srcdir, target, srcdir2, extrafiles, excludefiles) - - # get ordered list of files to compile - srcfiles = get_ordered_srcfiles(srcdir_temp, include_subdirs) - - # set intelwin flag to True in compiling on windows with Intel compilers - intelwin = False - if get_osname() == "win32": - if fc is not None: - if fc in ["ifort", "mpiifort"]: - intelwin = True - if cc is not None: - if cc in ["cl", "icl"]: - intelwin = True - - # update openspec files based on intelwin - if not intelwin: - create_openspec(verbose) - - # compile the executable - returncode = pymake_compile( - srcfiles, - target, - fc, - cc, - expedite, - dryrun, - double, - debug, - fflags, - cflags, - syslibs, - arch, - intelwin, - sharedobject, - verbose, - ) - - # create makefile - if makefile: - create_makefile( - target, - srcdir, - srcdir2, - extrafiles, - srcfiles, - debug, - double, - fc, - cc, - fflags, - cflags, - syslibs, - verbose, - ) - - # clean up temporary files - if makeclean and returncode == 0: - clean(target, intelwin, verbose) - else: - msg = ( - "Nothing to do, the srcdir ({}) ".format(srcdir) - + "and/or target ({}) ".format(target) - + "are not specified." - ) - raise ValueError(msg) - - return returncode diff --git a/pymake/pymake_parser.py b/pymake/pymake_parser.py new file mode 100644 index 00000000..26e1d15a --- /dev/null +++ b/pymake/pymake_parser.py @@ -0,0 +1,292 @@ +import argparse + +from .pymake import __description__ + + +def get_arg_dict(): + """Get command line argument dictionary + + Returns + ------- + return : dict + Dictionary of command line argument options + + """ + return { + "srcdir": { + "tag": ("srcdir",), + "help": "Path source directory.", + "default": None, + "choices": None, + "action": None, + }, + "target": { + "tag": ("target",), + "help": "Name of target to create. (can include path)", + "default": None, + "choices": None, + "action": None, + }, + "fc": { + "tag": ("-fc",), + "help": "Fortran compiler to use. (default is gfortran)", + "default": "gfortran", + "choices": ["ifort", "mpiifort", "gfortran", "none"], + "action": None, + }, + "cc": { + "tag": ("-cc",), + "help": "C/C++ compiler to use. (default is gcc)", + "default": "gcc", + "choices": [ + "gcc", + "clang", + "clang++", + "icc", + "icl", + "mpiicc", + "g++", + "cl", + "none", + ], + "action": None, + }, + "arch": { + "tag": ("-ar", "--arch"), + "help": """Architecture to use for Intel and Microsoft + compilers on Windows. (default is intel64)""", + "default": "intel64", + "choices": ["ia32", "ia32_intel64", "intel64"], + "action": None, + }, + "makeclean": { + "tag": ("-mc", "--makeclean"), + "help": """Clean temporary object, module, and source files when + done. (default is False)""", + "default": False, + "choices": None, + "action": "store_true", + }, + "double": { + "tag": ("-dbl", "--double"), + "help": "Force double precision. (default is False)", + "default": False, + "choices": None, + "action": "store_true", + }, + "debug": { + "tag": ("-dbg", "--debug"), + "help": "Create debug version. (default is False)", + "default": False, + "choices": None, + "action": "store_true", + }, + "expedite": { + "tag": ("-e", "--expedite"), + "help": """Only compile out of date source files. + Clean must not have been used on previous build. + (default is False)""", + "default": False, + "choices": None, + "action": "store_true", + }, + "dryrun": { + "tag": ("-dr", "--dryrun"), + "help": """Do not actually compile. Files will be + deleted, if --makeclean is used. + Does not work yet for ifort. (default is False)""", + "default": False, + "choices": None, + "action": "store_true", + }, + "include_subdirs": { + "tag": ("-sd", "--subdirs"), + "help": """Include source files in srcdir subdirectories. + (default is None)""", + "default": None, + "choices": None, + "action": "store_true", + }, + "fflags": { + "tag": ("-ff", "--fflags"), + "help": """Additional Fortran compiler flags. Fortran compiler + flags should be enclosed in quotes and start with a + blank space or separated from the name (-ff or + --fflags) with a equal sign (-ff='-O3'). + (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "cflags": { + "tag": ("-cf", "--cflags"), + "help": """Additional C/C++ compiler flags. C/C++ compiler + flags should be enclosed in quotes and start with a + blank space or separated from the name (-cf or + --cflags) with a equal sign (-cf='-O3'). + (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "syslibs": { + "tag": ("-sl", "--syslibs"), + "help": """Linker system libraries. Linker libraries should be + enclosed in quotes and start with a blank space or + separated from the name (-sl or --syslibs) with a + equal sign (-sl='-libgcc'). (default is None)""", + "default": None, + "choices": ["-lc", "-lm"], + "action": None, + }, + "makefile": { + "tag": ("-mf", "--makefile"), + "help": "Create a GNU make makefile. (default is False)", + "default": False, + "choices": None, + "action": "store_true", + }, + "srcdir2": { + "tag": ("-cs", "--commonsrc"), + "help": """Additional directory with common source files. + (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "extrafiles": { + "tag": ("-ef", "--extrafiles"), + "help": """List of extra source files to include in the + compilation. extrafiles can be either a list of files + or the name of a text file that contains a list of + files. (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "excludefiles": { + "tag": ("-exf", "--excludefiles"), + "help": """List of extra source files to exclude from the + compilation. excludefiles can be either a list of + files or the name of a text file that contains a list + of files. (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "sharedobject": { + "tag": ("-so", "--sharedobject"), + "help": """Create shared object or dll on Windows. + (default is False)""", + "default": False, + "choices": None, + "action": "store_true", + }, + "appdir": { + "tag": ("-ad", "--appdir"), + "help": """Target path that overides path defined target + path (default is None)""", + "default": None, + "choices": None, + "action": None, + }, + "verbose": { + "tag": ("-v", "--verbose"), + "help": "Verbose output to terminal. (default is False)", + "default": False, + "choices": None, + "action": "store_true", + }, + "keep": { + "tag": ("--keep",), + "help": "Keep existing executable. (default is False)", + "default": False, + "choices": None, + "action": "store_true", + }, + "zip": { + "tag": ("--zip",), + "help": "Zip built executable. (default is False)", + "default": None, + "choices": None, + "action": None, + }, + "inplace": { + "tag": ("--inplace",), + "help": """Source files in srcdir are used directly. + (default is False)""", + "default": False, + "choices": None, + "action": "store_true", + }, + } + + +def parser_setup(parser_obj, value, reset_default=False): + """Add argument to argparse object + + Parameters + ---------- + parser_obj : object + argparse object + value : dict + argparse settings + reset_default : bool + boolean that defines if default values should be used + + Returns + ------- + parser_obj : object + updated argparse object + + """ + if reset_default: + default = None + else: + default = value["default"] + if value["action"] is None: + parser_obj.add_argument( + *value["tag"], + help=value["help"], + default=default, + choices=value["choices"], + ) + else: + parser_obj.add_argument( + *value["tag"], + help=value["help"], + default=default, + action=value["action"], + ) + return parser_obj + + +def parser(): + """Construct the parser and return argument values. + + Parameters + ---------- + + Returns + ------- + args : list + command line argument list + + """ + description = __description__ + parser_obj = argparse.ArgumentParser( + description=description, + epilog="""Note that the source directory + should not contain any bad or duplicate + source files as all source files in the + source directory, the common source file + directory (srcdir2), and the extra files + (extrafiles) will be built and linked. + Files can be excluded by using the + excludefiles command line switch.""", + ) + + for _, value in get_arg_dict().items(): + my_parser = parser_setup(parser_obj, value) + parser_args = my_parser.parse_args() + return parser_args diff --git a/pymake/usgsprograms.py b/pymake/usgsprograms.py index df850f01..75f010df 100644 --- a/pymake/usgsprograms.py +++ b/pymake/usgsprograms.py @@ -16,7 +16,7 @@ class dotdict(dict): program_data_file = "usgsprograms.txt" # keys to create for each target -target_keys = [ +target_keys = ( "version", "current", "url", @@ -24,7 +24,7 @@ class dotdict(dict): "srcdir", "standard_switch", "double_switch", -] +) def str_to_bool(s): @@ -87,7 +87,19 @@ def _build_usgs_database(self): return dotdict(program_data) def _target_data(self, key): - """Get the dictionary for the target key.""" + """Get the dictionary for the target key. + + Parameters + ---------- + key : str + Program key (name) + + Returns + ------- + return : dict + dictionary with attributes for program key (name) + + """ if key not in self._program_dict: msg = '"{}" key does not exist. Available keys: '.format(key) for idx, k in enumerate(self._program_dict.keys()): @@ -98,7 +110,20 @@ def _target_data(self, key): return self._program_dict[key] def _target_keys(self, current=False): - """Get the target keys.""" + """Get the target keys. + + Parameters + ---------- + current : bool + boolean indicating if only current program versions should be + returned. (default is False) + + Returns + ------- + keys : list + list containing program keys (names) + + """ if current: keys = [ key @@ -116,7 +141,7 @@ def get_target(key): Parameters ---------- key : str - Target USGS program + Target USGS program that may have a path and an extension Returns ------- @@ -124,8 +149,12 @@ def get_target(key): Dictionary with USGS program attributes for the specified key """ - if ".exe" in key.lower(): - key = key[:-4] + # remove path and extension from key + key = os.path.basename(key) + if key.endswith(".exe") or key.endswith(".dll") or key.endswith(".so"): + key = os.path.splitext(key)[0] + + # return program attributes return usgs_program_data()._target_data(key) @staticmethod diff --git a/pymake/usgsprograms.txt b/pymake/usgsprograms.txt index 8b6e6d1a..9883c3c7 100644 --- a/pymake/usgsprograms.txt +++ b/pymake/usgsprograms.txt @@ -8,6 +8,7 @@ mfusg 1.5 True https://water.usgs.gov/water-resources/software/MO zonbudusg 1.5 True https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip mfusg1_5 src/zonebudusg True False mf6 6.1.1 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.1.zip mf6.1.1 src True False zbud6 6.1.1 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.1.zip mf6.1.1 utils/zonebudget/src True False +libmf6 6.1.1 True https://water.usgs.gov/water-resources/software/MODFLOW-6/mf6.1.1.zip mf6.1.1 srcbmi True False swtv4 4.00.05 True https://water.usgs.gov/water-resources/software/SEAWAT/swt_v4_00_05.zip swt_v4_00_05 source False True mp6 6.0.1 True https://water.usgs.gov/water-resources/software/MODPATH/modpath.6_0_01.zip modpath.6_0 src True False mp7 7.2.001 True https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip modpath_7_2_001 source True False diff --git a/pymake/visualize.py b/pymake/visualize.py index 17512958..36795f65 100644 --- a/pymake/visualize.py +++ b/pymake/visualize.py @@ -1,6 +1,6 @@ import os -from .compiler_language_files import get_ordered_srcfiles +from .compiler_language_files import get_srcfiles, get_ordered_srcfiles from .dag import get_f_nodelist try: @@ -140,7 +140,7 @@ def make_plots( ) raise ModuleNotFoundError(msg) - srcfiles = get_ordered_srcfiles(srcdir, include_subdir) + srcfiles = get_ordered_srcfiles(get_srcfiles(srcdir, include_subdir)) nodelist = get_f_nodelist(srcfiles) for idx, n in enumerate(nodelist): if verbose: diff --git a/setup.py b/setup.py index edd39b3b..3f9653e7 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ url='https://github.com/modflowpy/pymake.git', license='New BSD', platforms='Windows, Mac OS-X, Linux', - install_requires=[], # ['pydotplus>=2.0'], + install_requires=['requests'], # ['pydotplus>=2.0'], packages=['pymake'], include_package_data=True, version=__version__)