diff --git a/.gitignore b/.gitignore index 6536099..81a581c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Prerequisites -*.d +# *.d # Compiled Object files *.slo @@ -12,19 +12,18 @@ *.pch # Compiled Dynamic libraries -*.so -*.dylib -*.dll +# *.so +# *.dylib +# *.dll # Fortran module files *.mod *.smod # Compiled Static libraries -*.lai -*.la -*.a -*.lib +# *.lai +# *.la +# *.lib # Executables *.exe diff --git a/Dockerfile b/Dockerfile index 7683009..f7a9d42 100755 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN apt-get update && \ libcgal-dev \ libmpc-dev \ # ping \ + nano \ libstdc++5 \ libiberty-dev \ software-properties-common ;\ @@ -45,27 +46,31 @@ RUN apt-get update && \ WORKDIR /root/ -#RUN mkdir -p /root/local ;\ -# cd /root/local \ -# cd /root/ ;\ -# pwd ;\ -# git clone https://github.com/KarypisLab/GKlib.git +# Download and install GKlib, metis and parmetis dependencies -#RUN make -C /root/GKlib prefix=/root/local config -#RUN cd /root/GKlib && make install +RUN mkdir -p /root/local ;\ + cd /root/ ;\ + git clone https://github.com/KarypisLab/GKlib.git -#RUN cd /root/ ;\ -# git clone https://github.com/KarypisLab/METIS.git -#RUN make -C /root/METIS config shared=1 cc=gcc prefix=/root/local -#RUN cd /root/METIS && make install +RUN make -C /root/GKlib config cc=gcc prefix=/root/local +RUN cd /root/GKlib && make install +RUN cd /root/ ;\ + git clone https://github.com/KarypisLab/METIS.git +RUN make -C /root/METIS config cc=gcc prefix=/root/local +RUN cd /root/METIS && make install -#RUN cd /root/ ;\ -# git clone https://github.com/KarypisLab/ParMETIS.git -#RUN make -C /root/ParMETIS config cc=mpicc prefix=/root/local -#RUN cd /root/ParMETIS && make install +RUN cd /root/ ;\ + git clone https://github.com/KarypisLab/ParMETIS.git +RUN make -C /root/ParMETIS config cc=mpicc prefix=/root/local +RUN cd /root/ParMETIS && make install + +# Add src and compile ADD src /root/CODE -RUN cd /root/CODE && make -f Makefile_docker clean all +# Add tecplot dependency +# TODO build the download and compilation of libtecio.a +ADD bin/lib/tecplot/libtecio.a /root/CODE +RUN cd /root/CODE && make -f Makefile_docker all ENV OMPI_MCA_btl_vader_single_copy_mechanism=none # Make & set a rundir & copy executable @@ -83,3 +88,4 @@ WORKDIR /ucns3d_run COPY tests/taylor_green_vortex/* /ucns3d_run/ COPY tests/execute-tests.sh /ucns3d_run/ RUN chmod +x execute-tests.sh +RUN sed -i -e 's/\r$//' /ucns3d_run/execute-tests.sh diff --git a/README.md b/README.md index 1dbc0a5..a74d24f 100755 --- a/README.md +++ b/README.md @@ -40,19 +40,19 @@ There are two main methods to locally install the solver and run the solver. 2. In your bash terminal build the ucns3d image, you would need invoke `docker build` from the repository root directory: ``` -$ docker build . -t ucns3d -f Dockerfile +docker build . -t ucns3d -f Dockerfile ``` 3. Once the image is build run the image, you can run the image interactively like so: ``` -$ docker run -ti ucns3d +docker run -ti ucns3d ``` The current [Dockerfile](Dockerfile) contains an example case under [tests](/tests/execute-tests.sh). Alternatively, you can mount a tmp directory and copy other uses cases when you run the image like so: ``` -$ docker run -v $PWD/tmp/:/tmp/ -ti ucns3d +docker run -v $PWD/tmp/:/tmp/ -ti ucns3d ``` ### Manual Build @@ -86,11 +86,11 @@ compiler options. * For a clean installation ``` -$ make -f Makefile clean all +make -f Makefile clean all ``` * For recompiling changed files and their dependencies ``` -$ make -f Makefile +make -f Makefile ``` * the name of the executable is `ucns3d_p`. @@ -104,12 +104,12 @@ For running ucns3d you will need the following files in a directory of your choi * the executable ucns3d_p * For interactively running the code specify the number of threads to be used by typing in the terminal window ``` -$ export OMP_NUM_THREADS=N +export OMP_NUM_THREADS=N ``` N being the number of threads to be used (use 1 for MPI only mode) * in the same terminal window run the code by typing ``` -$ mpirun -np M ./ucns3d_p +mpirun -np M ./ucns3d_p ``` M being the number of MPI processes (at least 2 are required), for running at different HPC systems sample [scripts](/scripts) and [libraries](/bin/lib) are provided. diff --git a/bin/gnu-compiler/Makefile b/bin/gnu-compiler/Makefile index 78b0de8..349fe67 100755 --- a/bin/gnu-compiler/Makefile +++ b/bin/gnu-compiler/Makefile @@ -52,7 +52,7 @@ LIBS = -Wl,-Bstatic libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -Wl,-Bdynami # LIBS = libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -lstdc++ -include Makefile.common +include Makefile_common diff --git a/bin/lib/archer/Makefile_archer b/bin/gnu-compiler/archer/Makefile_archer similarity index 98% rename from bin/lib/archer/Makefile_archer rename to bin/gnu-compiler/archer/Makefile_archer index 3f19dab..86d7586 100755 --- a/bin/lib/archer/Makefile_archer +++ b/bin/gnu-compiler/archer/Makefile_archer @@ -50,7 +50,7 @@ LIBS = -Wl,-Bstatic libparmetis_gnu_mpi.a libmetis.a libtecio.a $(BLASLIBS) -Wl, #LIBS = libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -lstdc++ -include Makefile.common +include Makefile_common diff --git a/bin/gnu-compiler/archer/README.md b/bin/gnu-compiler/archer/README.md new file mode 100755 index 0000000..e0edfce --- /dev/null +++ b/bin/gnu-compiler/archer/README.md @@ -0,0 +1,3 @@ +# Setup for archer Supercomputer + + These are libraries that work with gnu and cray ftn compilers on archer. diff --git a/bin/gnu-compiler/docker-ubuntu/Makefile_docker b/bin/gnu-compiler/docker-ubuntu/Makefile_docker new file mode 100755 index 0000000..b79fbfb --- /dev/null +++ b/bin/gnu-compiler/docker-ubuntu/Makefile_docker @@ -0,0 +1,62 @@ +## Makefile to be build in Docker. + +##intel +# F90=mpiifort +#GFORTRAN +F90=mpif90 +LD=$(F90) +# ifdef HPCTOOLKIT +# $(info HPCToolkit is loaded - building version for profiling) +# DBGFLAGS=-g -debug inline-debug-info +# else +# $(info HPCToolkit is not loaded - building release version) +# DBGFLAGS= +# endif + + +#intel MKL +# BLASFLAGS= -I${MKLROOT}/include +# BLASLIBS= -Wl,--start-group ${MKLROOT}/lib/intel64/libmkl_intel_lp64.a ${MKLROOT}/lib/intel64/libmkl_sequential.a ${MKLROOT}/lib/intel64/libmkl_core.a -Wl,--end-group -lpthread -lm -ldl + +BLASFLAGS= -I /usr/include/x86_64-linux-gnu/openblas-pthread #Ensure that you include your BLAS library path here +BLASLIBS= /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.a #Ensure that you include your BLAS library path here +GKLIB= /root/local/lib/libGKlib.a +METISLIB= /root/local/lib/libmetis.a +PARMETISLIB= /root/local/lib/libparmetis.a + + +#amdblis aocl +#BLASFLAGS= -I/opt/hlrs/non-spack/libraries/aocl-gcc/2.1.0/amd-blis/include/ +#BLASLIBS= /opt/hlrs/non-spack/libraries/aocl-gcc/2.1.0/amd-blis/lib/libblis.a + +# OFLAGS= +# #intel compilers +# # debug run +# #FFLAGS=-i4 -r8 -O0 -g -debug all -xHost -traceback -qopenmp -qopenmp-link=static $(BLASFLAGS) -check bounds -fp-model strict -debug inline-debug-info -zero +# # production run +# #FFLAGS=-i4 -r8 -ipo -march=core-avx2 -mtune=core-avx2 -O3 -fp-model precise -zero -qopenmp -qopenmp-link=static $(BLASFLAGS) #amd zen +# FFLAGS=-i4 -r8 -ipo -xHost -O3 -fp-model precise -zero -qopenmp -qopenmp-link=static $(BLASFLAGS) + + +#gfortran +#debug run +#FFLAGS=-ffree-line-length-none -fdefault-real-8 -fopenmp -Wall -fbounds-check -finit-local-zero -fdump-parse-tree -fdump-core -fbacktrace -fdefault-double-8 -fbackslash -O0 -fcray-pointer -Wno-lto-type-mismatch -fPIE $(BLASFLAGS) +#production run +FFLAGS=-fdefault-real-8 -fdefault-double-8 -fbackslash -fopenmp -ffree-line-length-none -finit-local-zero -fimplicit-none -flto -fcray-pointer -O3 -march=native -Wno-lto-type-mismatch -fPIE $(BLASFLAGS) + +#aocl +#FFLAGS= -O3 -Mfreeform -Mallocatable=03 -Mdaz -fdefault-real-8 -fopenmp -mavx2 -march=znver2 $(BLASFLAGS) +#FFLAGS= -O2 -mp -fdefault-real-8 -flto -fopenmp -mavx2 -march=znver2 $(BLASFLAGS) + +#gnu hybrid static dynamic +# LIBS = -Wl,-Bstatic libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -Wl,-Bdynamic -lstdc++ -lpthread -lm -ldl -lc -lmpi +LIBS = -Wl,-V,-Bstatic libtecio.a $(BLASLIBS) $(GKLIB) $(METISLIB) $(PARMETISLIB) -Wl,-Bdynamic -lstdc++ -lpthread -lm -ldl -lc -lmpi + + +#intel static linking +# LIBS = libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -lstdc++ + +include Makefile_common + + + diff --git a/bin/intel-compiler/Makefile b/bin/intel-compiler/Makefile index 06e4ac3..06ce789 100755 --- a/bin/intel-compiler/Makefile +++ b/bin/intel-compiler/Makefile @@ -50,7 +50,7 @@ FFLAGS=-i4 -r8 -ipo -xHost -O3 -fp-model precise -zero -qopenmp -qopenmp-link=s LIBS = libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -lstdc++ -include Makefile.common +include Makefile_common diff --git a/bin/lib/README.md b/bin/lib/README.md new file mode 100644 index 0000000..90a326e --- /dev/null +++ b/bin/lib/README.md @@ -0,0 +1,3 @@ +# Static libraries required to compile and run UCNS3D + +Precompiled metis, parmetis and tecplot static libraries. They can be used to link during compilation. diff --git a/bin/lib/archer/libparmetis.a b/bin/lib/archer/libparmetis.a deleted file mode 100644 index ac4db1a..0000000 Binary files a/bin/lib/archer/libparmetis.a and /dev/null differ diff --git a/bin/lib/archer/readme.txt b/bin/lib/archer/readme.txt deleted file mode 100755 index 816952d..0000000 --- a/bin/lib/archer/readme.txt +++ /dev/null @@ -1 +0,0 @@ -These are libraries that work with gnu and cray ftn compilers on archer. diff --git a/bin/lib/libmetis.a b/bin/lib/libmetis.a deleted file mode 100644 index 995863a..0000000 Binary files a/bin/lib/libmetis.a and /dev/null differ diff --git a/bin/lib/libtecio.a b/bin/lib/libtecio.a deleted file mode 100644 index 03e208b..0000000 Binary files a/bin/lib/libtecio.a and /dev/null differ diff --git a/bin/lib/archer/libmetis.a b/bin/lib/metis/libmetis.a similarity index 100% rename from bin/lib/archer/libmetis.a rename to bin/lib/metis/libmetis.a diff --git a/src/mpmetis b/bin/lib/metis/mpmetis similarity index 100% rename from src/mpmetis rename to bin/lib/metis/mpmetis diff --git a/bin/lib/libparmetis.a b/bin/lib/parmetis/libparmetis.a similarity index 100% rename from bin/lib/libparmetis.a rename to bin/lib/parmetis/libparmetis.a diff --git a/bin/lib/archer/libtecio.a b/bin/lib/tecplot/libtecio.a similarity index 100% rename from bin/lib/archer/libtecio.a rename to bin/lib/tecplot/libtecio.a diff --git a/bin/macos/readme.txt b/bin/macos/README.md similarity index 56% rename from bin/macos/readme.txt rename to bin/macos/README.md index 7b9cb9d..1dc72fc 100755 --- a/bin/macos/readme.txt +++ b/bin/macos/README.md @@ -1,4 +1,6 @@ -. Use the following dynamic libraries: +# Setup on MacOS + +Use the following dynamic libraries: libmetis.dylib (Compiled from source) libparmetis.dylib (Compiled from source) libtecio.dylib (Copied from Tecplot executable) @@ -7,37 +9,49 @@ The above are also provided in this folder but probably won't work, you will hav -------------------------------------------------- Locate the required libraries: -1) Download and install the latest free trial of tecpot, then the dynamic library (libtecio.dylib), is available in 360’s folder, Tecplot 360 EX 2021 r1 /Contents/Frameworks/ (you can run ucns3d without tecplot, and you can use a vtk output but this is required for compiling only) +1. Download and install the latest free trial of tecpot, then the dynamic library (libtecio.dylib), is available in 360’s folder, Tecplot 360 EX 2021 r1 /Contents/Frameworks/ (you can run ucns3d without tecplot, and you can use a vtk output but this is required for compiling only) -2) Download the latest version of parmetis and in the CMakeLists.txt of the parmetis directory, in the section starting with: if(SHARED) you should add: +2. Download the latest version of parmetis and in the CMakeLists.txt of the parmetis directory, in the section starting with: if(SHARED) you should add: set(METIS_LIBRARY_TYPE SHARED) -a. Then open a terminal window in the parmetis directory and execute the following commands -i. make config shared=1 -ii. make -b. Then you can use the generated libmetis.dylib and libparmetis.dylib from your directory parmetis/build/Darwin…./libmetis/ and parmetis/build/Darwin…./libparmetis/ respectively + 2.1. Then open a terminal window in the parmetis directory and execute the following commands + +``` +make config shared=1 +``` +``` +make +``` + +2.2. Then you can use the generated libmetis.dylib and libparmetis.dylib from your directory parmetis/build/Darwin…./libmetis/ and parmetis/build/Darwin…./libparmetis/ respectively -3) Install OpenBlas with Homebrew and modify the Makefile to point to the correct location of the libraries. +3. Install OpenBlas with Homebrew and modify the Makefile to point to the correct location of the libraries. --------------------------------------------------------- +4. Use the Makefile.common and Makefile for MacOS from this folder and copy to the src directory -2. Use the Makefile.common and Makefile for MacOS from this folder and copy to the src directory +5. Open a terminal window and compile as: -3. Open a terminal window and compile as: +``` make -f Makefile clean all (this is for new make and clean) -or -make -f Makefile +``` +``` +make -f Makefile +``` -4. Type the following in a terminal window prior to running the application +6. Type the following in a terminal window prior to running the application +``` install_name_tool -change @rpath/libtecio.dylib /Users/Username/code_directory/libtecio.dylib /Users/Username/executable_directory/executable_name +``` - -5. Type the following in a terminal window (values for A greater than or equal to 1, and B greater than 1) +7. Type the following in a terminal window (values for A greater than or equal to 1, and B greater than 1) +``` export OMP_NUM_THREADS=A +``` +``` mpirun -np B ./ucns3d_p +``` -6. Enjoy! diff --git a/scripts/ARCHER_IB.pbs b/scripts/submission-script/ARCHER_IB.pbs similarity index 100% rename from scripts/ARCHER_IB.pbs rename to scripts/submission-script/ARCHER_IB.pbs diff --git a/scripts/ARCHER_KNL.pbs b/scripts/submission-script/ARCHER_KNL.pbs similarity index 100% rename from scripts/ARCHER_KNL.pbs rename to scripts/submission-script/ARCHER_KNL.pbs diff --git a/scripts/DELTA_BW.pbs b/scripts/submission-script/DELTA_BW.pbs similarity index 100% rename from scripts/DELTA_BW.pbs rename to scripts/submission-script/DELTA_BW.pbs diff --git a/scripts/XEON_PHI_KNL.peta4-knl b/scripts/submission-script/XEON_PHI_KNL.peta4-knl similarity index 100% rename from scripts/XEON_PHI_KNL.peta4-knl rename to scripts/submission-script/XEON_PHI_KNL.peta4-knl diff --git a/scripts/archer2.slurm b/scripts/submission-script/archer2.slurm similarity index 100% rename from scripts/archer2.slurm rename to scripts/submission-script/archer2.slurm diff --git a/scripts/delta2.sub b/scripts/submission-script/delta2.sub similarity index 100% rename from scripts/delta2.sub rename to scripts/submission-script/delta2.sub diff --git a/bin/translators/README.txt b/scripts/translators/README.md similarity index 64% rename from bin/translators/README.txt rename to scripts/translators/README.md index 7004280..08ceee5 100755 --- a/bin/translators/README.txt +++ b/scripts/translators/README.md @@ -1,8 +1,10 @@ +# Scripts to translate input formats + these translators can be compiled with the following command to translate meshes generated in STAR CD format or UGRID format to native UCNS3D binary format -compile with - -ifort -i4 -r8 "name of .f90 file" +compile with and run -execute +``` +ifort -i4 -r8 ".f90" +``` diff --git a/bin/translators/STAR2NAT.f90 b/scripts/translators/STAR2NAT.f90 similarity index 100% rename from bin/translators/STAR2NAT.f90 rename to scripts/translators/STAR2NAT.f90 diff --git a/bin/translators/UGRID2NAT.f90 b/scripts/translators/UGRID2NAT.f90 similarity index 100% rename from bin/translators/UGRID2NAT.f90 rename to scripts/translators/UGRID2NAT.f90 diff --git a/src/Makefile.common b/src/Makefile_common similarity index 100% rename from src/Makefile.common rename to src/Makefile_common diff --git a/src/Makefile_docker b/src/Makefile_docker index 8e76843..1a31162 100755 --- a/src/Makefile_docker +++ b/src/Makefile_docker @@ -1,57 +1,19 @@ -## Makefile to be build in Docker. +# Makefile to be build with Docker. -##intel -# F90=mpiifort -#GFORTRAN F90=mpif90 LD=$(F90) -# ifdef HPCTOOLKIT -# $(info HPCToolkit is loaded - building version for profiling) -# DBGFLAGS=-g -debug inline-debug-info -# else -# $(info HPCToolkit is not loaded - building release version) -# DBGFLAGS= -# endif - - -#intel MKL -# BLASFLAGS= -I${MKLROOT}/include -# BLASLIBS= -Wl,--start-group ${MKLROOT}/lib/intel64/libmkl_intel_lp64.a ${MKLROOT}/lib/intel64/libmkl_sequential.a ${MKLROOT}/lib/intel64/libmkl_core.a -Wl,--end-group -lpthread -lm -ldl BLASFLAGS= -I /usr/include/x86_64-linux-gnu/openblas-pthread #Ensure that you include your BLAS library path here BLASLIBS= /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.a #Ensure that you include your BLAS library path here -#PARMETISLIBS= /root/local/lib/libparmetis.a - -#amdblis aocl -#BLASFLAGS= -I/opt/hlrs/non-spack/libraries/aocl-gcc/2.1.0/amd-blis/include/ -#BLASLIBS= /opt/hlrs/non-spack/libraries/aocl-gcc/2.1.0/amd-blis/lib/libblis.a - -# OFLAGS= -# #intel compilers -# # debug run -# #FFLAGS=-i4 -r8 -O0 -g -debug all -xHost -traceback -qopenmp -qopenmp-link=static $(BLASFLAGS) -check bounds -fp-model strict -debug inline-debug-info -zero -# # production run -# #FFLAGS=-i4 -r8 -ipo -march=core-avx2 -mtune=core-avx2 -O3 -fp-model precise -zero -qopenmp -qopenmp-link=static $(BLASFLAGS) #amd zen -# FFLAGS=-i4 -r8 -ipo -xHost -O3 -fp-model precise -zero -qopenmp -qopenmp-link=static $(BLASFLAGS) +GKLIB= /root/local/lib/libGKlib.a +METISLIB= /root/local/lib/libmetis.a +PARMETISLIB= /root/local/lib/libparmetis.a - -#gfortran -#debug run -#FFLAGS=-ffree-line-length-none -fdefault-real-8 -fopenmp -Wall -fbounds-check -finit-local-zero -fdump-parse-tree -fdump-core -fbacktrace -fdefault-double-8 -fbackslash -O0 -fcray-pointer -Wno-lto-type-mismatch -fPIE $(BLASFLAGS) -#production run FFLAGS=-fdefault-real-8 -fdefault-double-8 -fbackslash -fopenmp -ffree-line-length-none -finit-local-zero -fimplicit-none -flto -fcray-pointer -O3 -march=native -Wno-lto-type-mismatch -fPIE $(BLASFLAGS) -#aocl -#FFLAGS= -O3 -Mfreeform -Mallocatable=03 -Mdaz -fdefault-real-8 -fopenmp -mavx2 -march=znver2 $(BLASFLAGS) -#FFLAGS= -O2 -mp -fdefault-real-8 -flto -fopenmp -mavx2 -march=znver2 $(BLASFLAGS) - -#gnu hybrid static dynamic -LIBS = -Wl,-Bstatic libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -Wl,-Bdynamic -lstdc++ -lpthread -lm -ldl -lc -lmpi - -#intel static linking -# LIBS = libparmetis.a libmetis.a libtecio.a $(BLASLIBS) -lstdc++ +LIBS = -Wl,-V,-Bstatic libtecio.a $(BLASLIBS) $(GKLIB) $(METISLIB) $(PARMETISLIB) -Wl,-Bdynamic -lstdc++ -lpthread -lm -ldl -lc -lmpi -include Makefile.common +include Makefile_common diff --git a/src/BUBBLES.DAT b/tests/bubbles/BUBBLES.DAT similarity index 100% rename from src/BUBBLES.DAT rename to tests/bubbles/BUBBLES.DAT diff --git a/src/UCNS3D.DAT b/tests/default-config/UCNS3D.DAT similarity index 100% rename from src/UCNS3D.DAT rename to tests/default-config/UCNS3D.DAT diff --git a/src/MOOD.DAT b/tests/mood/MOOD.DAT similarity index 100% rename from src/MOOD.DAT rename to tests/mood/MOOD.DAT diff --git a/src/MULTISPECIES.DAT b/tests/multispecies/MULTISPECIES.DAT similarity index 100% rename from src/MULTISPECIES.DAT rename to tests/multispecies/MULTISPECIES.DAT