diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2db9de2257..4eb524250b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,11 +3,13 @@ * @sbryngelson # Henry Le Berre (henryleberre) +docs/ @sbryngelson @henryleberre toolchain/ @sbryngelson @henryleberre tests/ @sbryngelson @henryleberre mfc.sh @sbryngelson @henryleberre mfc.bat @sbryngelson @henryleberre -defaults.yaml @sbryngelson @henryleberre CMakeLists.txt @sbryngelson @henryleberre .gitlab-ci.yml @sbryngelson @henryleberre +.vscode/ @sbryngelson @henryleberre .github/workflows/ @sbryngelson @henryleberre + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cffe22126..8472dc0c8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,74 +1,52 @@ -name: Continuous Integration - -on: - push: - paths: - - '**.f90' - - '**.fpp' - - 'mfc.sh' - - '**.py' - - '**.yaml' - - '**.yml' - - 'golden.txt' - - 'Makefile' - - 'makefile' - - 'CMakeLists.txt' - - Dockerfile - - pull_request: - - workflow_dispatch: - -jobs: - tests: - name: CI Test Runner - strategy: - matrix: - include: - - os: ubuntu-latest - unix: true - - - os: macos-latest - unix: true - gcc: 11 - runs-on: ${{ matrix.os }} - steps: - - name: Clone MFC - uses: actions/checkout@v3 - - - name: (macos-latest) Setup - if: matrix.os == 'macos-latest' - run: | - echo "OMPI_FC=gfortran-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "OMPI_CXX=g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "OMPI_MPICC=gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CC=gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "FC=gfortran-${{ matrix.gcc }}" >> $GITHUB_ENV - brew install wget make python make cmake coreutils gcc@${{ matrix.gcc }} - HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-${{ matrix.gcc }} --verbose --build-from-source open-mpi - - - name: (ubuntu-latest) Setup - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update && sudo apt upgrade - sudo apt install tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev - - - name: (${{ matrix.os }}) Test MFC - run: | - chmod +x ./mfc.sh - ./mfc.sh test -j $(nproc) - - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ matrix.os }} - path: | - build/**/*.log - build/**/*.txt - tests/**/*.dat - tests/**/*.out - tests/**/*.inp - tests/**/*.txt - tests/*.txt +name: Continuous Integration + +on: + push: + paths: + - '**.f90' + - '**.fpp' + - '**.py' + - '**.yml' + - 'mfc.sh' + - 'golden.txt' + - 'CMakeLists.txt' + + pull_request: + + workflow_dispatch: + +jobs: + tests: + name: Test Suite + strategy: + matrix: + include: + - os: ubuntu-latest + - os: macos-latest + gcc: 11 + runs-on: ${{ matrix.os }} + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Setup + if: matrix.os == 'macos-latest' + run: | + echo "OMPI_FC=gfortran-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "OMPI_CXX=g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "OMPI_MPICC=gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CC=gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "FC=gfortran-${{ matrix.gcc }}" >> $GITHUB_ENV + brew install wget make python make cmake coreutils gcc@${{ matrix.gcc }} + HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-${{ matrix.gcc }} --verbose --build-from-source open-mpi + + - name: Setup + if: matrix.os == 'ubuntu-latest' + run: sudo apt install tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev + + - name: Build + run: /bin/bash mfc.sh build -j $(nproc) + + - name: Test Suite + run: /bin/bash mfc.sh test -j $(nproc) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml deleted file mode 100644 index fd4e4933ac..0000000000 --- a/.github/workflows/doc.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Documentation - -on: - push: - branches: - - master - - workflow_dispatch: - -jobs: - doc: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Build Documentation - run: | - sudo apt install cmake ninja-build doxygen graphviz - pip3 install fypp - mkdir -p build/install - cd build - cmake -GNinja -DCMAKE_INSTALL_PREFIX=$(pwd)/install -DMFC_BUILD_DOC=ON .. - ninja install - - - name: Publish Documentation - run: | - set +e - cd build - git ls-remote "${{ secrets.DOC_PUSH_URL }}" -q - code=$? - if [ "$code" -ne "0" ]; then exit 0; fi - git config --global user.name 'MFC Action' - git config --global user.email '<>' - git clone "${{ secrets.DOC_PUSH_URL }}" - cd "$(find . -name '*.github.io')" - mv .github .. - rm -rf $(pwd)/* - mv ../.github . - mv ../install/doc/mfc/* . - git add -A - git commit -m "Docs @ ${GITHUB_SHA::7}" || true - git push - -# DOC_PUSH_URL should be of the format: -# --> https://:@github.com// - diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..49e4325450 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,51 @@ +name: Documentation + +on: + push: + branches: + - master + + workflow_dispatch: + +jobs: + docs: + name: Build & Publish + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + # We build doxygen from source because of + # https://github.com/doxygen/doxygen/issues/9016 + - name: Build Doxygen + run: | + sudo apt install cmake ninja-build graphviz graphviz + git clone https://github.com/doxygen/doxygen.git ../doxygen + cmake -S ../doxygen -B ../doxygen/build -G Ninja + sudo ninja -C ../doxygen/build install + + - name: Build Documentation + run: | + pip3 install fypp + cmake -S . -B build -G Ninja --install-prefix=$(pwd)/build/install -D MFC_DOCUMENTATION=ON + ninja -C build install + + - name: Publish Documentation + run: | + set +e + git ls-remote "${{ secrets.DOC_PUSH_URL }}" -q + if [ "$?" -ne "0" ]; then exit 0; fi + set -e + git config --global user.name 'MFC Action' + git config --global user.email '<>' + git clone "${{ secrets.DOC_PUSH_URL }}" ../www + mv ../www/.github .. + rm -rf ../www/* + mv ../.github ../www/ + mv build/install/docs/mfc/* ../www/ + git -C ../www add -A + git -C ../www commit -m "Docs @ ${GITHUB_SHA::7}" || true + git -C ../www push + +# DOC_PUSH_URL should be of the format: +# --> https://:@github.com// diff --git a/.gitignore b/.gitignore index d6ac28ec25..08a6a6584c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ tags src/*/tags +*.swo +*.swp + +*:Zone.Identifier + .vscode/ .nfs* defaults.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d35496a8f..c268403322 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,7 +63,7 @@ before_script: script: - cd "$CI_MFC_DIR/$JOB_MODE" && echo "[CI] Building in $(pwd):" - . ./mfc.sh load -c a -m $(if [ "$(echo "$JOB_MODE" | grep -i 'cpu' | wc -c)" -gt 0 ]; then echo "c"; else echo "g"; fi) - - ./mfc.sh build -j 8 -m "$JOB_MODE" + - ./mfc.sh build -j 8 $FLAGS tags: - nobatch @@ -72,45 +72,47 @@ before_script: script: - cd "$CI_MFC_DIR/$JOB_MODE" && echo "[CI] Running in $(pwd):" - . ./mfc.sh load -c a -m $(if [ "$(echo "$JOB_MODE" | grep -i 'cpu' | wc -c)" -gt 0 ]; then echo "c"; else echo "g"; fi) - - ./mfc.sh test -j 8 -m "$JOB_MODE" + - ./mfc.sh test -j 8 tags: - batch clone-cpu: variables: - JOB_MODE: "release-cpu" + JOB_MODE: "cpu" extends: .clone build-cpu: variables: - JOB_MODE: "release-cpu" + FLAGS: "" + JOB_MODE: "cpu" extends: .build needs: - clone-cpu test-cpu: variables: - JOB_MODE: "release-cpu" + JOB_MODE: "cpu" extends: .test needs: - build-cpu clone-gpu: variables: - JOB_MODE: "release-gpu" + JOB_MODE: "gpu" extends: .clone build-gpu: variables: - JOB_MODE: "release-gpu" + FLAGS: "--gpu" + JOB_MODE: "gpu" extends: .build needs: - clone-gpu test-gpu: variables: - JOB_MODE: "release-gpu" + JOB_MODE: "gpu" extends: .test needs: - build-gpu diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index bb3500ff3a..0000000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "type": "shell", - "label": "Build MFC", - "command": "./mfc.sh", - "args": [ - "build", - "-j", - "$(nproc)", - "-t", - "${input:target}" - ], - "group": { - "kind": "build", - "isDefault": true - }, - "icon": { - "id": "tools" - }, - "runOptions": { - "instanceLimit": 1 - }, - "problemMatcher": [] - }, - { - "type": "shell", - "label": "Test MFC", - "command": "./mfc.sh", - "args": [ - "test", - "-j", - "$(nproc)" - ], - "group": { - "kind": "test", - "isDefault": true - }, - "icon": { - "id": "run" - }, - "runOptions": { - "instanceLimit": 1 - } - } - ], - "inputs": [ - { - "id": "target", - "type": "pickString", - "description": "MFC Target Slug", - "options": [ - "pre_process", - "simulation", - "post_process" - ] - } - ] -} diff --git a/CMakeLists.txt b/CMakeLists.txt index c1f228d779..757c082a91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,22 +1,24 @@ cmake_minimum_required(VERSION 3.18) -project(MFC LANGUAGES C CXX Fortran) +# Note: If C is not included as a language, +# FIND_LIBRARY_USE_LIB64_PATHS is ignored. +project(MFC LANGUAGES C Fortran) # === Options -option(MFC_WITH_MPI "Build with MPI" ON) -option(MFC_WITH_OPEN_ACC "Build with OpenACC" OFF) -option(MFC_BUILD_PRE_PROCESS "Build pre_process" OFF) -option(MFC_BUILD_SIMULATION "Build simulation" OFF) -option(MFC_BUILD_POST_PROCESS "Build post_process" OFF) -option(MFC_BUILD_DOC "Build documentation" OFF) -option(MFC_BUILD_ALL "Build everything" OFF) +option(MFC_MPI "Build with MPI" ON) +option(MFC_OpenACC "Build with OpenACC" OFF) +option(MFC_PRE_PROCESS "Build pre_process" OFF) +option(MFC_SIMULATION "Build simulation" OFF) +option(MFC_POST_PROCESS "Build post_process" OFF) +option(MFC_DOCUMENTATION "Build documentation" OFF) +option(MFC_ALL "Build everything" OFF) if (MFC_BUILD_ALL) - set(MFC_BUILD_PRE_PROCESS ON FORCE) - set(MFC_BUILD_SIMULATION ON FORCE) - set(MFC_BUILD_POST_PROCESS ON FORCE) - set(MFC_BUILD_DOC ON FORCE) + set(MFC_PRE_PROCESS ON FORCE) + set(MFC_SIMULATION ON FORCE) + set(MFC_POST_PROCESS ON FORCE) + set(MFC_DOCUMENTATION ON FORCE) endif() @@ -24,31 +26,25 @@ endif() include(GNUInstallDirs) include(ExternalProject) include(CheckIPOSupported) +include(CMakeParseArguments) include(CheckFortranCompilerFlag) # === Check Compiler Support & Tools # === === Compiler Support if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 5) message(FATAL_ERROR "GNU v5.0 or newer is required to build MFC. " "[current: ${CMAKE_Fortran_COMPILER_VERSION}]") endif() - -elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR - (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")) - +elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")) if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 21.7) message(FATAL_ERROR "NVHPC v21.7 or newer is required to build MFC. " "[current: ${CMAKE_Fortran_COMPILER_VERSION}]") endif() - elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "AppleClang") - message(FATAL_ERROR "MFC does not support the Apple Clang compilers. " "Please consult the README for more details.") - endif() # === === Find Fypp @@ -68,9 +64,7 @@ list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake") # === Compiler Flags if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - add_compile_options( - -w $<$:-fdefault-real-8> $<$:-ffree-line-length-none> $<$:-fdefault-double-8> @@ -89,22 +83,24 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") -fallow-argument-mismatch ) endif() +elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + add_compile_options( + -h nomessage=296:878:1391:1069 + -h msgs -h static -h keepfiles + ) + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_options(-e D -h func_trace) + endif() elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Flang") - add_compile_options( $<$:-Mfreeform> $<$:-Mpreprocess> $<$:-fdefault-real-8> ) - elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") - add_compile_options($<$:-free>) - -elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR - (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")) - +elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")) add_compile_options( $<$:-r8> $<$:-Mfreeform> @@ -112,13 +108,11 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR $<$:-cpp> -Minfo=accel ) - endif() # === Release Optimizations if (CMAKE_BUILD_TYPE STREQUAL "Release") - # === === Processor tuning CHECK_FORTRAN_COMPILER_FLAG("-march=native" SUPPORTS_MARCH_NATIVE) if (SUPPORTS_MARCH_NATIVE) @@ -134,93 +128,105 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release") # === === Enable LTO/IPO if supported CHECK_IPO_SUPPORTED(RESULT SUPPORTS_IPO OUTPUT IPO_ERROR) if (SUPPORTS_IPO) - message(STATUS "IPO / LTO enabled") + message(STATUS "Enabled IPO / LTO") set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) else() - message(STATUS "IPO / LTO not available") + message(STATUS "IPO / LTO is NOT available") endif() - endif() +## === DEBUG -# This macro locates Fypp files in src// and instructs CMake how to -# preprocess them to generate their .f90 counterparts, should they be requested -# by an executable or library. They are preprocessed again if they are modified -# or if src/common/case.fpp changes because some #:include it, and produce -# a different .f90 file depending on case.fpp's contents. This macro sets the -# _srcs CMake cache variaible the full list of .f90 files, including the -# preprocessed Fypp files. -macro(HANDLE_FYPP target) - file(GLOB ${target}_f90s CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/*.f90") - file(GLOB ${target}_fpps CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/*.fpp") - - # Create src//autogen folder in which to generate Fypp'd .f90 files. - file(MAKE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/autogen") - - set(${target}_srcs "${${target}_f90s}") - - foreach(fpp_filepath IN LISTS ${target}_fpps) - string(REGEX REPLACE ".fpp\$" ".f90" f90_filepath "${fpp_filepath}") - - get_filename_component(fpp_filename "${fpp_filepath}" NAME) - get_filename_component(f90_dirpath "${f90_filepath}" DIRECTORY) - get_filename_component(f90_filename "${f90_filepath}" NAME) - - set(f90_filepath "${f90_dirpath}/autogen/${f90_filename}") +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_definitions(MFC_DEBUG) +endif() - set(depends_on "${fpp_filepath}") - if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/case.fpp") - list(APPEND depends_on "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/case.fpp") +## === HANDLE_SOURCES +# Outputs: +# - _SRCs: List of F90 source filepaths for src/[,common]/ (including .fpp -> .f90) +macro(HANDLE_SOURCES target) + set(${target}_DIR "${CMAKE_SOURCE_DIR}/src/${target}") + set(common_DIR "${CMAKE_SOURCE_DIR}/src/common") + + # Gather src/[,common]/*.f90 + file(GLOB ${target}_F90s CONFIGURE_DEPENDS "${${target}_DIR}/*.f90") + file(GLOB common_F90s CONFIGURE_DEPENDS "${common_DIR}/*.f90") + set(${target}_SRCs ${${target}_F90s} ${common_F90s}) + + # src/[,common]/FPP.fpp -> src//autogen/F90.f90 + file(GLOB ${target}_FPPs CONFIGURE_DEPENDS "${${target}_DIR}/*.fpp") + file(GLOB common_FPPs CONFIGURE_DEPENDS "${common_DIR}/*.fpp") + + file(MAKE_DIRECTORY "${${target}_DIR}/autogen") + foreach(fpp ${${target}_FPPs} ${common_FPPs}) + cmake_path(GET fpp FILENAME fpp_filename) + set(f90 "${${target}_DIR}/autogen/${fpp_filename}.f90") + set(dep "${fpp}") + if (EXISTS "${${target}_DIR}/case.fpp") + list(APPEND dep "${${target}_DIR}/case.fpp") endif() add_custom_command( - OUTPUT "${f90_filepath}" - COMMAND "${FYPP_EXE}" "${fpp_filepath}" "${f90_filepath}" - DEPENDS "${depends_on}" + OUTPUT ${f90} + COMMAND ${FYPP_EXE} -I ${common_DIR} ${fpp} ${f90} + DEPENDS ${dep} COMMENT "Preprocessing (Fypp) ${fpp_filename}" VERBATIM ) - list(APPEND ${target}_srcs "${f90_filepath}") - endforeach(fpp_filepath) - + list(APPEND ${target}_SRCs ${f90}) + endforeach() endmacro() -# === Handle src/ Folder -# === === Fypp / Sources -HANDLE_FYPP(common) -HANDLE_FYPP(pre_process) -HANDLE_FYPP(simulation) -HANDLE_FYPP(post_process) - +HANDLE_SOURCES(pre_process) +HANDLE_SOURCES(simulation) +HANDLE_SOURCES(post_process) # === === src/pre_process -if (MFC_BUILD_PRE_PROCESS) - add_executable(pre_process "${pre_process_srcs}" "${common_srcs}") +if (MFC_PRE_PROCESS) + add_executable(pre_process ${pre_process_SRCs}) - - if (MFC_WITH_MPI) + if (MFC_MPI) find_package(MPI COMPONENTS Fortran REQUIRED) target_compile_definitions(pre_process PRIVATE MFC_MPI) target_link_libraries (pre_process PRIVATE MPI::MPI_Fortran) endif() + if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + target_compile_options(pre_process PRIVATE -h noacc -x acc) + endif() - target_compile_options(pre_process PRIVATE -DMFC_PRE_PROCESS) - + target_compile_definitions(pre_process PRIVATE MFC_PRE_PROCESS) install(TARGETS pre_process RUNTIME DESTINATION bin) endif() # === === src/simulation -if (MFC_BUILD_SIMULATION) - add_executable(simulation "${simulation_srcs}" "${common_srcs}") +if (MFC_SIMULATION) + add_executable(simulation ${simulation_SRCs}) + + if (MFC_MPI) + find_package(MPI COMPONENTS Fortran REQUIRED) + + target_compile_definitions(simulation PRIVATE MFC_MPI) + target_link_libraries (simulation PRIVATE MPI::MPI_Fortran) + endif() + # === === === FFTW + if (MFC_OpenACC AND (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")) + find_package(CUDAToolkit REQUIRED) - if (MFC_WITH_OPEN_ACC) + target_link_libraries(simulation PRIVATE CUDA::cudart CUDA::cufft) + else() + find_package(FFTW REQUIRED) + + target_link_libraries(simulation PRIVATE FFTW::FFTW) + endif() + + if (MFC_OpenACC) find_package(OpenACC) # This should be equivalent to if (NOT OpenACC_FC_FOUND) @@ -230,30 +236,23 @@ if (MFC_BUILD_SIMULATION) target_link_libraries(simulation PRIVATE OpenACC::OpenACC_Fortran) - - if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI") - find_package(CUDAToolkit REQUIRED) - - target_link_libraries(simulation - PRIVATE CUDA::cudart - CUDA::cufft - ) - - + if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + target_compile_options(simulation PRIVATE + "-foffload=amdgcn-amdhsa='-march=gfx90a'" + "-foffload-options=-lgfortran\ -lm" + "-fno-exceptions") + elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI") find_package(cuTENSOR) - if (NOT cuTENSOR_FOUND) message(WARNING "Failed to locate the NVIDIA cuTENSOR library. MFC will be " "built without support for it, disallowing the use of " "cu_tensor=T. This can result in degraded performance.") else() - target_link_libraries(simulation PRIVATE cuTENSOR::cuTENSOR) - + target_link_libraries (simulation PRIVATE cuTENSOR::cuTENSOR) target_compile_definitions(simulation PRIVATE MFC_cuTENSOR) endif() - if (CMAKE_BUILD_TYPE STREQUAL "Release") target_compile_options(simulation PRIVATE -gpu=keep,ptxinfo,lineinfo @@ -263,40 +262,25 @@ if (MFC_BUILD_SIMULATION) PRIVATE -gpu=keep,ptxinfo,lineinfo,autocompare,debug ) endif() - endif() - else() - find_package(FFTW REQUIRED) - - target_link_libraries(simulation PRIVATE FFTW::FFTW) + elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + target_compile_options(simulation PRIVATE -h noacc -x acc) endif() - - if (MFC_WITH_MPI) - find_package(MPI COMPONENTS Fortran REQUIRED) - - target_compile_definitions(simulation PRIVATE MFC_MPI) - target_link_libraries (simulation PRIVATE MPI::MPI_Fortran) - endif() - - if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI") find_package(CUDAToolkit REQUIRED) - target_link_libraries(simulation PRIVATE CUDA::nvToolsExt) endif() - - target_compile_options(simulation PRIVATE -DMFC_SIMULATION) - + target_compile_definitions(simulation PRIVATE MFC_SIMULATION) install(TARGETS simulation RUNTIME DESTINATION bin) endif() # === === src/post_process -if (MFC_BUILD_POST_PROCESS) - add_executable(post_process "${post_process_srcs}" "${common_srcs}") +if (MFC_POST_PROCESS) + add_executable(post_process ${post_process_SRCs}) find_package(FFTW REQUIRED) find_package(SILO REQUIRED) @@ -304,44 +288,48 @@ if (MFC_BUILD_POST_PROCESS) target_link_libraries(post_process PRIVATE SILO::SILO HDF5::HDF5 FFTW::FFTW) - if (MFC_WITH_MPI) + if (MFC_MPI) find_package(MPI COMPONENTS Fortran REQUIRED) target_compile_definitions(post_process PRIVATE MFC_MPI) target_link_libraries (post_process PRIVATE MPI::MPI_Fortran) endif() + if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + target_compile_options(post_process PRIVATE -h noacc -x acc) + endif() + # -O0 is in response to https://github.com/MFlowCode/MFC-develop/issues/95 - target_compile_options(post_process PRIVATE -DMFC_POST_PROCESS -O0) + target_compile_definitions(post_process PRIVATE MFC_POST_PROCESS) + target_compile_options (post_process PRIVATE -O0) install(TARGETS post_process RUNTIME DESTINATION bin) endif() -add_custom_target("doc") +add_custom_target(documentation) macro(GEN_DOCS target name) - set(DOXYGEN_PROJECT_NAME "${name}") set(DOXYGEN_INPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/${target}/ ${CMAKE_CURRENT_BINARY_DIR}/doxygen/${target}/") - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/doc/${target}" + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/docs/${target}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/doxygen") - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/doc/doxygen" + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/docs/doxygen" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/doxygen/${target}") configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in" + "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/${target}/Doxyfile" @ONLY ) - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/doc/res" + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/docs/res" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/doxygen") - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/doc/config.js" + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/docs/config.js" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/doxygen") add_custom_command( @@ -351,7 +339,7 @@ macro(GEN_DOCS target name) COMMAND "${DOXYGEN_EXECUTABLE}" "Doxyfile" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doxygen/${target}" - COMMENT "${target}: Generating docs" + COMMENT "${target}: Generating documentation" ) add_custom_target( @@ -360,29 +348,27 @@ macro(GEN_DOCS target name) ) add_dependencies("${target}_doxygen" doxygen-awesome-css) - add_dependencies("doc" "${target}_doxygen") + add_dependencies(documentation "${target}_doxygen") install( DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doxygen/${target}/html/" - DESTINATION "doc/mfc/${target}" + DESTINATION "docs/mfc/${target}" ) install( DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doxygen/res/" - DESTINATION "doc/mfc/res" + DESTINATION "docs/mfc/res" ) install( - FILES "${CMAKE_CURRENT_SOURCE_DIR}/doc/index.html" - DESTINATION "doc/mfc" + FILES "${CMAKE_CURRENT_SOURCE_DIR}/docs/index.html" + DESTINATION "docs/mfc" ) - endmacro() # === Documentation -if (MFC_BUILD_DOC) - +if (MFC_DOCUMENTATION) # === === Find Doxygen find_package(Doxygen REQUIRED dot REQUIRED) @@ -407,5 +393,5 @@ if (MFC_BUILD_DOC) # === === Generate Landing Page GEN_DOCS(documentation "MFC") - endif() + diff --git a/README.md b/README.md index da9a9c3369..460bc65f84 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@

- MFC Banner + + MFC Banner +

@@ -9,39 +11,35 @@ - - - - - - - - - DOI

-Welcome to the home of MFC! -MFC simulates compressible multi-component and multi-phase flows, amongst other things. -It scales ideally to tens of thousands of GPUs. -MFC is written in Fortran and makes use of metaprogramming to keep the codebase readable and compact. -Please contact the developers, like [Spencer](mailto:shb@gatech.edu), if you have any questions. -We have an active Slack channel to help ease in new MFC users and support development. +

+ Welcome to the home of MFC! + MFC simulates compressible multi-component and multi-phase flows, amongst other things. + It scales ideally to tens of thousands of GPUs. + MFC is written in Fortran and makes use of metaprogramming to keep the codebase readable and compact. + Please contact the developers, like Spencer, if you have any questions. + We have an active Slack channel to help ease in new MFC users and support development. -MFC has both API and high-level documentation. -It is available on [the website](https://mflowcode.github.io/) and in markdown format at [doc/documentation/readme.md](doc/documentation/readme.md). + MFC has both API and high-level documentation on [its website](https://mflowcode.github.io/). + The latter is also available in [markdown](docs/documentation/readme.md). +

If you use MFC, consider citing it: -* + +

+ S. H. Bryngelson, K. Schmidmayer, V. Coralic, K. Maeda, J. Meng, T. Colonius (2021) Computer Physics Communications 4655, 107396 - + +

-``` +```bibtex @article{Bryngelson_2021, title = {MFC: An open-source high-order multi-component, multi-phase, and multi-scale compressible flow solver}, author = {Spencer H. Bryngelson and Kevin Schmidmayer and Vedran Coralic and Jomela C. Meng and Kazuki Maeda and Tim Colonius}, @@ -60,6 +58,6 @@ MFC is under the MIT license (see [LICENSE](LICENSE) file for full text). ## Acknowledgements

- MFC development was supported by multiple current and past grants from the US Office of Naval Research (ONR), the US National Institute of Health (NIH), and the US National Science Foundation (NSF). + MFC development was supported by multiple current and past grants from the US Office of Naval Research (ONR), the US National Institute of Health (NIH), and the US National Science Foundation (NSF). MFC computations utilize the Extreme Science and Engineering Discovery Environment (XSEDE, now ACCESS), under allocations TG-CTS120005 (PI Colonius) and TG-PHY210084 (PI Bryngelson) and OLCF Summit under allocation CFD154 (PI Bryngelson).

diff --git a/defaults.yaml b/defaults.yaml deleted file mode 100644 index 2e8d41c373..0000000000 --- a/defaults.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# -# This configuration file is geared towards users of MFC. -# More information can be found in the README. -# - -build: - threads: 1 - -run: - nodes: 1 - account: "" - partition: "" - cpus-per-node: 1 - gpus-per-node: 0 - walltime: "01:00:00" - email: "" - name: "unnamed" - flags: [] - -modes: -- name: release-cpu - type: Release - flags: [] - -- name: debug-cpu - type: Debug - flags: [] - -- name: release-gpu - type: Release - flags: - - -DMFC_WITH_OPEN_ACC=ON - -- name: debug-gpu - type: Debug - flags: - - -DMFC_WITH_OPEN_ACC=ON diff --git a/doc/documentation/readme.md b/doc/documentation/readme.md deleted file mode 100644 index 44057c8e76..0000000000 --- a/doc/documentation/readme.md +++ /dev/null @@ -1,120 +0,0 @@ -# Welcome to MFC! - -The MFC is a fully-documented parallel simulation software for multi-component, multi-phase, and bubbly flows. The source code of MFC is available [on GitHub](https://github.com/MFlowCode/MFC) and is open to contributions! - -## Code Documentation - -- [Pre Process](../pre_process/) -- [Simulation](../simulation/) -- [Post Process](../post_process/) - -## Publications - -### Primary Paper - -The paper that describes the MFC's capabilities: -* - S. H. Bryngelson, K. Schmidmayer, V. Coralic, K. Maeda, J. Meng, T. Colonius (2021) Computer Physics Communications 4655, 107396 - - -``` -@article{Bryngelson_2021, - title = {MFC: An open-source high-order multi-component, multi-phase, and multi-scale compressible flow solver}, - author = {Spencer H. Bryngelson and Kevin Schmidmayer and Vedran Coralic and Jomela C. Meng and Kazuki Maeda and Tim Colonius}, - journal = {Computer Physics Communications}, - doi = {10.1016/j.cpc.2020.107396}, - year = 2021, - month = {may}, - publisher = {Elsevier {BV}}, - pages = {107396}, -} -``` - -### Related publications - -Several publications have used the MFC in various stages of its -development. A partial list is included here. - -Journal papers: -* - S. H. Bryngelson, R. O. Fox, T. Colonius (2021) arXiv: 2112.14172. - -* - S. H. Bryngelson and T. Colonius (2020) Journal of the Acoustical Society of America, Vol. 147, pp. 1126-1135 - -* - K. Schmidmayer, S. H. Bryngelson, T. Colonius (2020) Journal of Computational Physics, Vol. 402, 109080 - -* - S. H. Bryngelson, K. Schmidmayer, T. Colonius (2019) International Journal of Multiphase Flow, Vol. 115, pp. 137-143 - -* - K. Maeda and T. Colonius (2019) Journal of Fluid Mechanics, Vol. 862, pp. 1105-1134 - -* - K. Maeda and T. Colonius (2018) Journal of Computational Physics, Vol. 371, pp. 994-1017 - -* - J. C. Meng and T. Colonius (2018) Journal of Fluid Mechanics, Vol. 835, pp. 1108-1135 - -* - K. Maeda and T. Colonius (2017) Wave Motion, Vol. 75, pp. 36-49 - -* - J. C. Meng and T. Colonius (2015) Shock Waves, Vol. 25(4), pp. 399-414 - -* - V. Coralic and T. Colonius (2014) Journal of Computational Physics, Vol. 274, pp. 95-121 - - -Ph.D. Disserations: -* - J.-C. Veilleux (2019) Ph.D. thesis, California Institute of Technology - -* - K. Maeda (2018) Ph.D. thesis, California Institute of Technology - -* - J. Meng (2016) Ph.D. thesis, California Institute of Technology - -* - V. Coralic (2014) Ph.D. thesis, California Institute of Technology - - -### User Configuration - -The [mfc.sh](https://github.com/MFlowCode/MFC/blob/master/mfc.sh) script used in the previous section is configured through the [defaults.yaml](https://github.com/MFlowCode/MFC/blob/master/defaults.yaml) file. - -## Development - -### Fypp - -MFC uses [Fypp](https://github.com/aradi/fypp), a Python-based Fortran preprocessor to reduce code duplication. `.fpp` files are converted into regular `.f90` files as part of the build process. Documentation for Fypp can be found [here](https://fypp.readthedocs.io/en/stable/). - -You can inspect the generated `.f90` files in `src//autogen/`. - -## Useful Scripts - -### Loading Modules - -On computer systems that run using environment modules, with implementations like [TACC's Lmod](https://github.com/TACC/Lmod), MFC provides a script that can load modules that have been used by contributors. - -```console -$ . ./mfc.sh load -``` - -The list of modules offered by a system is subject to change. The aforementioned script serves as a convenient way to load modules that should work for most users of MFC. - -### OpenACC Memory Profiling - -You can append `-DMFC_MEMORY_DUMP` to `release-gpu`'s Fortran compiler options in [defaults.yaml](defaults.yaml) to make the [simulation code](src/simulation/) call `acc_present_dump()` at various stages of program execution to obtain a printout of on-device memory usage. The [mem_parse.sh](misc/mem_parse.sh) script can be given as an argument the path to a file containing MFC's output, in order to aggregate the data and produce tables with formatted output. - -## License - -Copyright 2022. -MFC is under the MIT license (see [LICENSE](https://github.com/MFlowCode/MFC/blob/master/LICENSE) file for full text). - -## Acknowledgements - -The development of the MFC was supported in part by multiple current and past grants from the US Office of Naval Research (ONR), the US National Institute of Health (NIH), and the US National Science Foundation (NSF). -MFC computations utilize the Extreme Science and Engineering Discovery Environment (XSEDE), under allocations TG-CTS120005 (PI Colonius) and TG-PHY210084 (PI Bryngelson) and ORNL Summit under allocation CFD154 (PI Bryngelson). diff --git a/doc/index.html b/doc/index.html deleted file mode 100644 index 39cd048881..0000000000 --- a/doc/index.html +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - MFC | Home - - - - - -
-
-
-
- - -

MFC

- -
-
- - - -
-
-
- -
-
-
- - - Multi-Component Flow Code - -
- GitHub | - Download | - Documentation -
-
-
- Welcome to the home of MFC! MFC simulates compressible multi-component and multi-phase flows, amongst other things. It scales ideally to tens of thousands of GPUs. MFC is a Fortran codebase that makes use of metaprogramming to keep the codebase readable and compact. Please contact the developers, like Spencer, if you have any questions. We have an active Slack channel to help ease in new MFC users and support development.
-
-
-
- -
- - - -
-
-

- Acknowledgements -

-
-
- MFC development was supported by multiple current and past grants from the US Office of Naval Research (ONR), the US National Institute of Health (NIH), and the US National Science Foundation (NSF). MFC computations utilize the Extreme Science and Engineering Discovery Environment (XSEDE), under allocations TG-CTS120005 (PI Colonius) and TG-PHY210084 (PI Bryngelson) and OLCF Summit under allocation CFD154 (PI Bryngelson). -
-
- - diff --git a/doc/res/github.png b/doc/res/github.png deleted file mode 100644 index 192846a120..0000000000 Binary files a/doc/res/github.png and /dev/null differ diff --git a/doc/Doxyfile.in b/docs/Doxyfile.in similarity index 100% rename from doc/Doxyfile.in rename to docs/Doxyfile.in diff --git a/doc/config.js b/docs/config.js similarity index 100% rename from doc/config.js rename to docs/config.js diff --git a/doc/documentation/CODE_OF_CONDUCT.md b/docs/documentation/CODE_OF_CONDUCT.md similarity index 100% rename from doc/documentation/CODE_OF_CONDUCT.md rename to docs/documentation/CODE_OF_CONDUCT.md diff --git a/doc/documentation/authors.md b/docs/documentation/authors.md similarity index 100% rename from doc/documentation/authors.md rename to docs/documentation/authors.md diff --git a/doc/documentation/building.md b/docs/documentation/getting-started.md similarity index 60% rename from doc/documentation/building.md rename to docs/documentation/getting-started.md index 9683a193a9..06f5546e5d 100644 --- a/doc/documentation/building.md +++ b/docs/documentation/getting-started.md @@ -1,19 +1,17 @@ -# Building +# Getting Started ## Build Environment -To fetch, build, and run MFC and its dependencies on a UNIX-like system, you -must have installed common utilities such as GNU's Make, Python3, its development -headers and libraries, a C/C++ compiler -(GCC, NVHPC, etc., but *not Clang*), and an MPI wrapper (like Open MPI). -Below are some commands for popular operating systems and package managers. +MFC can be built in multiple ways on various operating systems. Please select your desired configuration from the list bellow: -[Anaconda](https://www.anaconda.com/) may interfere with the building process. -If an issue arises, you can either uninstall the affected Anaconda packages, -change the ordering of directory paths in your `$PATH`, or make aliases to the -correct binaries. +
+

*nix

 -### *nix +- **On supported clusters:** Load environment modules + +```console +$ . ./mfc.sh load +``` - **Via [Aptitude](https://wiki.debian.org/Aptitude):** @@ -37,14 +35,20 @@ $ sudo pacman -S base-devel coreutils \ wget tree ``` -If you wish to build MFC using [NVidia's NVHPC SDK](https://developer.nvidia.com/hpc-sdk), follow the instructions [here](https://developer.nvidia.com/nvidia-hpc-sdk-downloads). +If you wish to build MFC using [NVidia's NVHPC SDK](https://developer.nvidia.com/hpc-sdk), +first follow the instructions [here](https://developer.nvidia.com/nvidia-hpc-sdk-downloads). -### Windows +
-On Windows, you can either use Intel Compilers with the standard Microsoft toolchain, [Docker](https://docs.docker.com/get-docker/) or -the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/) for a Linux experience. +
+

Windows

 -#### Windows + Intel (Native) +On Windows, you can either use Intel Compilers with the standard Microsoft toolchain, +[Docker](https://docs.docker.com/get-docker/) or the +[Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/) for a Linux experience. + +
+

Windows + Intel (Native)

 Install the latest version of: - [Microsoft Visual Studio Community](https://visualstudio.microsoft.com/) @@ -61,11 +65,10 @@ commands. `./mfc.sh` is intended Unix-like systems. You will also have access to Microsoft Visual Studio solution files for an IDE (Integrated Development Environment). -#### Windows + Docker - -See the instructions in the "Docker (Cross-Platform)" section of this document. +
-#### Windows + WSL +
+

Windows + WSL

 Install the latest version of the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/) as well as a distribution such as Ubuntu which can be found [here](https://apps.microsoft.com/store/detail/ubuntu/9PDXGNCFSCZV). Acquiring an interactive session is as simple as typing `wsl` in your @@ -74,7 +77,12 @@ available in the [Microsoft Terminal](https://apps.microsoft.com/store/detail/wi You can now follow the appropriate instructions for your distribution. -### MacOS (x86 and Apple Silicon) +
+ +
+ +
+

MacOS (x86 and Apple Silicon)

 **Note:** macOS remains the most difficult platform to consistently compile MFC on. If you run into issues, we suggest you try using Docker (instructions above). @@ -116,7 +124,10 @@ $ HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-$MFC_GCC_VER --verbose --bui They will download the dependencies MFC requires to build itself. `open-mpi` will be compiled from source, using the version of GCC we specified above with the environment variables `HOMEBREW_CC` and `HOMEBREW_CXX`. Building this package might take a while. -### Docker (Cross-Platform) +
+ +
+

Docker

 Docker is a lightweight, cross-platform, and performant alternative to Virtual Machines (VMs). We build a Docker Image that contains the packages required to build and run MFC on your local machine. @@ -153,45 +164,61 @@ access your local copy of MFC, available at `~/MFC`. You will be logged-in as th Thus, any modification outside of `~/MFC` should be considered as permanently lost upon session exit. -## Fetch & Build +
-MFC can be built without a helper script by directly running CMake but they -(`mfc.sh` on Linux and `mfc.bat` on Windows) offer convenience features that -assist in building, testing, optimization, as well as interactive and batch execution. To -build MFC without a helper script, consult the [CMakeLists.txt](CMakeLists.txt) -file for a full list of options, as well as [toolchain/dependencies/CMakeLists.txt](toolchain/dependencies/CMakeLists.txt) -for a CMake superbuild file that fetches and builds MFC's main dependencies with -supported versions. +## Fetching MFC -+ **Fetch MFC:** +You can either download MFC's [latest release from GitHub](https://github.com/MFlowCode/MFC/releases/latest) or clone the repository: ```console -$ git clone https://github.com/MFlowCode/MFC +$ git clone https://github.com/MFlowCode/MFC.git $ cd MFC +$ git checkout ``` -+ **(Optional) Configure MFC defaults in [defaults.yaml](defaults.yaml):** +## Building MFC + +MFC can be built with support for various (compile-time) features: -If you wish, you can override MFC's default build parameters in [defaults.yaml](defaults.yaml), a file intended for user customization. This can greatly reduce the number of command-line arguments you have to pass to [mfc.sh](mfc.sh)` in the following sections. You can do this at any time. +| Feature | Enable | Disable | Default | Description | +| :-------: | :-------: | :----------: | :-----: | --------------------------------------------------------------- | +| **MPI** | `--mpi` | `--no-mpi` | On | Lets MFC run on multiple processors (and nodes) simultaneously. | +| **GPU** | `--gpu` | `--no-gpu` | Off | Enables GPU acceleration via OpenACC. | +| **Debug** | `--debug` | `--no-debug` | Off | Requests the compiler build MFC in debug mode. | -+ **Build MFC's codes in `release-cpu` mode with 8 threads:** +_⚠️ The `--gpu` option requires that your compiler supports OpenACC for Fortran for your target GPU architecture._ +When these options are given to `mfc.sh`, they will be remembered when you issue future commands. You can enable and disable features at any time by passing any of the arguments above. For example, if you have previously built MFC with MPI support and no longer wish to run using MPI, you can pass `--no-mpi` once, for the change to be permanent. + +MFC is composed of three codes, each being a separate _target_. By default, all targets (`pre_process`, `simulation`, and `post_process`) are selected. To only select a subset, use the `-t` (i.e `--targets`) argument. For a detailed list of options, arguments, and features, please refer to `./mfc.sh build --help`. + +Most first-time users will want to build MFC using 8 threads (or more!) with MPI support: ```console -$ ./mfc.sh build -t pre_process simulation post_process -j 8 +$ ./mfc.sh build -j 8 ``` -To build MFC in different configurations (herein, *modes*), the `-m ` option -can be specified to each call to `mfc.sh`. A full list of modes is located in -[defaults.yaml](defaults.yaml). It can be modified to work with system, and additional -modes can be created at your discretion. The default mode is `release-cpu` but -you can use others such as `release-gpu`. +Examples: + +- Build MFC using 8 threads with MPI and GPU acceleration: `$ ./mfc.sh build --gpu -j 8`. +- Build MFC using a single thread without MPI, GPU, and Debug support: `$ ./mfc.sh build --no-mpi`. +- Build MFC's `simulation` code in Debug mode with MPI and GPU support: `$ ./mfc.sh build --debug --gpu -t simulation`. -**IMPORTANT NOTE**: This same mode will be used for any future commands such as `./mfc.sh test` and `./mfc.sh run` until you specify `-m` again (in any of these commands). +## Running the Test Suite -+ **Run MFC's tests in `release-cpu` mode with 8 threads:** +Run MFC's test suite with 8 threads: ```console $ ./mfc.sh test -j 8 ``` -Please refer to the [Testing](#testing-mfc) section of this document for more information. +Please refer to the [Testing](testing.md) document for more information. + +## Running an Example Case + +MFC has example cases in the `examples` folder. You can run such a case interactively using 2 tasks by typing: + +```console +$ ./mfc.sh run examples/2D_shockbubble/case.py -n 2 +``` + +Please refer to the [Running](running.md) document for more information on `case.py` files and how to run them. diff --git a/docs/documentation/readme.md b/docs/documentation/readme.md new file mode 100644 index 0000000000..3db95a9bda --- /dev/null +++ b/docs/documentation/readme.md @@ -0,0 +1,19 @@ +# Documentation + +## User Documentation + +- [Getting Started](getting-started.md) +- [Running MFC](running.md) +- [Testing MFC](testing.md) +- [Flow Visualisation](visualisation.md) +- [MFC's Authors](authors.md) +- [References](references.md) + +## Code/API Documentation + +MFC's three codes have their own documentation: + +- [Pre Process](../pre_process/) +- [Simulation](../simulation/) +- [Post Process](../post_process/) + diff --git a/doc/documentation/references.md b/docs/documentation/references.md similarity index 100% rename from doc/documentation/references.md rename to docs/documentation/references.md diff --git a/doc/documentation/running.md b/docs/documentation/running.md similarity index 100% rename from doc/documentation/running.md rename to docs/documentation/running.md diff --git a/doc/documentation/testing.md b/docs/documentation/testing.md similarity index 100% rename from doc/documentation/testing.md rename to docs/documentation/testing.md diff --git a/doc/documentation/visualisation.md b/docs/documentation/visualisation.md similarity index 100% rename from doc/documentation/visualisation.md rename to docs/documentation/visualisation.md diff --git a/doc/doxygen/footer.html b/docs/doxygen/footer.html similarity index 100% rename from doc/doxygen/footer.html rename to docs/doxygen/footer.html diff --git a/doc/doxygen/header.html b/docs/doxygen/header.html similarity index 100% rename from doc/doxygen/header.html rename to docs/doxygen/header.html diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000000..49f3556960 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,172 @@ + + + + + + + MFC | Home + + + + + + +
+
+
+ +
Multi-Component Flow Code
+
+
+
+
+
+ +
+
+ MFC simulates compressible multi-component and multi-phase flows, amongst other things. + It scales ideally to tens of thousands of GPUs. +
+
+
+
+ + + GitHub + + + +
+ + +
+ + + + Getting Started + + + + Documentation + + + + Primary Paper + +
+
+
+
+
+
+ + + +
+
+
+
+ © 2022 Computational Physics Group at the Georgia Institute of Technology +
+
+ MFC development was supported by multiple current and past grants from the US Office of Naval Research (ONR), the US National Institute of Health (NIH), and the US National Science Foundation (NSF). MFC computations utilize the Extreme Science and Engineering Discovery Environment (XSEDE), under allocations TG-CTS120005 (PI Colonius) and TG-PHY210084 (PI Bryngelson) and OLCF Summit under allocation CFD154 (PI Bryngelson). +
+
+ + diff --git a/doc/post_process/readme.md b/docs/post_process/readme.md similarity index 100% rename from doc/post_process/readme.md rename to docs/post_process/readme.md diff --git a/doc/pre_process/readme.md b/docs/pre_process/readme.md similarity index 100% rename from doc/pre_process/readme.md rename to docs/pre_process/readme.md diff --git a/doc/res/logo_light.png b/docs/res/banner.png similarity index 100% rename from doc/res/logo_light.png rename to docs/res/banner.png diff --git a/doc/res/icon.ico b/docs/res/icon.ico similarity index 100% rename from doc/res/icon.ico rename to docs/res/icon.ico diff --git a/doc/res/logo.png b/docs/res/logo.png similarity index 100% rename from doc/res/logo.png rename to docs/res/logo.png diff --git a/doc/res/banner.png b/docs/res/readme.png similarity index 100% rename from doc/res/banner.png rename to docs/res/readme.png diff --git a/doc/res/visit.png b/docs/res/visit.png similarity index 100% rename from doc/res/visit.png rename to docs/res/visit.png diff --git a/doc/simulation/readme.md b/docs/simulation/readme.md similarity index 100% rename from doc/simulation/readme.md rename to docs/simulation/readme.md diff --git a/mfc.bat b/mfc.bat index 37894a2c65..893d64dd53 100644 --- a/mfc.bat +++ b/mfc.bat @@ -1,32 +1,16 @@ @echo off - -if "%1" == "" goto label_help -if "%1" == "-h" goto label_help -if "%1" == "--help" goto label_help if "%1" == "docker" goto label_docker goto label_windows - -:label_help -echo. -echo mfc.bat: Windows proxy script for mfc.sh using Docker. -echo. -echo Usage (1): mfc.bat ^[COMMAND^] # Equivalent to "./mfc.sh [COMMAND]" -echo Usage (2): mfc.bat docker # Obtain an interactive bash session -echo. -exit /b 0 -:label_help_after - - :label_windows if not exist "%cd%\toolchain\mfc.py" ( echo. echo ^[mfc.bat^] You must call this script from within MFC's root folder echo. - exit /b %errorlevel% + exit /b 1 ) mkdir "%cd%\build" 2> NUL @@ -37,15 +21,7 @@ if not exist "%cd%\build\venv" ( echo. echo ^[mfc.bat^] Failed to create the Python virtual environment. Delete the build/venv folder and try again. echo. - exit /b %errorlevel% - ) - - pip3 install pyyaml fypp rich argparse dataclasses - if %errorlevel% neq 0 ( - echo. - echo ^[mfc.bat^] Failed to install Python dependencies via Pip. Delete the build/venv folder and try again. - echo. - exit /b %errorlevel% + exit /b 1 ) ) @@ -54,7 +30,14 @@ if %errorlevel% neq 0 ( echo. echo ^[mfc.bat^] Failed to activate the Python virtual environment. echo. - exit /b %errorlevel% + exit /b 1 +) + +fc /b "%cd%\build\requirements.txt" "%cd%\toolchain\requirements.txt" 2> NUL +if %errorlevel% neq 0 ( + pip3 install -r toolchain/requirements.txt + + copy "%cd%\toolchain\requirements.txt" "%cd%\build" 2> NUL ) python3 "%cd%\toolchain\mfc.py" %* @@ -79,7 +62,7 @@ if %errorlevel% neq 0 ( echo. echo ^[mfc.bat^] You must have Docker installed. echo Please install Docker and try again. - exit /b %errorlevel% + exit /b 1 ) @@ -89,7 +72,7 @@ if %errorlevel% neq 0 ( echo. echo ^[mfc.bat^] Docker: Failed to fetch image. echo Pleasure ensure docker is running. - exit /b %errorlevel% + exit /b 1 ) echo ^[mfc.bat^] Starting container... @@ -100,7 +83,7 @@ docker run --interactive --tty --rm ^ if %errorlevel% neq 0 ( echo. echo Docker: Fatal container runtime error. - exit /b %errorlevel% + exit /b 1 ) diff --git a/mfc.sh b/mfc.sh index 4d60dfc3e1..833182d967 100755 --- a/mfc.sh +++ b/mfc.sh @@ -17,28 +17,236 @@ source "$(pwd)/toolchain/util.sh" # Handle upgrading from older MFC build systems if [ -d "$(pwd)/bootstrap" ] || [ -d "$(pwd)/dependencies" ] || [ -f "$(pwd)/build/mfc.lock.yaml" ]; then error "You are upgrading from an older version of$MAGENTA MFC$COLOR_RESET. Please remove, if applicable, the dependencies/, bootstrap/, and build/ directories before running this command again." - + exit 1 fi # If the user wishes to run the "load" script if [ "$1" == "load" ]; then shift; - source "$(pwd)/toolchain/scripts/load.sh" "$@" + + # Reset u_computer & u_cg to known values since this script is run with "source" + # Therefore, values of variables defined here are kept when the script runs again. + u_computer=""; u_cg="" + + # If there are command-line arguments, parse them: + while [[ $# -gt 0 ]]; do + case $1 in + -c|--computer) u_computer="$2"; shift; shift; ;; + -m|--mode) u_cg="$2"; shift; shift; ;; + -*|--*) echo "Unknown option $1"; return; ;; + esac + done + + # Get computer (if not supplied in command-line) + if [ -v $u_computer ]; then + log "Select a system:" + log "$G""ORNL$W: Ascent (a), Crusher (c), Summit (s), Wombat (w)" + log "$C""ACCESS$W: Bridges2 (b), Expanse (e)" + log "$Y""GaTech$W: Phoenix (p)" + log "$R""CALTECH$W: Richardson (r)" + log_n "($G""a$W/$G""c$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$Y""p$CR/$R""r$CR): " + read u_computer + log + fi + + # Get CPU/GPU (if not supplied in command-line) + if [ -v $u_cg ]; then + log "Select configuration:" + log " - CPU (c)" + log " - GPU (g)" + log_n "(c/g): " + read u_cg + log + fi + + # User input to lowercase + u_computer=$(echo "$u_computer" | tr '[:upper:]' '[:lower:]') + u_cg=$(echo "$u_cg" | tr '[:upper:]' '[:lower:]') + + COMPUTER=""; CG="" + declare -a MODULES + + if [ "$u_cg" == "c" ]; then # for CPU + CG="CPU" + elif [ "$u_cg" == "g" ]; then # For GPU + CG="GPU" + fi + + if [ "$u_computer" == "s" ]; then # For Summit + if [ "$u_cg" == "c" ]; then + MODULES=("gcc/12.1.0") + elif [ "$u_cg" == "g" ]; then + MODULES=("nvhpc/22.5" "cuda/nvhpc") + fi + + MODULES=("${MODULES[@]}" "python/3.8.10" "darshan-runtime/3.3.1-lite" + "hsi/5.0.2.p5" "xalt/1.2.1" "lsf-tools/2.0" + "cmake/3.23.1" "ninja/1.10.2" "spectrum-mpi/10.4.0.3-20210112") + elif [ "$u_computer" == "b" ]; then # Bridges2 + if [ "$u_cg" == "c" ]; then + MODULES=("allocations/1.0" "gcc/10.2.0" "python/3.8.6" + "openmpi/4.0.5-gcc10.2.0") + elif [ "$u_cg" == "g" ]; then + MODULES=("nvhpc/22.9" "openmpi/4.0.5-nvhpc22.9") + fi + + MODULES=("${MODULES[@]}" "python/3.8.6") + elif [ "$u_computer" == "a" ]; then # For Ascent + if [ "$u_cg" == "c" ]; then + MODULES=("gcc/11.1.0" "spectrum-mpi" "cuda") + elif [ "$u_cg" == "g" ]; then + MODULES=("nvhpc/21.11" "spectrum-mpi" "cuda/nvhpc" + "nsight-compute" "nsight-systems") + fi + + MODULES=("${MODULES[@]}" "python" "cmake/3.22.2") + elif [ "$u_computer" == "r" ]; then # Richardson + if [ "$u_cg" == "c" ]; then + MODULES=("gcc/9.3.0" "openmpi-2.0/gcc-9.3.0") + elif [ "$u_cg" == "g" ]; then + error "GPU not supported on Richardson." + + return + fi + + MODULES=("${MODULES[@]}" "python/3.7") + elif [ "$u_computer" == "w" ]; then # For Wombat + if [ "$u_cg" == "c" ]; then + MODULES=("gcc/11.1.0" "openmpi/4.0.5_gcc") + elif [ "$u_cg" == "g" ]; then + MODULES=("cuda/11.5.1" "/sw/wombat/Nvidia_HPC_SDK/modulefiles/nvhpc/22.1") + fi + + MODULES=("${MODULES[@]}" "cmake/3.22.1" "python/3.9.9") + elif [ "$u_computer" == "e" ]; then # Expanse + if [ "$u_cg" == "c" ]; then + warn "Please set CC=icc, CXX=icx, and FC=ifort." + log + + MODULES=("cpu/0.15.4" "gcc/9.2.0" "openmpi/4.1.1" "cmake/3.18.2") + elif [ "$u_cg" == "g" ]; then + MODULES=("gpu/0.15.4" "cuda/11.0.2" "nvhpc/22.2" "openmpi/4.0.5" "cmake/3.19.8") + fi + + MODULES=("${MODULES[@]}" "python/3.8.5") + elif [ "$u_computer" == "p" ]; then # Phoenix + if [ "$u_cg" == "c" ]; then + MODULES=("intel/19.0.5" "mvapich2/2.3.2") + elif [ "$u_cg" == "g" ]; then + MODULES=("cuda/11.2" "nvhpc/22.1") + fi + + MODULES=("${MODULES[@]}" "python/3.7.4" "cmake/3.20.3") + elif [ "$u_computer" == "c" ]; then # Crusher + if [ "$u_cg" == "c" ]; then + MODULES=() + elif [ "$u_cg" == "g" ]; then + MODULES=("rocm/5.1.0" "craype-accel-amd-gfx90a") + fi + + MODULES=("${MODULES[@]}" "cmake/3.23.2" "cray-fftw/3.3.10.2" "hdf5/1.12.1" "cray-python/3.9.13.1" "ninja/1.10.2" "cray-mpich/8.1.23") + else + echo -e $RED"Error: Requested system $u_computer is not supported (yet!)"$COLOR_RESET + + return + fi + + log "Loading modules for $CG mode:" + + # Reset modules to default system configuration + if [ "$u_computer" != "p" ]; then + module reset > /dev/null 2>&1 + code="$?" + + # Purge if reset is not available + if [ "$code" -ne "0" ]; then + module purge > /dev/null 2>&1 + fi + else + module purge > /dev/null 2>&1 + fi + + # Find length of longest module_name in $MODULES for $COMPUTER + max_module_length="0" + for module_name in ${MODULES[@]}; do + module_length="${#module_name}" + + if [ "$module_length" -gt "$max_module_length" ]; then + max_module_length="$module_length" + fi + done + + # Load modules ($MODULES) + for module_name in ${MODULES[@]}; do + log_n " - Load $CYAN$module_name$COLOR_RESET " + + # Add padding spaces + module_length="${#module_name}" + delta="$((max_module_length-module_length-2))" + if [ "$delta" -ne "-2" ]; then + printf "%0.s-" $(seq 0 $delta) + echo -n " " + fi + + # Load the module + module load "$module_name" > /dev/null 2>&1 + + # Handle Success / Failure + code=$? + if [ "$code" == "0" ]; then + echo -e "[$G""SUCCESS$W]" + else + echo -e "[$R""FAILURE$W]" + + # Run load again to show error message + module load "$module_name" + + return + fi + done + + ok "All modules have been loaded." return elif [ "$1" == "format" ]; then - "$SHELL" "$(pwd)/toolchain/scripts/format.sh" + if ! command -v fprettify > /dev/null 2>&1; then + pip3 install --upgrade fprettify + fi - exit $? -fi + fprettify src --exclude "src/*/autogen" --recursive \ + --indent 4 --c-relations --enable-replacements --enable-decl \ + --whitespace-comma 1 --whitespace-multdiv 0 --whitespace-plusminus 1 \ + --case 1 1 1 1 --strict-indent + ret="$?" + + if [ "$ret" != "0" ]; then + error "failed to execute fprettify." + error "MFC has not been fprettify'ied." + + exit 1 + fi + + ok "MFC has been fprettify'ied." + + exit 0 +elif [ "$1" == "cloc" ]; then + if ! command -v cloc > /dev/null 2>&1; then + error "cloc (github.com/AlDanial/cloc) is not installed." + + exit 1 + fi -if [ "$1" == "docker" ]; then + cloc . --exclude-dir=build,tests,examples,.vscode,.github \ + --fullpath --not-match-d=src/*/*/autogen + + exit $? +elif [ "$1" == "docker" ]; then shift; if ! command -v docker > /dev/null 2>&1; then error "$MAGENTA""Docker$COLOR_RESET is not installed." - + exit 1 fi @@ -46,17 +254,17 @@ if [ "$1" == "docker" ]; then log " - Fetching image..." if ! docker pull henryleberre/mfc; then error "Failed to fetch$MAGENTA Docker$COLOR_RESET image from$MAGENTA Docker Hub$COLOR_RESET." - + exit 1 fi - echo " - Starting container..." + log " - Starting container..." docker run --interactive --tty --rm \ --mount type=bind,source="$(pwd)",target=/home/me/MFC \ henryleberre/mfc if (($?)); then error "Failed to start Docker container." - + exit 1 fi @@ -81,7 +289,7 @@ if ! command -v cmake > /dev/null 2>&1; then # Not installed bShouldInstallCMake=true - warning "$MAGENTA""CMake$COLOR_RESET is not installed." + warn "$MAGENTA""CMake$COLOR_RESET is not installed." else cmake_verstr=$(cmake --version | tr ' ' '\n' | sed -n 3p) cmake_major=$(echo $cmake_verstr | tr '.' '\n' | sed -n 1p) @@ -135,7 +343,7 @@ if [ "$bShouldInstallCMake" = true ]; then fi mkdir -p "$(pwd)/build/cmake" - + filename="cmake-$version-linux-$arch.sh" repository="https://github.com/Kitware/CMake" url="$repository/releases/download/v$version/$filename" @@ -155,7 +363,7 @@ if [ "$bShouldInstallCMake" = true ]; then if ! command -v wget > /dev/null 2>&1; then error "$MAGENTA""wget$COLOR_RESET is not installed but is necessary to download$MAGENTA CMake$COLOR_RESET." - + cmake_fatal_error fi @@ -169,7 +377,7 @@ if [ "$bShouldInstallCMake" = true ]; then if ! $SHELL "$(pwd)/build/cmake/$filename" "--skip-license" "--prefix=$(pwd)/build/cmake"; then error "Failed to install a compatible version of CMake." - + cmake_fatal_error fi @@ -195,25 +403,25 @@ if ! command -v pip3 > /dev/null 2>&1 && [ ! -f "$(pwd)/build/venv/bin/activate" # Check whether python3 is in the $PATH / is accessible. if ! command -v python3 > /dev/null 2>&1; then error "Couldn't find$MAGENTA Python$COLOR_RESET. Please ensure it is discoverable." - + exit 1 fi # CHeck Python's version for compatibility if ! python3 -c "import sys; exit(int(not (sys.version_info[0]==$MFC_PYTHON_MIN_MAJOR and sys.version_info[1] >= $MFC_PYTHON_MIN_MINOR)))"; then error "$(python3 --version) is incompatible.$MAGENTA Python$COLOR_RESET v$MFC_PYTHON_MIN_MAJOR.$MFC_PYTHON_MIN_MINOR or higher is required." - + exit 1 fi get_pip_url="https://bootstrap.pypa.io/pip/get-pip.py" - warning "$MAGENTA""Python$COLOR_RESET's$MAGENTA PIP$COLOR_RESET is not installed." - log "Downloading$MAGENTA Python$COLOR_RESET's$MAGENTA PIP$COLOR_RESET from $get_pip_url..." + warn "$MAGENTA""Python$COLOR_RESET's$MAGENTA PIP$COLOR_RESET is not installed." + log "Downloading$MAGENTA Python$COLOR_RESET's$MAGENTA PIP$COLOR_RESET from $get_pip_url..." if ! wget -O "$(pwd)/build/get-pip.py" "$get_pip_url"; then error "Couldn't download get-pip.py." - + exit 1 fi @@ -221,7 +429,7 @@ if ! command -v pip3 > /dev/null 2>&1 && [ ! -f "$(pwd)/build/venv/bin/activate" export PIP_DISABLE_PIP_VERSION_CHECK=1 if ! python3 "$(pwd)/build/get-pip.py" --user; then error "Couldn't install$MAGENTA pip$COLOR_RESET with get-pip.py" - + exit 1 fi @@ -233,7 +441,7 @@ fi if [ ! -f "$(pwd)/build/venv/bin/activate" ]; then if ! python3 -m venv "$(pwd)/build/venv"; then error "Failed to create a$MAGENTA Python$COLOR_RESET virtual environment. Delete the build/venv folder and try again." - + exit 1 fi @@ -268,37 +476,39 @@ fi # Activate the Python venv source "$(pwd)/build/venv/bin/activate" -ok "Entered the$MAGENTA Python$COLOR_RESET virtual environment (venv)." +ok "(venv) Entered the$MAGENTA Python$COLOR_RESET virtual environment." -# Fetch required Python modules. -# Some modules which are now in Python's standard library -# are imported as backports to support Python v3.6. -declare -a REQUIRED_PYTHON_MODULES=("wheel,wheel" "argparse,argparse" "dataclasses,dataclasses" "typing,typing" "yaml,pyyaml" "rich,rich" "fypp,fypp") +# Install Python dependencies if, either: +# - This script is running for the first time +# (or) +# - The requirements.txt file has changed +if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /dev/null 2>&1; then + log "(venv) (Re)Installing mfc.sh's Python dependencies (via Pip)." -for module in "${REQUIRED_PYTHON_MODULES[@]}"; do - import_name=$(echo $module | tr ',' '\n' | head -n 1) - install_name=$(echo $module | tr ',' '\n' | tail -n 1) + if ! PIP_DISABLE_PIP_VERSION_CHECK=1 pip3 install -r "$(pwd)/toolchain/requirements.txt"; then + error "(venv) Installation failed." - if ! python3 -c "import $import_name" > /dev/null 2>&1; then - if ! PIP_DISABLE_PIP_VERSION_CHECK=1 pip3 install "$install_name"; then - error "Failed to install $import_name/$install_name through Python3's pip." - - exit 1 - fi + log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment." + deactivate - ok "Installed$MAGENTA Python PIP$COLOR_RESET package $MAGENTA$install_name$COLOR_RESET (into venv)." + exit 1 fi -done + + ok "(venv) Installation succeeded." + + # Save the new/current requirements.txt + cp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/" +fi -# Run the mfc.py bootstrap script +# Run the main.py bootstrap script python3 "$(pwd)/toolchain/mfc.py" "$@" code=$? # Deactivate the Python virtualenv in case the user "source"'d this script -log "Exiting the$MAGENTA Python$COLOR_RESET virtual environment." +log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment." deactivate diff --git a/misc/gnu13oacc.sh b/misc/gnu13oacc.sh new file mode 100644 index 0000000000..4e5b2a1f23 --- /dev/null +++ b/misc/gnu13oacc.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Authors: Alex Woods (GaTech) and Henry Le Berre (GaTech). +# Description: Builds GNU 13 compilers with amdgcn-amdhsa offloading. + +set -v + +work_dir=$HOME/USERSCRATCH/build-gcc-amdgpu/ +install_dir=$HOME/tools/gcc/13 + +mkdir -p $work_dir +cd $work_dir + +mkdir builds +build_dir=$work_dir/builds + +## Clone and build llvm 13 +git clone -b release/13.x https://github.com/llvm/llvm-project.git + +llvmsrcdir=$work_dir/llvm-project +cd $build_dir +mkdir llvm +cd llvm +cmake -D 'LLVM_TARGETS_TO_BUILD=X86;AMDGPU' -D LLVM_ENABLE_PROJECTS=lld $llvmsrcdir/llvm +make -j $(nproc) + +mkdir -p $install_dir/amdgcn-amdhsa/bin +cp -a bin/llvm-ar $install_dir/amdgcn-amdhsa/bin/ar +cp -a bin/llvm-ar $install_dir/amdgcn-amdhsa/bin/ranlib +cp -a bin/llvm-mc $install_dir/amdgcn-amdhsa/bin/as +cp -a bin/llvm-nm $install_dir/amdgcn-amdhsa/bin/nm +cp -a bin/lld $install_dir/amdgcn-amdhsa/bin/ld + +# Clone gcc, newlib and link newlib in +cd $work_dir +git clone git://sourceware.org/git/newlib-cygwin.git newlib +git clone git://gcc.gnu.org/git/gcc.git gcc +cd gcc +contrib/download_prerequisites +ln -s ../newlib/newlib newlib +cd .. +target=$(gcc/config.guess) + +# Build offloading GCC +cd $build_dir +mkdir build-offload-gcc +cd build-offload-gcc +$work_dir/gcc/configure \ + --target=amdgcn-amdhsa --enable-languages=c,lto,fortran \ + --disable-sjlj-exceptions --with-newlib \ + --enable-as-accelerator-for=x86_64-pc-linux-gnu \ + --with-build-time-tools=$install_dir/amdgcn-amdhsa/bin \ + --disable-libquadmath --prefix=$install_dir +make -j $(nproc) || exit 1 +make install || exit 1 +cd .. + +rm $work_dir/gcc/newlib + +# Build host GCC +mkdir build-host-gcc +cd build-host-gcc +$work_dir/gcc/configure \ + --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu \ + --target=x86_64-pc-linux-gnu \ + --enable-offload-targets=amdgcn-amdhsa=$install_dir/amdgcn-amdhsa \ + --enable-languages="c,c++,fortran,lto" \ + --disable-multilib --prefix=$install_dir + +make -j $(nproc) || exit 1 +make install || exit 1 +cd .. diff --git a/runTimeTests.sh b/runTimeTests.sh deleted file mode 100644 index b282e6411d..0000000000 --- a/runTimeTests.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!bash - -for i in (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - ./mfc.sh run ../MonopoleTests/1C0780C8Big/case.py -n 6 -g 6 -t pre_process simulation diff --git a/src/common/m_compile_specific.f90 b/src/common/m_compile_specific.f90 index aeac4cea54..32c16e3faa 100644 --- a/src/common/m_compile_specific.f90 +++ b/src/common/m_compile_specific.f90 @@ -5,7 +5,11 @@ !> @brief This module contains subroutines that are compiler specific module m_compile_specific + ! Dependencies ============================================================= + use m_mpi_proxy + implicit none + ! ========================================================================== contains @@ -69,16 +73,22 @@ subroutine s_get_basename(dirpath, basename) character(LEN=*), intent(IN) :: dirpath character(LEN=*), intent(OUT) :: basename + integer :: iUnit + character(len=30) :: tmpfilepath + + write (tmpfilepath, '(A,I0)') 'basename_', proc_rank + #ifdef _WIN32 - call system('for /F %i in ("'//dirpath//'") do @echo %~ni > basename') + call system('for /F %i in ("'//trim(dirpath)//'") do @echo %~ni > '//trim(tmpfilepath)) #else - call system('basename "'//dirpath//'" > basename') + call system('basename "'//trim(dirpath)//'" > '//trim(tmpfilepath)) #endif - open (100, FILE='basename', FORM='formatted', STATUS='old') - read (100, '(A)') basename; close (100) + open (newunit=iUnit, FILE=trim(tmpfilepath), FORM='formatted', STATUS='old') + read (iUnit, '(A)') basename + close (iUnit) - call s_delete_file("basename") + call s_delete_file(trim(tmpfilepath)) end subroutine s_get_basename diff --git a/src/common/m_variables_conversion.fpp b/src/common/m_variables_conversion.fpp index 60ad56977f..704637236e 100644 --- a/src/common/m_variables_conversion.fpp +++ b/src/common/m_variables_conversion.fpp @@ -2,6 +2,8 @@ !! @file m_variables_conversion.f90 !! @brief Contains module m_variables_conversion +#:include 'macros.fpp' + !> @brief This module consists of subroutines used in the conversion of the !! conservative variables into the primitive ones and vice versa. In !! addition, the module also contains the subroutines used to obtain @@ -100,18 +102,21 @@ module m_variables_conversion !> @} integer :: ixb, ixe, iyb, iye, izb, ize + !$acc declare create(ixb, ixe, iyb, iye, izb, ize) + !! In simulation, gammas and pi_infs is already declared in m_global_variables #ifndef MFC_SIMULATION real(kind(0d0)), allocatable, dimension(:) :: gammas, pi_infs + !$acc declare create(gammas, pi_infs) #endif - real(kind(0d0)), allocatable, dimension(:) :: Gs - integer, allocatable, dimension(:) :: bubrs + real(kind(0d0)), allocatable, dimension(:) :: Gs + integer, allocatable, dimension(:) :: bubrs real(kind(0d0)), allocatable, dimension(:, :) :: Res -!$acc declare create(ixb, ixe, iyb, iye, izb, ize, momxb, momxe, bubxb, bubxe, contxb, contxe, advxb, advxe, strxb, strxe, gammas, pi_infs, bubrs, Gs, Res) + !$acc declare create(bubrs, Gs, Res) integer :: is1b, is2b, is3b, is1e, is2e, is3e -!$acc declare create(is1b, is2b, is3b, is1e, is2e, is3e) + !$acc declare create(is1b, is2b, is3b, is1e, is2e, is3e) real(kind(0d0)), allocatable, dimension(:, :, :), target, public :: rho_sf !< Scalar density function real(kind(0d0)), allocatable, dimension(:, :, :), target, public :: gamma_sf !< Scalar sp. heat ratio function @@ -609,77 +614,58 @@ contains iyb = 0; iye = 0; izb = 0; ize = 0; if (n > 0) then - iyb = -buff_size - iye = n - iyb - end if + iyb = -buff_size; iye = n - iyb - if (p > 0) then - izb = -buff_size - ize = p - izb - end if + if (p > 0) then + izb = -buff_size; ize = p - izb + end if + end if #endif -!$acc update device(ixb, ixe, iyb, iye, izb, ize) - - allocate (gammas(1:num_fluids)) - allocate (pi_infs(1:num_fluids)) - allocate (Gs(1:num_fluids)) + !$acc update device(ixb, ixe, iyb, iye, izb, ize) - allocate (bubrs(1:nb)) + @:ALLOCATE(gammas (1:num_fluids)) + @:ALLOCATE(pi_infs(1:num_fluids)) + @:ALLOCATE(Gs (1:num_fluids)) do i = 1, num_fluids - gammas(i) = fluid_pp(i)%gamma + gammas(i) = fluid_pp(i)%gamma pi_infs(i) = fluid_pp(i)%pi_inf - Gs(i) = fluid_pp(i)%G + Gs(i) = fluid_pp(i)%G end do -!$acc update device(gammas, pi_infs, Gs) - -!TODO: this update was in previous version: (no longer needed?) -!!!$acc update device(small_alf, dflt_real, dflt_int, pi, nnode, sgm_eps) + !$acc update device(gammas, pi_infs, Gs) #ifdef MFC_SIMULATION if (any(Re_size > 0)) then - allocate (Res(1:2, 1:maxval(Re_size))) - end if - - if (any(Re_size > 0)) then + @:ALLOCATE(Res(1:2, 1:maxval(Re_size))) + do i = 1, 2 do j = 1, Re_size(i) Res(i, j) = fluid_pp(Re_idx(i, j))%Re(i) end do end do -!$acc update device(Res, Re_idx, Re_size) + + !$acc update device(Res, Re_idx, Re_size) end if #endif if (bubbles) then + @:ALLOCATE(bubrs(1:nb)) do i = 1, nb bubrs(i) = bub_idx%rs(i) end do + !$acc update device(bubrs) end if -!$acc update device(bubrs) !$acc update device(dt, sys_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, nb, weight, grid_geometry, cyl_coord, mapped_weno, mp_weno, weno_eps) !$acc update device(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma) !$acc update device(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam) -!TODO: this was in previous version: (check that not needed) -!!$acc update device(monopole, num_mono, mono) -! do i = 1, num_mono -!!$acc update device(mono(i)%mag) -!!$acc update device(mono(i)%length) -!!$acc update device(mono(i)%npulse) -!!$acc update device(mono(i)%dir) -!!$acc update device(mono(i)%delay) -!!$acc update device(mono(i)%aperture) -!!$acc update device(mono(i)%foc_length) -! end do - - !$acc update device(monopole, num_mono) +!$acc update device(monopole, num_mono) #ifdef MFC_POST_PROCESS ! Allocating the density, the specific heat ratio function and the @@ -760,19 +746,14 @@ contains gm_alphaK_vf, & ix, iy, iz) - type(scalar_field), & - dimension(sys_size), & - intent(INOUT) :: qK_cons_vf - - type(scalar_field), & - dimension(sys_size), & - intent(INOUT) :: qK_prim_vf + type(scalar_field), dimension(sys_size), intent(INOUT) :: qK_cons_vf + type(scalar_field), dimension(sys_size), intent(INOUT) :: qK_prim_vf type(scalar_field), & allocatable, optional, dimension(:), & intent(IN) :: gm_alphaK_vf - type(int_bounds_info), optional, intent(IN) :: ix, iy, iz + type(int_bounds_info), optional, intent(IN) :: ix, iy, iz real(kind(0d0)), dimension(num_fluids) :: alpha_K, alpha_rho_K real(kind(0d0)), dimension(2) :: Re_K @@ -793,12 +774,13 @@ contains allocate(nRtmp(0)) endif -!$acc parallel loop collapse(3) gang vector default(present) private(alpha_K, alpha_rho_K, Re_K, nRtmp, rho_K, gamma_K, pi_inf_K, dyn_pres_K) + !$acc parallel loop collapse(3) gang vector default(present) private(alpha_K, alpha_rho_K, Re_K, nRtmp, rho_K, gamma_K, pi_inf_K, dyn_pres_K) do l = izb, ize do k = iyb, iye do j = ixb, ixe dyn_pres_K = 0d0 -!$acc loop seq + + !$acc loop seq do i = 1, num_fluids alpha_rho_K(i) = qK_cons_vf(i)%sf(j, k, l) alpha_K(i) = qK_cons_vf(advxb + i - 1)%sf(j, k, l) @@ -832,7 +814,7 @@ contains #ifdef MFC_SIMULATION rho_K = max(rho_K, sgm_eps) #endif -!$acc loop seq + !$acc loop seq do i = momxb, momxe if (model_eqns /= 4) then qK_prim_vf(i)%sf(j, k, l) = qK_cons_vf(i)%sf(j, k, l) & @@ -851,7 +833,7 @@ contains qK_prim_vf(E_idx)%sf(j, k, l) = pres if (bubbles) then -!$acc loop seq + !$acc loop seq do i = 1, nb nRtmp(i) = qK_cons_vf(bubrs(i))%sf(j, k, l) end do @@ -859,14 +841,15 @@ contains vftmp = qK_cons_vf(alf_idx)%sf(j, k, l) call s_comp_n_from_cons(vftmp, nRtmp, nbub_sc) -!$acc loop seq + + !$acc loop seq do i = bubxb, bubxe qK_prim_vf(i)%sf(j, k, l) = qK_cons_vf(i)%sf(j, k, l)/nbub_sc end do end if if (hypoelasticity) then -!$acc loop seq + !$acc loop seq do i = strxb, strxe qK_prim_vf(i)%sf(j, k, l) = qK_cons_vf(i)%sf(j, k, l) & /rho_K @@ -887,7 +870,7 @@ contains end do end do end do -!$acc end parallel loop + !$acc end parallel loop end subroutine s_convert_conservative_to_primitive_variables ! --------- @@ -1172,13 +1155,14 @@ contains ! Deallocating the density, the specific heat ratio function and the ! liquid stiffness function #ifdef MFC_POST_PROCESS - deallocate (rho_sf) - deallocate (gamma_sf) - deallocate (pi_inf_sf) + deallocate(rho_sf, gamma_sf, pi_inf_sf) #endif - deallocate (gammas, pi_infs, Gs) - deallocate (bubrs) + @:DEALLOCATE(gammas, pi_infs, Gs) + + if (bubbles) then + @:DEALLOCATE(bubrs) + end if ! Nullifying the procedure pointer to the subroutine transfering/ ! computing the mixture/species variables to the mixture variables @@ -1186,5 +1170,4 @@ contains end subroutine s_finalize_variables_conversion_module ! ---------------- - -end module m_variables_conversion \ No newline at end of file +end module m_variables_conversion diff --git a/src/common/macros.fpp b/src/common/macros.fpp new file mode 100644 index 0000000000..def5f139cb --- /dev/null +++ b/src/common/macros.fpp @@ -0,0 +1,44 @@ +#:def LOG(expr) +#ifdef MFC_DEBUG +print *, '${_FILE_.split('/')[-1]}$:${_LINE_}$', ${expr}$ +#endif +#:enddef + + +#:def ALLOCATE(*args) + +! ================================================ +! ==== BEGIN ALLOCATE (src/common/macros.fpp) ==== +! ================================================ + +#! @:LOG({'@:ALLOCATE(${', '.join(args)}$)'}) + +allocate(${', '.join(args)}$) + +!$acc enter data create(${', '.join(args)}$) + +! ================================================ +! ==== END ALLOCATE (src/common/macros.fpp) ==== +! ================================================ + +#:enddef ALLOCATE + + + +#:def DEALLOCATE(*args) + +! ================================================== +! ==== BEGIN DEALLOCATE (src/common/macros.fpp) ==== +! ================================================== + +#! @:LOG({'@:DEALLOCATE(${', '.join(args)}$)'}) + +deallocate(${', '.join(args)}$) + +!$acc exit data delete(${', '.join(args)}$) + +! ================================================== +! ==== END DEALLOCATE (src/common/macros.fpp) ==== +! ================================================== + +#:enddef DEALLOCATE diff --git a/src/post_process/case.fpp b/src/post_process/case.fpp deleted file mode 100644 index 3fd2045031..0000000000 --- a/src/post_process/case.fpp +++ /dev/null @@ -1,5 +0,0 @@ -! This file was generated by MFC. It is only used if the --case-optimization -! option is passed to ./mfc.sh run or test, enabling a GPU-oriented optimization -! that hard-codes certain case parameters from the input file. - -#:set MFC_CASE_OPTIMIZATION = False diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 9ef4044825..5017576aa1 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -221,11 +221,7 @@ subroutine s_read_parallel_data_files(t_step) ! --------------------------- integer, intent(IN) :: t_step -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_read_parallel_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI real(kind(0d0)), allocatable, dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb diff --git a/src/post_process/m_derived_variables.f90 b/src/post_process/m_derived_variables.f90 index 61fa89550a..16513f4a71 100644 --- a/src/post_process/m_derived_variables.f90 +++ b/src/post_process/m_derived_variables.f90 @@ -464,7 +464,7 @@ subroutine s_derive_curvature(i, q_prim_vf, q_sf) ! -------------------- integer :: stencil_j_min, stencil_j_max integer :: stencil_k_min, stencil_k_max integer :: stencil_l_min, stencil_l_max - type(bounds_info) :: iz1 + type(int_bounds_info) :: iz1 if (p > 0) then ! 3D simulation allocate (A(10, 10)) diff --git a/src/post_process/m_global_parameters.f90 b/src/post_process/m_global_parameters.f90 index aea7c5aad8..1c201bd93a 100644 --- a/src/post_process/m_global_parameters.f90 +++ b/src/post_process/m_global_parameters.f90 @@ -99,16 +99,16 @@ module m_global_parameters !> @name Annotations of the structure, i.e. the organization, of the state vectors !> @{ - type(bounds_info) :: cont_idx !< Indexes of first & last continuity eqns. - type(bounds_info) :: mom_idx !< Indexes of first & last momentum eqns. - integer :: E_idx !< Index of energy equation - type(bounds_info) :: adv_idx !< Indexes of first & last advection eqns. - type(bounds_info) :: internalEnergies_idx !< Indexes of first & last internal energy eqns. + type(int_bounds_info) :: cont_idx !< Indexes of first & last continuity eqns. + type(int_bounds_info) :: mom_idx !< Indexes of first & last momentum eqns. + integer :: E_idx !< Index of energy equation + type(int_bounds_info) :: adv_idx !< Indexes of first & last advection eqns. + type(int_bounds_info) :: internalEnergies_idx !< Indexes of first & last internal energy eqns. type(bub_bounds_info) :: bub_idx !< Indexes of first & last bubble variable eqns. - integer :: gamma_idx !< Index of specific heat ratio func. eqn. - integer :: alf_idx !< Index of specific heat ratio func. eqn. - integer :: pi_inf_idx !< Index of liquid stiffness func. eqn. - type(bounds_info) :: stress_idx !< Indices of elastic stresses + integer :: gamma_idx !< Index of specific heat ratio func. eqn. + integer :: alf_idx !< Index of specific heat ratio func. eqn. + integer :: pi_inf_idx !< Index of liquid stiffness func. eqn. + type(int_bounds_info) :: stress_idx !< Indices of elastic stresses !> @} !> @name Boundary conditions in the x-, y- and z-coordinate directions @@ -262,11 +262,8 @@ subroutine s_assign_default_values_to_user_inputs() ! ------------------ case_dir = ' ' ! Computational domain parameters - m = dflt_int + m = dflt_int; n = 0; p = 0 m_root = dflt_int - n = dflt_int - p = dflt_int - cyl_coord = .false. t_step_start = dflt_int @@ -783,11 +780,7 @@ subroutine s_initialize_parallel_io() ! -------------------------------- if (parallel_io .neqv. .true.) return -#ifndef MFC_MPI - - print '(A)', '[m_global_parameters] s_initialize_parallel_io not supported without MPI.' - -#else +#ifdef MFC_MPI ! Option for Lustre file system (Darter/Comet/Stampede) write (mpiiofs, '(A)') '/lustre_' diff --git a/src/post_process/m_mpi_proxy.f90 b/src/post_process/m_mpi_proxy.f90 index eee0008785..6b37e0e269 100644 --- a/src/post_process/m_mpi_proxy.f90 +++ b/src/post_process/m_mpi_proxy.f90 @@ -101,11 +101,7 @@ subroutine s_initialize_mpi_data(q_cons_vf) ! -------------------------- dimension(sys_size), & intent(IN) :: q_cons_vf -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_initialize_mpi_data not supported without MPI.' - -#else +#ifdef MFC_MPI integer, dimension(num_dims) :: sizes_glb, sizes_loc integer :: ierr @@ -139,11 +135,7 @@ end subroutine s_initialize_mpi_data ! --------------------------------- !>Halts all processes until all have reached barrier. subroutine s_mpi_barrier() ! ------------------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_barrier not supported without MPI.' - -#else +#ifdef MFC_MPI ! Calling MPI_BARRIER call MPI_BARRIER(MPI_COMM_WORLD, ierr) @@ -156,11 +148,7 @@ end subroutine s_mpi_barrier ! ----------------------------------------- !! any other tasks needed to properly setup the module subroutine s_initialize_mpi_proxy_module() ! ------------------------------ -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_initialize_mpi_proxy_module not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: i !< Generic loop iterator @@ -256,11 +244,7 @@ end subroutine s_initialize_mpi_proxy_module ! ---------------------------- !! information. subroutine s_mpi_bcast_user_inputs() ! --------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_bcast_user_inputs not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: i !< Generic loop iterator @@ -422,11 +406,7 @@ end subroutine s_mpi_bcast_user_inputs ! ------------------------------- !! is overseen by the local processor. subroutine s_mpi_decompose_computational_domain() ! -------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_decompose_computational_domain not supported without MPI.' - -#else +#ifdef MFC_MPI ! # of processors in the x-, y- and z-coordinate directions integer :: num_procs_x, num_procs_y, num_procs_z @@ -874,11 +854,7 @@ subroutine s_mpi_sendrecv_grid_vars_buffer_regions(pbc_loc, sweep_coord) character(LEN=3), intent(IN) :: pbc_loc character, intent(IN) :: sweep_coord -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_sendrecv_grid_vars_buffer_regions not supported without MPI.' - -#else +#ifdef MFC_MPI ! Communications in the x-direction ================================ @@ -1083,11 +1059,7 @@ subroutine s_mpi_sendrecv_cons_vars_buffer_regions(q_cons_vf, pbc_loc, & character, intent(IN) :: sweep_coord -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_sendrecv_cons_vars_buffer_regions not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: i, j, k, l, r !< Generic loop iterators @@ -1615,11 +1587,7 @@ subroutine s_mpi_reduce_maxloc(var_loc) ! ------------------------------ real(kind(0d0)), dimension(2), intent(INOUT) :: var_loc -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_reduce_maxloc not supported without MPI.' - -#else +#ifdef MFC_MPI real(kind(0d0)), dimension(2) :: var_glb !< !! Temporary storage variable that holds the reduced maximum value @@ -1649,11 +1617,7 @@ subroutine s_mpi_gather_spatial_extents(spatial_extents) ! ------------- real(kind(0d0)), dimension(1:, 0:), intent(INOUT) :: spatial_extents -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_gather_spatial_extents not supported without MPI.' - -#else +#ifdef MFC_MPI ! Simulation is 3D if (p > 0) then @@ -1770,11 +1734,7 @@ end subroutine s_mpi_gather_spatial_extents ! -------------------------- !! simulations. subroutine s_mpi_defragment_1d_grid_variable() ! ----------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_defragment_1d_grid_variable not supported without MPI.' - -#else +#ifdef MFC_MPI ! Silo-HDF5 database format if (format == 1) then @@ -1816,11 +1776,7 @@ subroutine s_mpi_gather_data_extents(q_sf, data_extents) ! ------------- dimension(1:2, 0:num_procs - 1), & intent(INOUT) :: data_extents -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_gather_data_extents not supported without MPI.' - -#else +#ifdef MFC_MPI ! Mimimum flow variable extent call MPI_GATHERV(minval(q_sf), 1, MPI_DOUBLE_PRECISION, & @@ -1852,11 +1808,7 @@ subroutine s_mpi_defragment_1d_flow_variable(q_sf, q_root_sf) ! -------- dimension(0:m_root, 0:0, 0:0), & intent(INOUT) :: q_root_sf -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_defragment_1d_flow_variable not supported without MPI.' - -#else +#ifdef MFC_MPI ! Gathering the sub-domain flow variable data from all the processes ! and putting it back together for the entire computational domain @@ -1872,11 +1824,7 @@ end subroutine s_mpi_defragment_1d_flow_variable ! --------------------- !> Deallocation procedures for the module subroutine s_finalize_mpi_proxy_module() ! --------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_finalize_mpi_proxy_module not supported without MPI.' - -#else +#ifdef MFC_MPI ! Deallocating the conservative variables buffer vectors if (buff_size > 0) then @@ -1898,11 +1846,7 @@ end subroutine s_finalize_mpi_proxy_module ! ------------------------- !> Finalization of all MPI related processes subroutine s_mpi_finalize() ! ------------------------------ -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_finalize not supported without MPI.' - -#else +#ifdef MFC_MPI ! Terminating the MPI environment call MPI_FINALIZE(ierr) diff --git a/src/post_process/p_main.f90 b/src/post_process/p_main.f90 index 7db7301ab6..d1981fda24 100644 --- a/src/post_process/p_main.f90 +++ b/src/post_process/p_main.f90 @@ -63,6 +63,8 @@ program p_main call s_assign_default_values_to_user_inputs() call s_read_input_file() call s_check_input_file() + + print '(" Post-processing a "I0"x"I0"x"I0" case on "I0" rank(s)")', m, n, p, num_procs end if ! Broadcasting the user inputs to all of the processors and performing the @@ -93,6 +95,13 @@ program p_main ! Time-Marching Loop ======================================================= do + if (proc_rank == 0) then + print '(" ["I3"%] Save "I8" of "I0" @ t_step = "I0"")', & + int(100*(real(t_step + 1)/(t_step_stop - t_step_start + 1))), & + (t_step - t_step_start)/t_step_save + 1, & + (t_step_stop - t_step_start)/t_step_save + 1, & + t_step + end if ! Populating the grid and conservative variables call s_read_data_files(t_step) diff --git a/src/pre_process/m_data_output.f90 b/src/pre_process/m_data_output.f90 index 706f81d9f3..a4d4c08cbf 100644 --- a/src/pre_process/m_data_output.f90 +++ b/src/pre_process/m_data_output.f90 @@ -247,11 +247,7 @@ subroutine s_write_parallel_data_files(q_cons_vf) ! -- dimension(sys_size), & intent(IN) :: q_cons_vf -#ifndef MFC_MPI - - print '(A)', '[m_data_output] s_write_parallel_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status diff --git a/src/pre_process/m_global_parameters.fpp b/src/pre_process/m_global_parameters.fpp index 2b7fc03a2c..867c016950 100644 --- a/src/pre_process/m_global_parameters.fpp +++ b/src/pre_process/m_global_parameters.fpp @@ -206,9 +206,7 @@ contains t_step_old = dflt_int ! Computational domain parameters - m = dflt_int - n = dflt_int - p = dflt_int + m = dflt_int; n = 0; p = 0 cyl_coord = .false. @@ -775,11 +773,7 @@ contains if (parallel_io .neqv. .true.) return -#ifndef MFC_MPI - - print '(A)', '[m_global_parameters] s_initialize_parallel_io not supported without MPI.' - -#else +#ifdef MFC_MPI ! Option for Lustre file system (Darter/Comet/Stampede) write (mpiiofs, '(A)') '/lustre_' diff --git a/src/pre_process/m_grid.f90 b/src/pre_process/m_grid.f90 index 3e18b25351..8287bc77e3 100644 --- a/src/pre_process/m_grid.f90 +++ b/src/pre_process/m_grid.f90 @@ -203,11 +203,7 @@ subroutine s_generate_parallel_grid(dflt_int) !------------------------- integer, intent(IN) :: dflt_int -#ifndef MFC_MPI - - print '(A)', '[m_grid] s_generate_parallel_grid not supported without MPI.' - -#else +#ifdef MFC_MPI real(kind(0d0)) :: length !< domain lengths diff --git a/src/pre_process/m_mpi_proxy.f90 b/src/pre_process/m_mpi_proxy.f90 index fd8c285896..9657d168f4 100644 --- a/src/pre_process/m_mpi_proxy.f90 +++ b/src/pre_process/m_mpi_proxy.f90 @@ -82,11 +82,7 @@ subroutine s_initialize_mpi_data(q_cons_vf) ! -------------------------- dimension(sys_size), & intent(IN) :: q_cons_vf -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_initialize_mpi_data not supported without MPI.' - -#else +#ifdef MFC_MPI integer, dimension(num_dims) :: sizes_glb, sizes_loc integer :: ierr @@ -121,11 +117,7 @@ end subroutine s_initialize_mpi_data ! --------------------------------- !> Halts all processes until all have reached barrier. subroutine s_mpi_barrier() ! ------------------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_barrier not supported without MPI.' - -#else +#ifdef MFC_MPI ! Calling MPI_BARRIER call MPI_BARRIER(MPI_COMM_WORLD, ierr) @@ -141,11 +133,7 @@ end subroutine s_mpi_barrier ! ----------------------------------------- !! information. subroutine s_mpi_bcast_user_inputs() ! --------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_bcast_user_inputs not supported without MPI.' - -#else +#ifdef MFC_MPI ! Generic loop iterator integer :: i @@ -428,11 +416,7 @@ subroutine mpi_bcast_time_step_values(proc_time, time_avg) real(kind(0d0)), dimension(0:num_procs - 1), intent(INOUT) :: proc_time real(kind(0d0)), intent(INOUT) :: time_avg -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] mpi_bcast_time_step_values not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: j @@ -449,11 +433,7 @@ end subroutine mpi_bcast_time_step_values !! overseen by the local processor. subroutine s_mpi_decompose_computational_domain() ! -------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_decompose_computational_domain not supported without MPI.' - -#else +#ifdef MFC_MPI ! # of processors in the x-, y- and z-coordinate directions integer :: num_procs_x, num_procs_y, num_procs_z @@ -873,11 +853,7 @@ subroutine s_mpi_reduce_min(var_loc) ! --------------------------------- real(kind(0d0)), intent(INOUT) :: var_loc -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_reduce_min not supported without MPI.' - -#else +#ifdef MFC_MPI ! Temporary storage variable that holds the reduced minimum value real(kind(0d0)) :: var_glb @@ -899,11 +875,7 @@ end subroutine s_mpi_reduce_min ! -------------------------------------- !> Finalization of all MPI related processes subroutine s_mpi_finalize() ! ------------------------------ -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_finalize not supported without MPI.' - -#else +#ifdef MFC_MPI ! Terminating the MPI environment call MPI_FINALIZE(ierr) diff --git a/src/pre_process/m_start_up.fpp b/src/pre_process/m_start_up.fpp index 2cc1a873cb..38239c5bab 100644 --- a/src/pre_process/m_start_up.fpp +++ b/src/pre_process/m_start_up.fpp @@ -85,8 +85,8 @@ contains !! is or is not present in the designated location ! Namelist for all of the parameters to be inputed by the user - namelist /user_inputs/ case_dir, old_grid, old_ic, t_step_old, m, & - n, p, x_domain, y_domain, z_domain, & + namelist /user_inputs/ case_dir, old_grid, old_ic, & + t_step_old, m, n, p, x_domain, y_domain, z_domain, & stretch_x, stretch_y, stretch_z, a_x, a_y, & a_z, x_a, y_a, z_a, x_b, y_b, z_b, & model_eqns, num_fluids, & @@ -158,6 +158,9 @@ contains 'bubbles and model_eqns. '// & 'Exiting ...' call s_mpi_abort() + elseif (bubbles .and. nb < 1) then + print '(A)', 'The Ensemble-Averaged Bubble Model requires nb >= 1' + call s_mpi_abort() elseif (bubbles .and. polydisperse .and. (nb == 1)) then print '(A)', 'Polydisperse bubble dynamics requires nb > 1 '// & 'Exiting ...' @@ -1912,11 +1915,7 @@ contains integer, intent(IN) :: dflt_int -#ifndef MFC_MPI - - print '(A)', '[m_start_up] s_read_parallel_grid_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI real(kind(0d0)), allocatable, dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb @@ -2027,11 +2026,7 @@ contains dimension(sys_size), & intent(INOUT) :: q_cons_vf -#ifndef MFC_MPI - - print '(A)', '[m_start_up] s_read_parallel_ic_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status diff --git a/src/pre_process/p_main.f90 b/src/pre_process/p_main.f90 index 020fba6f27..d5c516d110 100644 --- a/src/pre_process/p_main.f90 +++ b/src/pre_process/p_main.f90 @@ -49,6 +49,8 @@ program p_main call s_assign_default_values_to_user_inputs() call s_read_input_file() call s_check_input_file() + + print '(" Pre-processing a "I0"x"I0"x"I0" case on "I0" rank(s)")', m, n, p, num_procs end if ! Broadcasting the user inputs to all of the processors and performing the diff --git a/src/simulation/m_bubbles.f90 b/src/simulation/m_bubbles.fpp similarity index 99% rename from src/simulation/m_bubbles.f90 rename to src/simulation/m_bubbles.fpp index 8b71e918ba..cafc7e3ba0 100644 --- a/src/simulation/m_bubbles.f90 +++ b/src/simulation/m_bubbles.fpp @@ -2,6 +2,8 @@ !! @file m_bubbles.f90 !! @brief Contains module m_bubbles +#:include 'macros.fpp' + !> @brief This module is used to compute the ensemble-averaged bubble dynamic variables module m_bubbles @@ -46,11 +48,11 @@ subroutine s_initialize_bubbles_module() integer :: i, j, k, l, q - allocate (rs(1:nb)) - allocate (vs(1:nb)) + @:ALLOCATE(rs(1:nb)) + @:ALLOCATE(vs(1:nb)) if (.not. polytropic) then - allocate (ps(1:nb)) - allocate (ms(1:nb)) + @:ALLOCATE(ps(1:nb)) + @:ALLOCATE(ms(1:nb)) end if do l = 1, nb @@ -64,7 +66,7 @@ subroutine s_initialize_bubbles_module() !$acc update device(rs, vs) if (.not. polytropic) then - !$acc update device(ps, ms) + !$acc update device(ps, ms) end if end subroutine diff --git a/src/simulation/m_data_output.f90 b/src/simulation/m_data_output.fpp similarity index 99% rename from src/simulation/m_data_output.f90 rename to src/simulation/m_data_output.fpp index 0c5be03c0d..a3557e2abc 100644 --- a/src/simulation/m_data_output.f90 +++ b/src/simulation/m_data_output.fpp @@ -2,6 +2,8 @@ !! @file m_data_output.f90 !! @brief Contains module m_data_output +#:include 'macros.fpp' + !> @brief The primary purpose of this module is to output the grid and the !! conservative variables data at the chosen time-step interval. In !! addition, this module is also in charge of outputting a run-time @@ -109,8 +111,6 @@ end subroutine s_write_abstract_data_files ! ------------------- !! time-step. subroutine s_open_run_time_information_file() ! ------------------------ - character(LEN=name_len) :: dir_name !< Name of the case directory - character(LEN=name_len) :: file_name = 'run_time.inf' !< !! Name of the run-time information file @@ -123,8 +123,6 @@ subroutine s_open_run_time_information_file() ! ------------------------ logical :: file_exist !< !! Logical used to check existence of run-time information file - character(LEN=path_len) :: cwd - ! Opening the run-time information file file_path = trim(case_dir)//'/'//trim(file_name) @@ -138,11 +136,6 @@ subroutine s_open_run_time_information_file() ! ------------------------ ! Generating file header for a new run-time information file if (file_exist .neqv. .true.) then - call s_get_cwd(cwd) - call s_get_basename(cwd, dir_name) - - write (1, '(A)') 'MFC - Case - '//trim(dir_name)// & - ': '//trim(file_name) write (1, '(A)') 'Description: Stability information at '// & 'each time-step of the simulation. This' write (1, '(13X,A)') 'data is composed of the inviscid '// & @@ -848,11 +841,7 @@ subroutine s_write_parallel_data_files(q_cons_vf, t_step) ! -- integer, intent(IN) :: t_step -#ifndef MFC_MPI - - print '(A)', '[m_data_output] s_write_parallel_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status @@ -1818,7 +1807,9 @@ subroutine s_write_probe_files(t_step, q_cons_vf, accel_mag) ! ----------- nbub = DSQRT((4.d0*pi/3.d0)*nR3/alf) - if (DEBUG) print *, 'In probe, nbub: ', nbub +#ifdef DEBUG + print *, 'In probe, nbub: ', nbub +#endif if (qbmm) then M00 = q_cons_vf(bub_idx%moms(1, 1))%sf(j - 2, k, l)/nbub @@ -2447,9 +2438,16 @@ subroutine s_initialize_data_output_module() ! ------------------------- ! Allocating/initializing ICFL, VCFL, CCFL and Rc stability criteria - allocate (icfl_sf(0:m, 0:n, 0:p)); icfl_max = 0d0 - if (any(Re_size > 0)) allocate (vcfl_sf(0:m, 0:n, 0:p)); vcfl_max = 0d0 - if (any(Re_size > 0)) allocate (Rc_sf(0:m, 0:n, 0:p)); Rc_min = 1d3 + @:ALLOCATE(icfl_sf(0:m, 0:n, 0:p)) + icfl_max = 0d0 + + if (any(Re_size > 0)) then + @:ALLOCATE(vcfl_sf(0:m, 0:n, 0:p)) + @:ALLOCATE(Rc_sf (0:m, 0:n, 0:p)) + + vcfl_max = 0d0 + Rc_min = 1d3 + end if ! Associating the procedural pointer to the appropriate subroutine ! that will be utilized in the conversion to the mixture variables @@ -2508,29 +2506,27 @@ subroutine s_finalize_data_output_module() ! --------------------------- integer :: i !< Generic loop iterator ! Deallocating the ICFL, VCFL, CCFL, and Rc stability criteria - deallocate (icfl_sf) - if (any(Re_size > 0)) deallocate (vcfl_sf) - if (any(Re_size > 0)) deallocate (Rc_sf) + @:DEALLOCATE(icfl_sf) + if (any(Re_size > 0)) then + @:DEALLOCATE(vcfl_sf, Rc_sf) + end if ! Deallocating the storage employed for the flow variables that ! were written to CoM and probe files if (any(com_wrt)) then - deallocate (q_com) - deallocate (moments) + deallocate(q_com, moments) end if if (any(cb_wrt)) then - deallocate (cb_mass) - deallocate (bounds) - deallocate (cntrline) + deallocate(cb_mass, bounds, cntrline) end if + if (probe_wrt) then - deallocate (accel_mag) - deallocate (x_accel) + deallocate(accel_mag, x_accel) if (n > 0) then - deallocate (y_accel) + deallocate(y_accel) if (p > 0) then - deallocate (z_accel) + deallocate(z_accel) end if end if end if diff --git a/src/simulation/m_fftw.fpp b/src/simulation/m_fftw.fpp index cb1bd253e5..f4ee99bb92 100644 --- a/src/simulation/m_fftw.fpp +++ b/src/simulation/m_fftw.fpp @@ -2,6 +2,8 @@ !! @file m_fftw.f90 !! @brief Contains module m_fftw +#:include 'macros.fpp' + !> @brief The module contains the subroutines for the FFT routines module m_fftw @@ -42,19 +44,18 @@ module m_fftw !! Filtered complex data in Fourier space #if defined(_OPENACC) && defined(__PGI) - real(kind(0d0)), pointer :: data_real_gpu(:) - - complex(kind(0d0)), pointer :: data_cmplx_gpu(:) + !$acc declare create(real_size, cmplx_size, x_size, batch_size) - complex(kind(0d0)), pointer :: data_fltr_cmplx_gpu(:) + real(kind(0d0)), allocatable :: data_real_gpu(:) + complex(kind(0d0)), allocatable :: data_cmplx_gpu(:) + complex(kind(0d0)), allocatable :: data_fltr_cmplx_gpu(:) + !$acc declare create(data_real_gpu, data_cmplx_gpu, data_fltr_cmplx_gpu) integer :: fwd_plan_gpu, bwd_plan_gpu, ierr integer, allocatable :: cufft_size(:), iembed(:), oembed(:) integer :: istride, ostride, idist, odist, rank - - !$acc declare create(real_size, cmplx_size, x_size, batch_size, data_real_gpu, data_cmplx_gpu, data_fltr_cmplx_gpu) #endif contains @@ -83,7 +84,7 @@ contains #if defined(_OPENACC) && defined(__PGI) rank = 1; istride = 1; ostride = 1 - allocate (cufft_size(1:rank), iembed(1:rank), oembed(1:rank)) + allocate(cufft_size(1:rank), iembed(1:rank), oembed(1:rank)) cufft_size(1) = real_size; iembed(1) = 0 @@ -104,14 +105,16 @@ contains fwd_plan = fftw_plan_dft_r2c_1d(real_size, data_real, data_cmplx, FFTW_ESTIMATE) bwd_plan = fftw_plan_dft_c2r_1d(real_size, data_fltr_cmplx, data_real, FFTW_ESTIMATE) #endif + #if defined(_OPENACC) && defined(__PGI) - allocate (data_real_gpu(1:real_size*x_size*sys_size)) - allocate (data_cmplx_gpu(1:cmplx_size*x_size*sys_size)) - allocate (data_fltr_cmplx_gpu(1:cmplx_size*x_size*sys_size)) + @:ALLOCATE(data_real_gpu(1:real_size*x_size*sys_size)) + @:ALLOCATE(data_cmplx_gpu(1:cmplx_size*x_size*sys_size)) + @:ALLOCATE(data_fltr_cmplx_gpu(1:cmplx_size*x_size*sys_size)) -ierr = cufftPlanMany(fwd_plan_gpu, rank, cufft_size, iembed, istride, real_size, oembed, ostride, cmplx_size, CUFFT_D2Z, batch_size) -ierr = cufftPlanMany(bwd_plan_gpu, rank, cufft_size, iembed, istride, cmplx_size, oembed, ostride, real_size, CUFFT_Z2D, batch_size) + ierr = cufftPlanMany(fwd_plan_gpu, rank, cufft_size, iembed, istride, real_size, oembed, ostride, cmplx_size, CUFFT_D2Z, batch_size) + ierr = cufftPlanMany(bwd_plan_gpu, rank, cufft_size, iembed, istride, cmplx_size, oembed, ostride, real_size, CUFFT_Z2D, batch_size) #endif + end subroutine s_initialize_fftw_module ! ------------------------------ !> The purpose of this subroutine is to apply a Fourier low- @@ -268,7 +271,7 @@ ierr = cufftPlanMany(bwd_plan_gpu, rank, cufft_size, iembed, istride, cmplx_size subroutine s_finalize_fftw_module() ! ------------------------------------ #if defined(_OPENACC) && defined(__PGI) - deallocate (data_real_gpu, data_fltr_cmplx_gpu, data_cmplx_gpu) + @:DEALLOCATE(data_real_gpu, data_fltr_cmplx_gpu, data_cmplx_gpu) ierr = cufftDestroy(fwd_plan_gpu) ierr = cufftDestroy(bwd_plan_gpu) #else diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp index 534400dddf..054d45688c 100644 --- a/src/simulation/m_global_parameters.fpp +++ b/src/simulation/m_global_parameters.fpp @@ -3,6 +3,7 @@ !! @brief Contains module m_global_parameters #:include 'case.fpp' +#:include 'macros.fpp' !> @brief The module contains all of the parameters describing the program !! logistics, the computational domain and the simulation algorithm. @@ -39,7 +40,6 @@ module m_global_parameters integer, parameter :: fourier_rings = 5 !< Fourier filter ring limit character(LEN=path_len) :: case_dir !< Case folder location logical :: run_time_info !< Run-time output flag - logical :: debug !< Debug mode print statements integer :: t_step_old !< Existing IC/grid folder real(kind(0d0)), parameter :: small_alf = 1d-7 !< Small alf tolerance ! ========================================================================== @@ -170,7 +170,7 @@ module m_global_parameters integer :: alf_idx !< Index of void fraction integer :: gamma_idx !< Index of specific heat ratio func. eqn. integer :: pi_inf_idx !< Index of liquid stiffness func. eqn. - type(bounds_info) :: stress_idx !< Indexes of first and last shear stress eqns. + type(int_bounds_info) :: stress_idx !< Indexes of first and last shear stress eqns. !> @} !$acc declare create(bub_idx) @@ -341,10 +341,8 @@ contains run_time_info = .false. t_step_old = dflt_int - debug = .false. - ! Computational domain parameters - m = dflt_int; n = dflt_int; p = dflt_int + m = dflt_int; n = 0; p = 0 cyl_coord = .false. @@ -413,6 +411,7 @@ contains nb = dflt_int weno_order = dflt_int #:endif + R0_type = dflt_int ! User inputs for qbmm for simulation code @@ -569,9 +568,9 @@ contains ! print*, 'alf idx', alf_idx ! print*, 'bub -idx beg end', bub_idx%beg, bub_idx%end - allocate (weight(nb), R0(nb), V0(nb)) - allocate (bub_idx%rs(nb), bub_idx%vs(nb)) - allocate (bub_idx%ps(nb), bub_idx%ms(nb)) + @:ALLOCATE(weight(nb), R0(nb), V0(nb)) + @:ALLOCATE(bub_idx%rs(nb), bub_idx%vs(nb)) + @:ALLOCATE(bub_idx%ps(nb), bub_idx%ms(nb)) if (num_fluids == 1) then gam = 1.d0/fluid_pp(num_fluids + 1)%gamma + 1.d0 @@ -580,7 +579,7 @@ contains end if if (qbmm) then - allocate (bub_idx%moms(nb, nmom)) + @:ALLOCATE(bub_idx%moms(nb, nmom)) do i = 1, nb do j = 1, nmom bub_idx%moms(i, j) = bub_idx%beg + (j - 1) + (i - 1)*nmom @@ -670,9 +669,9 @@ contains end if sys_size = bub_idx%end - allocate (bub_idx%rs(nb), bub_idx%vs(nb)) - allocate (bub_idx%ps(nb), bub_idx%ms(nb)) - allocate (weight(nb), R0(nb), V0(nb)) + @:ALLOCATE(bub_idx%rs(nb), bub_idx%vs(nb)) + @:ALLOCATE(bub_idx%ps(nb), bub_idx%ms(nb)) + @:ALLOCATE(weight(nb), R0(nb), V0(nb)) do i = 1, nb if (polytropic) then @@ -725,7 +724,7 @@ contains ! fluids whose interface will support effects of surface tension if (any(Re_size > 0)) then - allocate (Re_idx(1:2, 1:maxval(Re_size))) + @:ALLOCATE(Re_idx(1:2, 1:maxval(Re_size))) k = 0 do i = 1, num_fluids @@ -770,11 +769,16 @@ contains ! Configuring Coordinate Direction Indexes ========================= if (bubbles) then ix%beg = -buff_size; iy%beg = 0; iz%beg = 0 - if (n > 0) iy%beg = -buff_size; if (p > 0) iz%beg = -buff_size + if (n > 0) then + iy%beg = -buff_size + if (p > 0) then + iz%beg = -buff_size + end if + end if + ix%end = m - ix%beg; iy%end = n - iy%beg; iz%end = p - iz%beg - allocate (ptil(ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) + + @:ALLOCATE(ptil(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end if if (probe_wrt) then @@ -819,20 +823,21 @@ contains !$acc update device(momxb, momxe, advxb, advxe, contxb, contxe, bubxb, bubxe, intxb, intxe, sys_size, buff_size, E_idx, alf_idx, strxb, strxe) ! Allocating grid variables for the x-, y- and z-directions - allocate (x_cb(-1 - buff_size:m + buff_size)) - allocate (x_cc(-buff_size:m + buff_size)) - allocate (dx(-buff_size:m + buff_size)) - - if (n == 0) return; allocate (y_cb(-1 - buff_size:n + buff_size)) - allocate (y_cc(-buff_size:n + buff_size)) - allocate (dy(-buff_size:n + buff_size)) - - if (p == 0) return; allocate (z_cb(-1 - buff_size:p + buff_size)) - allocate (z_cc(-buff_size:p + buff_size)) - allocate (dz(-buff_size:p + buff_size)) - - - + @:ALLOCATE(x_cb(-1 - buff_size:m + buff_size)) + @:ALLOCATE(x_cc(-buff_size:m + buff_size)) + @:ALLOCATE(dx(-buff_size:m + buff_size)) + + if (n == 0) return; + + @:ALLOCATE(y_cb(-1 - buff_size:n + buff_size)) + @:ALLOCATE(y_cc(-buff_size:n + buff_size)) + @:ALLOCATE(dy(-buff_size:n + buff_size)) + + if (p == 0) return; + + @:ALLOCATE(z_cb(-1 - buff_size:p + buff_size)) + @:ALLOCATE(z_cc(-buff_size:p + buff_size)) + @:ALLOCATE(dz(-buff_size:p + buff_size)) end subroutine s_initialize_global_parameters_module ! ----------------- @@ -860,9 +865,9 @@ contains rhol0 = rhoref pl0 = pref - allocate (pb0(nb), mass_n0(nb), mass_v0(nb), Pe_T(nb)) - allocate (k_n(nb), k_v(nb), omegaN(nb)) - allocate (Re_trans_T(nb), Re_trans_c(nb), Im_trans_T(nb), Im_trans_c(nb)) + @:ALLOCATE(pb0(nb), mass_n0(nb), mass_v0(nb), Pe_T(nb)) + @:ALLOCATE(k_n(nb), k_v(nb), omegaN(nb)) + @:ALLOCATE(Re_trans_T(nb), Re_trans_c(nb), Im_trans_T(nb), Im_trans_c(nb)) pb0(:) = dflt_real mass_n0(:) = dflt_real @@ -989,11 +994,7 @@ contains if (parallel_io .neqv. .true.) return -#ifndef MFC_MPI - - print '(A)', '[m_global_parameters] s_initialize_parallel_io not supported without MPI.' - -#else +#ifdef MFC_MPI ! Option for Lustre file system (Darter/Comet/Stampede) write (mpiiofs, '(A)') '/lustre_' @@ -1021,14 +1022,18 @@ contains ! Deallocating the variables bookkeeping the indexes of any viscous ! fluids and any pairs of fluids whose interfaces supported effects ! of surface tension - if (any(Re_size > 0)) deallocate (Re_idx) + if (any(Re_size > 0)) then + @:DEALLOCATE(Re_idx) + end if ! Deallocating grid variables for the x-, y- and z-directions - deallocate (x_cb, x_cc, dx) - - if (n == 0) return; deallocate (y_cb, y_cc, dy) + @:DEALLOCATE(x_cb, x_cc, dx) + + if (n == 0) return; + @:DEALLOCATE(y_cb, y_cc, dy) - if (p == 0) return; deallocate (z_cb, z_cc, dz) + if (p == 0) return; + @:DEALLOCATE(z_cb, z_cc, dz) deallocate (proc_coords) if (parallel_io) then diff --git a/src/simulation/m_hypoelastic.f90 b/src/simulation/m_hypoelastic.fpp similarity index 96% rename from src/simulation/m_hypoelastic.f90 rename to src/simulation/m_hypoelastic.fpp index 61cc26bf23..0aef75cd5e 100644 --- a/src/simulation/m_hypoelastic.f90 +++ b/src/simulation/m_hypoelastic.fpp @@ -2,6 +2,8 @@ !! @file m_hypoelastic.f90 !! @brief Contains module m_hypoelastic +#:include 'macros.fpp' + !> @brief This module is used to compute source terms for hypoelastic model module m_hypoelastic @@ -22,7 +24,7 @@ module m_hypoelastic real(kind(0d0)), allocatable, dimension(:) :: Gs -!$acc declare create( Gs) +!$acc declare create(Gs) real(kind(0d0)), allocatable, dimension(:, :, :) :: du_dx, du_dy, du_dz @@ -40,19 +42,16 @@ subroutine s_initialize_hypoelastic_module() ! -------------------- integer :: i - allocate(Gs(1:num_fluids)) - allocate(rho_K_field(0:m,0:n,0:p), G_K_field(0:m,0:n,0:p)) - allocate(du_dx(0:m,0:n,0:p)) + @:ALLOCATE(Gs(1:num_fluids)) + @:ALLOCATE(rho_K_field(0:m,0:n,0:p), G_K_field(0:m,0:n,0:p)) + @:ALLOCATE(du_dx(0:m,0:n,0:p)) if (n > 0) then - allocate(du_dy(0:m,0:n,0:p), dv_dx(0:m,0:n,0:p), dv_dy(0:m,0:n,0:p)) - !$acc enter data create(du_dy, dv_dx, dv_dy) + @:ALLOCATE(du_dy(0:m,0:n,0:p), dv_dx(0:m,0:n,0:p), dv_dy(0:m,0:n,0:p)) if (p > 0) then - allocate(du_dz(0:m,0:n,0:p), dv_dz(0:m,0:n,0:p)) - allocate(dw_dx(0:m,0:n,0:p), dw_dy(0:m,0:n,0:p), dw_dz(0:m,0:n,0:p)) - !$acc enter data create(du_dz, dv_dz, dw_dx, dw_dy, dw_dz) + @:ALLOCATE(du_dz(0:m,0:n,0:p), dv_dz(0:m,0:n,0:p)) + @:ALLOCATE(dw_dx(0:m,0:n,0:p), dw_dy(0:m,0:n,0:p), dw_dz(0:m,0:n,0:p)) end if end if - !$acc enter data create(rho_K_field,G_K_field,du_dx) do i = 1, num_fluids diff --git a/src/simulation/m_monopole.f90 b/src/simulation/m_monopole.fpp similarity index 98% rename from src/simulation/m_monopole.f90 rename to src/simulation/m_monopole.fpp index 7ba782360c..55bc808f90 100644 --- a/src/simulation/m_monopole.f90 +++ b/src/simulation/m_monopole.fpp @@ -2,6 +2,8 @@ !! @file m_viscous.f90 !! @brief Contains module m_viscous +#:include 'macros.fpp' + !> @brief The module contains the subroutines used to create a monopole pressure source term module m_monopole @@ -37,7 +39,7 @@ module m_monopole subroutine s_initialize_monopole_module() integer :: i, j !< generic loop variables - allocate(mag(1:num_mono), support(1:num_mono), length(1:num_mono), npulse(1:num_mono), pulse(1:num_mono), dir(1:num_mono), delay(1:num_mono), loc_mono(1:3, 1:num_mono), foc_length(1:num_mono), aperture(1:num_mono)) + @:ALLOCATE(mag(1:num_mono), support(1:num_mono), length(1:num_mono), npulse(1:num_mono), pulse(1:num_mono), dir(1:num_mono), delay(1:num_mono), loc_mono(1:3, 1:num_mono), foc_length(1:num_mono), aperture(1:num_mono)) do i = 1, num_mono mag(i) = mono(i)%mag diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index 0c2cdf8388..f472d8c9e0 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -3,6 +3,7 @@ !! @brief Contains module m_mpi_proxy #:include 'case.fpp' +#:include 'macros.fpp' !> @brief The module serves as a proxy to the parameters and subroutines !! available in the MPI implementation's MPI module. Specifically, @@ -108,11 +109,7 @@ contains integer :: i !< Generic loop iterator -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_initialize_mpi_data not supported without MPI.' - -#else +#ifdef MFC_MPI do i = 1, sys_size MPI_IO_DATA%var(i)%sf => q_cons_vf(i)%sf(0:m, 0:n, 0:p) @@ -141,11 +138,7 @@ contains !> Halts all processes until all have reached barrier. subroutine s_mpi_barrier() ! ------------------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_barrier not supported without MPI.' - -#else +#ifdef MFC_MPI ! Calling MPI_BARRIER call MPI_BARRIER(MPI_COMM_WORLD, ierr) @@ -159,36 +152,28 @@ contains !! other procedures that are necessary to setup the module. subroutine s_initialize_mpi_proxy_module() ! --------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_initialize_mpi_proxy_module not supported without MPI.' - -#else +#ifdef MFC_MPI ! Allocating q_cons_buff_send and q_cons_buff_recv. Please note that ! for the sake of simplicity, both variables are provided sufficient ! storage to hold the largest buffer in the computational domain. if (n > 0) then - if (p > 0) then - allocate (q_cons_buff_send(0:-1 + buff_size*sys_size* & - (m + 2*buff_size + 1)* & - (n + 2*buff_size + 1)* & - (p + 2*buff_size + 1)/ & - (min(m, n, p) + 2*buff_size + 1))) + @:ALLOCATE(q_cons_buff_send(0:-1 + buff_size*sys_size* & + & (m + 2*buff_size + 1)* & + & (n + 2*buff_size + 1)* & + & (p + 2*buff_size + 1)/ & + & (min(m, n, p) + 2*buff_size + 1))) else - allocate (q_cons_buff_send(0:-1 + buff_size*sys_size* & - (max(m, n) + 2*buff_size + 1))) + @:ALLOCATE(q_cons_buff_send(0:-1 + buff_size*sys_size* & + & (max(m, n) + 2*buff_size + 1))) end if - else - - allocate (q_cons_buff_send(0:-1 + buff_size*sys_size)) - + @:ALLOCATE(q_cons_buff_send(0:-1 + buff_size*sys_size)) end if - allocate (q_cons_buff_recv(0:ubound(q_cons_buff_send, 1))) + @:ALLOCATE(q_cons_buff_recv(0:ubound(q_cons_buff_send, 1))) #endif @@ -201,11 +186,7 @@ contains !! remaining processors in the communicator. subroutine s_mpi_bcast_user_inputs() ! --------------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_bcast_user_inputs not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: i, j !< Generic loop iterator @@ -234,9 +215,6 @@ contains call MPI_BCAST(t_step_stop, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(t_step_save, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(debug, 1, MPI_LOGICAL, & - 0, MPI_COMM_WORLD, ierr) - ! Simulation algorithm parameters call MPI_BCAST(model_eqns, 1, MPI_INTEGER, & 0, MPI_COMM_WORLD, ierr) @@ -487,11 +465,7 @@ contains real(kind(0d0)), dimension(0:num_procs - 1), intent(INOUT) :: proc_time real(kind(0d0)), intent(INOUT) :: time_avg -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] mpi_bcast_time_step_values not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: j @@ -509,11 +483,7 @@ contains !! global parameters. subroutine s_mpi_decompose_computational_domain() ! -------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_decompose_computational_domain not supported without MPI.' - -#else +#ifdef MFC_MPI integer :: num_procs_x, num_procs_y, num_procs_z !< !! Optimal number of processors in the x-, y- and z-directions @@ -901,11 +871,7 @@ contains integer, intent(IN) :: mpi_dir integer, intent(IN) :: pbc_loc -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_sendrecv_grid_variables_buffers not supported without MPI.' - -#else +#ifdef MFC_MPI ! MPI Communication in x-direction ================================= if (mpi_dir == 1) then @@ -1110,11 +1076,7 @@ contains real(kind(0d0)), intent(OUT) :: ccfl_max_glb real(kind(0d0)), intent(OUT) :: Rc_min_glb -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_reduce_stability_criteria_extrema not supported without MPI.' - -#else +#ifdef MFC_MPI ! Reducing local extrema of ICFL, VCFL, CCFL and Rc numbers to their ! global extrema and bookkeeping the results on the rank 0 processor @@ -1147,11 +1109,7 @@ contains real(kind(0d0)), intent(IN) :: var_loc real(kind(0d0)), intent(OUT) :: var_glb -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_allreduce_sum not supported without MPI.' - -#else +#ifdef MFC_MPI ! Performing the reduction procedure call MPI_ALLREDUCE(var_loc, var_glb, 1, MPI_DOUBLE_PRECISION, & @@ -1173,11 +1131,7 @@ contains real(kind(0d0)), intent(IN) :: var_loc real(kind(0d0)), intent(OUT) :: var_glb -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_allreduce_min not supported without MPI.' - -#else +#ifdef MFC_MPI ! Performing the reduction procedure call MPI_ALLREDUCE(var_loc, var_glb, 1, MPI_DOUBLE_PRECISION, & @@ -1199,11 +1153,7 @@ contains real(kind(0d0)), intent(IN) :: var_loc real(kind(0d0)), intent(OUT) :: var_glb -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_allreduce_max not supported without MPI.' - -#else +#ifdef MFC_MPI ! Performing the reduction procedure call MPI_ALLREDUCE(var_loc, var_glb, 1, MPI_DOUBLE_PRECISION, & @@ -1229,11 +1179,7 @@ contains integer :: i, j, k, l, r !< Generic loop iterators -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_sendrecv_conservative_variables_buffers not supported without MPI.' - -#else +#ifdef MFC_MPI !nCalls_time = nCalls_time + 1 @@ -2137,14 +2083,10 @@ contains !> Module deallocation and/or disassociation procedures subroutine s_finalize_mpi_proxy_module() ! ----------------------------- -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_finalize_mpi_proxy_module not supported without MPI.' - -#else +#ifdef MFC_MPI ! Deallocating q_cons_buff_send and q_cons_buff_recv - deallocate (q_cons_buff_send, q_cons_buff_recv) + @:DEALLOCATE(q_cons_buff_send, q_cons_buff_recv) #endif @@ -2153,11 +2095,7 @@ contains !> The subroutine finalizes the MPI execution environment. subroutine s_mpi_finalize() ! ------------------------------------------ -#ifndef MFC_MPI - - print '(A)', '[m_mpi_proxy] s_mpi_finalize not supported without MPI.' - -#else +#ifdef MFC_MPI ! Finalizing the MPI environment call MPI_FINALIZE(ierr) diff --git a/src/simulation/nvtx.f90 b/src/simulation/m_nvtx.f90 similarity index 98% rename from src/simulation/nvtx.f90 rename to src/simulation/m_nvtx.f90 index c3673bef92..383a492c3f 100644 --- a/src/simulation/nvtx.f90 +++ b/src/simulation/m_nvtx.f90 @@ -1,5 +1,4 @@ - -module nvtx +module m_nvtx use iso_c_binding @@ -80,4 +79,4 @@ subroutine nvtxEndRange #endif end subroutine -end module nvtx +end module m_nvtx diff --git a/src/simulation/m_qbmm.fpp b/src/simulation/m_qbmm.fpp index 584b6e5fdf..721a7407fa 100644 --- a/src/simulation/m_qbmm.fpp +++ b/src/simulation/m_qbmm.fpp @@ -3,6 +3,7 @@ !! @brief Contains module m_qbmm #:include 'case.fpp' +#:include 'macros.fpp' !> @brief This module is used to compute moment inversion via qbmm module m_qbmm @@ -34,11 +35,11 @@ module m_qbmm type(int_bounds_info) :: is1, is2, is3 - integer, allocatable, dimension(:) :: bubrs + integer, allocatable, dimension(:) :: bubrs integer, allocatable, dimension(:, :) :: bubmoms !$acc declare create(momrhs, nterms, is1, is2, is3) -!$acc declare create( bubrs, bubmoms) +!$acc declare create(bubrs, bubmoms) contains @@ -60,7 +61,7 @@ contains #:endif - allocate (momrhs(3, 0:2, 0:2, nterms, nb)) + @:ALLOCATE(momrhs(3, 0:2, 0:2, nterms, nb)) momrhs = 0d0 ! Assigns the required RHS moments for moment transport equations @@ -154,11 +155,8 @@ contains !$acc update device(momrhs) - allocate (bubrs(1:nb)) - - allocate (bubmoms(1:nb, 1:nmom)) - - + @:ALLOCATE(bubrs(1:nb)) + @:ALLOCATE(bubmoms(1:nb, 1:nmom)) do i = 1, nb bubrs(i) = bub_idx%rs(i) diff --git a/src/simulation/m_rhs.f90 b/src/simulation/m_rhs.fpp similarity index 87% rename from src/simulation/m_rhs.f90 rename to src/simulation/m_rhs.fpp index 6019854c06..8c77940c78 100644 --- a/src/simulation/m_rhs.f90 +++ b/src/simulation/m_rhs.fpp @@ -2,6 +2,8 @@ !! @file m_rhs.f90 !! @brief Contains module m_rhs +#:include 'macros.fpp' + !> @brief The module contains the subroutines used to calculate the right- !! hand-side (RHS) in the quasi-conservative, shock- and interface- !! capturing finite-volume framework for the multicomponent Navier- @@ -37,11 +39,12 @@ module m_rhs use m_hypoelastic - use nvtx - use m_monopole use m_viscous + + use m_nvtx + ! ========================================================================== implicit none @@ -213,37 +216,28 @@ subroutine s_initialize_rhs_module() ! --------------------------------- !$acc update device(ix, iy, iz) if (any(Re_size > 0) .and. cyl_coord) then - allocate (tau_Re_vf(1:sys_size)) + @:ALLOCATE(tau_Re_vf(1:sys_size)) do i = 1, num_dims - allocate (tau_Re_vf(cont_idx%end + i)%sf(ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(tau_Re_vf(cont_idx%end + i)%sf(ix%beg:ix%end, & -!$acc iy%beg:iy%end, & -!$acc iz%beg:iz%end)) + @:ALLOCATE(tau_Re_vf(cont_idx%end + i)%sf(ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do - allocate (tau_Re_vf(E_idx)%sf(ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create (tau_Re_vf(E_idx)%sf(ix%beg:ix%end, & -!$acc iy%beg:iy%end, & -!$acc iz%beg:iz%end)) + @:ALLOCATE(tau_Re_vf(E_idx)%sf(ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end if ixt = ix; iyt = iy; izt = iz - allocate (q_cons_qp%vf(1:sys_size)) - allocate (q_prim_qp%vf(1:sys_size)) -!$acc enter data create(q_cons_qp%vf(1:sys_size),q_prim_qp%vf(1:sys_size)) + @:ALLOCATE(q_cons_qp%vf(1:sys_size)) + @:ALLOCATE(q_prim_qp%vf(1:sys_size)) do l = 1, sys_size - allocate (q_cons_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) -!$acc enter data create(q_cons_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(q_cons_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end do do l = mom_idx%beg, E_idx - allocate (q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) -!$acc enter data create(q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end do ! if (hypoelasticity) then @@ -254,8 +248,7 @@ subroutine s_initialize_rhs_module() ! --------------------------------- ! end if do l = adv_idx%end + 1, sys_size - allocate (q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) -!$acc enter data create(q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(q_prim_qp%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end do do l = 1, cont_idx%end @@ -279,97 +272,93 @@ subroutine s_initialize_rhs_module() ! --------------------------------- ! ================================================================== if (qbmm) then - allocate (mom_sp(1:nmomsp), mom_3d(0:2, 0:2, nb)) - - do i = 0, 2; do j = 0, 2; do k = 1, nb - allocate (mom_3d(i, j, k)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(mom_3d(i, j, k)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) - end do; end do; end do + @:ALLOCATE(mom_sp(1:nmomsp), mom_3d(0:2, 0:2, nb)) + + do i = 0, 2 + do j = 0, 2 + do k = 1, nb + @:ALLOCATE(mom_3d(i, j, k)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + end do + end do + end do do i = 1, nmomsp - allocate (mom_sp(i)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(mom_sp(i)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(mom_sp(i)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do end if ! Allocation/Association of qK_cons_n and qK_prim_n ========== - allocate (qL_cons_n(1:num_dims)) - allocate (qR_cons_n(1:num_dims)) - allocate (qL_prim_n(1:num_dims)) - allocate (qR_prim_n(1:num_dims)) + @:ALLOCATE(qL_cons_n(1:num_dims)) + @:ALLOCATE(qR_cons_n(1:num_dims)) + @:ALLOCATE(qL_prim_n(1:num_dims)) + @:ALLOCATE(qR_prim_n(1:num_dims)) - allocate (qL_prim(1:num_dims)) - allocate (qR_prim(1:num_dims)) + @:ALLOCATE(qL_prim(1:num_dims)) + @:ALLOCATE(qR_prim(1:num_dims)) do i = 1, num_dims - allocate (qL_prim(i)%vf(1:sys_size)) -!$acc enter data create(qL_prim(i)%vf(1:sys_size)) - allocate (qR_prim(i)%vf(1:sys_size)) -!$acc enter data create(qR_prim(i)%vf(1:sys_size)) + @:ALLOCATE(qL_prim(i)%vf(1:sys_size)) + @:ALLOCATE(qR_prim(i)%vf(1:sys_size)) end do if (weno_Re_flux) then do i = 1, num_dims do l = mom_idx%beg, mom_idx%end - allocate (qL_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) -!$acc enter data create(qL_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) - allocate (qR_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) -!$acc enter data create(qR_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(qL_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(qR_prim(i)%vf(l)%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end do end do end if - allocate (myflux_vf(1:num_dims)) - allocate (myflux_src_vf(1:num_dims)) + @:ALLOCATE(myflux_vf(1:num_dims)) + @:ALLOCATE(myflux_src_vf(1:num_dims)) if (mpp_lim .and. bubbles) then - allocate (alf_sum%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) - - !$acc enter data create(alf_sum%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(alf_sum%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end if do i = 1, num_dims - allocate (qL_cons_n(i)%vf(1:sys_size)) - allocate (qR_cons_n(i)%vf(1:sys_size)) - allocate (qL_prim_n(i)%vf(1:sys_size)) - allocate (qR_prim_n(i)%vf(1:sys_size)) - allocate (myflux_vf(i)%vf(1:sys_size)) - allocate (myflux_src_vf(i)%vf(1:sys_size)) + @:ALLOCATE(qL_cons_n(i)%vf(1:sys_size)) + @:ALLOCATE(qR_cons_n(i)%vf(1:sys_size)) + @:ALLOCATE(qL_prim_n(i)%vf(1:sys_size)) + @:ALLOCATE(qR_prim_n(i)%vf(1:sys_size)) + @:ALLOCATE(myflux_vf(i)%vf(1:sys_size)) + @:ALLOCATE(myflux_src_vf(i)%vf(1:sys_size)) end do ! END: Allocation/Association of qK_cons_n and qK_prim_n ===== - allocate (qL_rsx_vf(ix%beg:ix%end, & + @:ALLOCATE(qL_rsx_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) - allocate (qR_rsx_vf(ix%beg:ix%end, & + @:ALLOCATE(qR_rsx_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) if (n > 0) then - allocate (qL_rsy_vf(iy%beg:iy%end, & + @:ALLOCATE(qL_rsy_vf(iy%beg:iy%end, & ix%beg:ix%end, iz%beg:iz%end, 1:sys_size)) - allocate (qR_rsy_vf(iy%beg:iy%end, & + @:ALLOCATE(qR_rsy_vf(iy%beg:iy%end, & ix%beg:ix%end, iz%beg:iz%end, 1:sys_size)) else - allocate (qL_rsy_vf(ix%beg:ix%end, & + @:ALLOCATE(qL_rsy_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) - allocate (qR_rsy_vf(ix%beg:ix%end, & + @:ALLOCATE(qR_rsy_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) end if if (p > 0) then - allocate (qL_rsz_vf(iz%beg:iz%end, & + @:ALLOCATE(qL_rsz_vf(iz%beg:iz%end, & iy%beg:iy%end, ix%beg:ix%end, 1:sys_size)) - allocate (qR_rsz_vf(iz%beg:iz%end, & + @:ALLOCATE(qR_rsz_vf(iz%beg:iz%end, & iy%beg:iy%end, ix%beg:ix%end, 1:sys_size)) else - allocate (qL_rsz_vf(ix%beg:ix%end, & + @:ALLOCATE(qL_rsz_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) - allocate (qR_rsz_vf(ix%beg:ix%end, & + @:ALLOCATE(qR_rsz_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, 1:sys_size)) end if @@ -378,44 +367,39 @@ subroutine s_initialize_rhs_module() ! --------------------------------- if (any(Re_size > 0)) then - allocate (dq_prim_dx_qp%vf(1:sys_size)) - allocate (dq_prim_dy_qp%vf(1:sys_size)) - allocate (dq_prim_dz_qp%vf(1:sys_size)) - allocate (gm_vel_qp%vf(1:sys_size)) -!$acc enter data create(dq_prim_dx_qp%vf(1:sys_size),dq_prim_dy_qp%vf(1:sys_size)) -!$acc enter data create(dq_prim_dz_qp%vf(1:sys_size),gm_vel_qp%vf(1:sys_size)) + @:ALLOCATE(dq_prim_dx_qp%vf(1:sys_size)) + @:ALLOCATE(dq_prim_dy_qp%vf(1:sys_size)) + @:ALLOCATE(dq_prim_dz_qp%vf(1:sys_size)) + @:ALLOCATE(gm_vel_qp%vf(1:sys_size)) + if (any(Re_size > 0)) then do l = mom_idx%beg, mom_idx%end - allocate (dq_prim_dx_qp%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) - allocate (gm_vel_qp%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dq_prim_dx_qp%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) -!$acc enter data create(gm_vel_qp%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dq_prim_dx_qp%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + @:ALLOCATE(gm_vel_qp%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do if (n > 0) then do l = mom_idx%beg, mom_idx%end - allocate (dq_prim_dy_qp%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dq_prim_dy_qp%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dq_prim_dy_qp%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do if (p > 0) then do l = mom_idx%beg, mom_idx%end - allocate (dq_prim_dz_qp%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dq_prim_dz_qp%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dq_prim_dz_qp%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do end if @@ -427,69 +411,58 @@ subroutine s_initialize_rhs_module() ! --------------------------------- ! END: Allocation of dq_prim_ds_qp ================================= ! Allocation/Association of dqK_prim_ds_n ======================= - allocate (dqL_prim_dx_n(1:num_dims)) - allocate (dqL_prim_dy_n(1:num_dims)) - allocate (dqL_prim_dz_n(1:num_dims)) - allocate (dqR_prim_dx_n(1:num_dims)) - allocate (dqR_prim_dy_n(1:num_dims)) - allocate (dqR_prim_dz_n(1:num_dims)) + @:ALLOCATE(dqL_prim_dx_n(1:num_dims)) + @:ALLOCATE(dqL_prim_dy_n(1:num_dims)) + @:ALLOCATE(dqL_prim_dz_n(1:num_dims)) + @:ALLOCATE(dqR_prim_dx_n(1:num_dims)) + @:ALLOCATE(dqR_prim_dy_n(1:num_dims)) + @:ALLOCATE(dqR_prim_dz_n(1:num_dims)) if (any(Re_size > 0)) then do i = 1, num_dims - allocate (dqL_prim_dx_n(i)%vf(1:sys_size)) - allocate (dqL_prim_dy_n(i)%vf(1:sys_size)) - allocate (dqL_prim_dz_n(i)%vf(1:sys_size)) - allocate (dqR_prim_dx_n(i)%vf(1:sys_size)) - allocate (dqR_prim_dy_n(i)%vf(1:sys_size)) - allocate (dqR_prim_dz_n(i)%vf(1:sys_size)) -!$acc enter data create(dqL_prim_dx_n(i)%vf(1:sys_size)) -!$acc enter data create(dqL_prim_dy_n(i)%vf(1:sys_size)) -!$acc enter data create(dqL_prim_dz_n(i)%vf(1:sys_size)) -!$acc enter data create(dqR_prim_dx_n(i)%vf(1:sys_size)) -!$acc enter data create(dqR_prim_dy_n(i)%vf(1:sys_size)) -!$acc enter data create(dqR_prim_dz_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqL_prim_dx_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqL_prim_dy_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqL_prim_dz_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqR_prim_dx_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqR_prim_dy_n(i)%vf(1:sys_size)) + @:ALLOCATE(dqR_prim_dz_n(i)%vf(1:sys_size)) + if (any(Re_size > 0)) then do l = mom_idx%beg, mom_idx%end - allocate (dqL_prim_dx_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) - allocate (dqR_prim_dx_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dqL_prim_dx_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) -!$acc enter data create(dqR_prim_dx_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dqL_prim_dx_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + @:ALLOCATE(dqR_prim_dx_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do if (n > 0) then do l = mom_idx%beg, mom_idx%end - allocate (dqL_prim_dy_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) - allocate (dqR_prim_dy_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dqL_prim_dy_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) -!$acc enter data create(dqR_prim_dy_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dqL_prim_dy_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + @:ALLOCATE(dqR_prim_dy_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do end if if (p > 0) then do l = mom_idx%beg, mom_idx%end - allocate (dqL_prim_dz_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) - allocate (dqR_prim_dz_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(dqL_prim_dz_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) -!$acc enter data create(dqR_prim_dz_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(dqL_prim_dz_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + @:ALLOCATE(dqR_prim_dz_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do end if @@ -501,34 +474,34 @@ subroutine s_initialize_rhs_module() ! --------------------------------- if (any(Re_size > 0)) then if (weno_Re_flux) then - allocate (dqL_rsx_vf(ix%beg:ix%end, & + @:ALLOCATE(dqL_rsx_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) - allocate (dqR_rsx_vf(ix%beg:ix%end, & + @:ALLOCATE(dqR_rsx_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) if (n > 0) then - allocate (dqL_rsy_vf(iy%beg:iy%end, & + @:ALLOCATE(dqL_rsy_vf(iy%beg:iy%end, & ix%beg:ix%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) - allocate (dqR_rsy_vf(iy%beg:iy%end, & + @:ALLOCATE(dqR_rsy_vf(iy%beg:iy%end, & ix%beg:ix%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) else - allocate (dqL_rsy_vf(ix%beg:ix%end, & + @:ALLOCATE(dqL_rsy_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) - allocate (dqR_rsy_vf(ix%beg:ix%end, & + @:ALLOCATE(dqR_rsy_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) end if if (p > 0) then - allocate (dqL_rsz_vf(iz%beg:iz%end, & + @:ALLOCATE(dqL_rsz_vf(iz%beg:iz%end, & iy%beg:iy%end, ix%beg:ix%end, mom_idx%beg:mom_idx%end)) - allocate (dqR_rsz_vf(iz%beg:iz%end, & + @:ALLOCATE(dqR_rsz_vf(iz%beg:iz%end, & iy%beg:iy%end, ix%beg:ix%end, mom_idx%beg:mom_idx%end)) else - allocate (dqL_rsz_vf(ix%beg:ix%end, & + @:ALLOCATE(dqL_rsz_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) - allocate (dqR_rsz_vf(ix%beg:ix%end, & + @:ALLOCATE(dqR_rsz_vf(ix%beg:ix%end, & iy%beg:iy%end, iz%beg:iz%end, mom_idx%beg:mom_idx%end)) end if @@ -538,106 +511,93 @@ subroutine s_initialize_rhs_module() ! --------------------------------- ! ================================================================== ! Allocation of gm_alphaK_n ===================================== - allocate (gm_alphaL_n(1:num_dims)) - allocate (gm_alphaR_n(1:num_dims)) + @:ALLOCATE(gm_alphaL_n(1:num_dims)) + @:ALLOCATE(gm_alphaR_n(1:num_dims)) ! ================================================================== if (bubbles) then - allocate (bub_adv_src(0:m, 0:n, 0:p)) + @:ALLOCATE(bub_adv_src(0:m, 0:n, 0:p)) if (qbmm) then - allocate (bub_mom_src(1:nmom, 0:m, 0:n, 0:p, 1:nb)) + @:ALLOCATE(bub_mom_src(1:nmom, 0:m, 0:n, 0:p, 1:nb)) else - allocate (bub_r_src(0:m, 0:n, 0:p, 1:nb)) - allocate (bub_v_src(0:m, 0:n, 0:p, 1:nb)) - allocate (bub_p_src(0:m, 0:n, 0:p, 1:nb)) - allocate (bub_m_src(0:m, 0:n, 0:p, 1:nb)) + @:ALLOCATE(bub_r_src(0:m, 0:n, 0:p, 1:nb)) + @:ALLOCATE(bub_v_src(0:m, 0:n, 0:p, 1:nb)) + @:ALLOCATE(bub_p_src(0:m, 0:n, 0:p, 1:nb)) + @:ALLOCATE(bub_m_src(0:m, 0:n, 0:p, 1:nb)) end if end if if (monopole) then - allocate (mono_mass_src(0:m, 0:n, 0:p)) - allocate (mono_mom_src(1:num_dims, 0:m, 0:n, 0:p)) - allocate (mono_E_src(0:m, 0:n, 0:p)) -!$acc enter data create(mono_mass_src(0:m, 0:n, 0:p), mono_mom_src(1:num_dims, 0:m, 0:n, 0:p), mono_E_src(0:m, 0:n, 0:p)) + @:ALLOCATE(mono_mass_src(0:m, 0:n, 0:p)) + @:ALLOCATE(mono_mom_src(1:num_dims, 0:m, 0:n, 0:p)) + @:ALLOCATE(mono_E_src(0:m, 0:n, 0:p)) end if - allocate (divu%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(divu%sf( ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) + @:ALLOCATE(divu%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) ! Configuring Coordinate Direction Indexes ========================= ! ================================================================== ! Allocation/Association of flux_n, flux_src_n, and flux_gsrc_n === - allocate (flux_n(1:num_dims)) - allocate (flux_src_n(1:num_dims)) - allocate (flux_gsrc_n(1:num_dims)) + @:ALLOCATE(flux_n(1:num_dims)) + @:ALLOCATE(flux_src_n(1:num_dims)) + @:ALLOCATE(flux_gsrc_n(1:num_dims)) do i = 1, num_dims - allocate (flux_n(i)%vf(1:sys_size)) - allocate (flux_src_n(i)%vf(1:sys_size)) - allocate (flux_gsrc_n(i)%vf(1:sys_size)) -!$acc enter data create(flux_n(i)%vf(1:sys_size),flux_src_n(i)%vf(1:sys_size),flux_gsrc_n(i)%vf(1:sys_size)) + @:ALLOCATE(flux_n(i)%vf(1:sys_size)) + @:ALLOCATE(flux_src_n(i)%vf(1:sys_size)) + @:ALLOCATE(flux_gsrc_n(i)%vf(1:sys_size)) if (i == 1) then do l = 1, sys_size - allocate (flux_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) - allocate (flux_gsrc_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(flux_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) -!$acc enter data create(flux_gsrc_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(flux_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + @:ALLOCATE(flux_gsrc_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do if (any(Re_size > 0)) then do l = mom_idx%beg, E_idx - allocate (flux_src_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(flux_src_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(flux_src_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do end if - allocate (flux_src_n(i)%vf(adv_idx%beg)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(flux_src_n(i)%vf(adv_idx%beg)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(flux_src_n(i)%vf(adv_idx%beg)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) + if (riemann_solver == 1) then do l = adv_idx%beg + 1, adv_idx%end - allocate (flux_src_n(i)%vf(l)%sf( & - ix%beg:ix%end, & - iy%beg:iy%end, & - iz%beg:iz%end)) -!$acc enter data create(flux_src_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) + @:ALLOCATE(flux_src_n(i)%vf(l)%sf( & + & ix%beg:ix%end, & + & iy%beg:iy%end, & + & iz%beg:iz%end)) end do else - !IF ( (num_fluids > 1) .AND. (bubbles .NEQV. .TRUE.)) THEN do l = adv_idx%beg + 1, adv_idx%end flux_src_n(i)%vf(l)%sf => & flux_src_n(i)%vf(adv_idx%beg)%sf !$acc enter data attach(flux_src_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do - !END IF end if else do l = 1, sys_size - allocate (flux_gsrc_n(i)%vf(l)%sf( & + @:ALLOCATE(flux_gsrc_n(i)%vf(l)%sf( & ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(flux_gsrc_n(i)%vf(l)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do do l = 1, sys_size flux_n(i)%vf(l)%sf => & @@ -654,12 +614,12 @@ subroutine s_initialize_rhs_module() ! --------------------------------- ! END: Allocation/Association of flux_n, flux_src_n, and flux_gsrc_n === if (alt_soundspeed) then - allocate (blkmod1(0:m, 0:n, 0:p), blkmod2(0:m, 0:n, 0:p), alpha1(0:m, 0:n, 0:p), alpha2(0:m, 0:n, 0:p), Kterm(0:m, 0:n, 0:p)) + @:ALLOCATE(blkmod1(0:m, 0:n, 0:p), blkmod2(0:m, 0:n, 0:p), alpha1(0:m, 0:n, 0:p), alpha2(0:m, 0:n, 0:p), Kterm(0:m, 0:n, 0:p)) end if - allocate (gamma_min(1:num_fluids), pres_inf(1:num_fluids)) + @:ALLOCATE(gamma_min(1:num_fluids), pres_inf(1:num_fluids)) do i = 1, num_fluids gamma_min(i) = 1d0/fluid_pp(i)%gamma + 1d0 @@ -668,7 +628,7 @@ subroutine s_initialize_rhs_module() ! --------------------------------- !$acc update device(gamma_min, pres_inf) if (any(Re_size > 0)) then - allocate (Res(1:2, 1:maxval(Re_size))) + @:ALLOCATE(Res(1:2, 1:maxval(Re_size))) end if if (any(Re_size > 0)) then @@ -722,7 +682,7 @@ subroutine s_initialize_rhs_module() ! --------------------------------- end do if (bubbles) then - allocate (nbub(0:m, 0:n, 0:p)) + @:ALLOCATE(nbub(0:m, 0:n, 0:p)) end if end subroutine s_initialize_rhs_module ! ------------------------------- @@ -2558,37 +2518,37 @@ subroutine s_finalize_rhs_module() ! ----------------------------------- end do do j = mom_idx%beg, E_idx - deallocate (q_cons_qp%vf(j)%sf) - deallocate (q_prim_qp%vf(j)%sf) + @:DEALLOCATE(q_cons_qp%vf(j)%sf) + @:DEALLOCATE(q_prim_qp%vf(j)%sf) end do - deallocate (q_cons_qp%vf, q_prim_qp%vf) + @:DEALLOCATE(q_cons_qp%vf, q_prim_qp%vf) - deallocate (qL_rsx_vf, qR_rsx_vf) + @:DEALLOCATE(qL_rsx_vf, qR_rsx_vf) if (n > 0) then - deallocate (qL_rsy_vf, qR_rsy_vf) + @:DEALLOCATE(qL_rsy_vf, qR_rsy_vf) end if if (p > 0) then - deallocate (qL_rsz_vf, qR_rsz_vf) + @:DEALLOCATE(qL_rsz_vf, qR_rsz_vf) end if if (weno_Re_flux) then - deallocate (dqL_rsx_vf, dqR_rsx_vf) + @:DEALLOCATE(dqL_rsx_vf, dqR_rsx_vf) if (n > 0) then - deallocate (dqL_rsy_vf, dqR_rsy_vf) + @:DEALLOCATE(dqL_rsy_vf, dqR_rsy_vf) end if if (p > 0) then - deallocate (dqL_rsz_vf, dqR_rsz_vf) + @:DEALLOCATE(dqL_rsz_vf, dqR_rsz_vf) end if end if do i = num_dims, 1, -1 - deallocate (qL_cons_n(i)%vf, qL_prim_n(i)%vf) - deallocate (qR_cons_n(i)%vf, qR_prim_n(i)%vf) + @:DEALLOCATE(qL_cons_n(i)%vf, qL_prim_n(i)%vf) + @:DEALLOCATE(qR_cons_n(i)%vf, qR_prim_n(i)%vf) end do if (mpp_lim .and. bubbles) then @@ -2597,32 +2557,32 @@ subroutine s_finalize_rhs_module() ! ----------------------------------- !$acc exit data delete(alf_sum%sf(ix%beg:ix%end, iy%beg:iy%end, iz%beg:iz%end)) end if - deallocate (qL_cons_n, qR_cons_n, qL_prim_n, qR_prim_n) + @:DEALLOCATE(qL_cons_n, qR_cons_n, qL_prim_n, qR_prim_n) if (any(Re_size > 0)) then do l = mom_idx%beg, mom_idx%end - deallocate (dq_prim_dx_qp%vf(l)%sf) - deallocate (gm_vel_qp%vf(l)%sf) + @:DEALLOCATE(dq_prim_dx_qp%vf(l)%sf) + @:DEALLOCATE(gm_vel_qp%vf(l)%sf) end do if (n > 0) then do l = mom_idx%beg, mom_idx%end - deallocate (dq_prim_dy_qp%vf(l)%sf) + @:DEALLOCATE(dq_prim_dy_qp%vf(l)%sf) end do if (p > 0) then do l = mom_idx%beg, mom_idx%end - deallocate (dq_prim_dz_qp%vf(l)%sf) + @:DEALLOCATE(dq_prim_dz_qp%vf(l)%sf) end do end if end if - deallocate (dq_prim_dx_qp%vf) - deallocate (dq_prim_dy_qp%vf) - deallocate (dq_prim_dz_qp%vf) - deallocate (gm_vel_qp%vf) + @:DEALLOCATE(dq_prim_dx_qp%vf) + @:DEALLOCATE(dq_prim_dy_qp%vf) + @:DEALLOCATE(dq_prim_dz_qp%vf) + @:DEALLOCATE(gm_vel_qp%vf) end if if (any(Re_size > 0)) then @@ -2630,44 +2590,44 @@ subroutine s_finalize_rhs_module() ! ----------------------------------- if (any(Re_size > 0)) then do l = mom_idx%beg, mom_idx%end - deallocate (dqL_prim_dx_n(i)%vf(l)%sf) - deallocate (dqR_prim_dx_n(i)%vf(l)%sf) + @:DEALLOCATE(dqL_prim_dx_n(i)%vf(l)%sf) + @:DEALLOCATE(dqR_prim_dx_n(i)%vf(l)%sf) end do if (n > 0) then do l = mom_idx%beg, mom_idx%end - deallocate (dqL_prim_dy_n(i)%vf(l)%sf) - deallocate (dqR_prim_dy_n(i)%vf(l)%sf) + @:DEALLOCATE(dqL_prim_dy_n(i)%vf(l)%sf) + @:DEALLOCATE(dqR_prim_dy_n(i)%vf(l)%sf) end do end if if (p > 0) then do l = mom_idx%beg, mom_idx%end - deallocate (dqL_prim_dz_n(i)%vf(l)%sf) - deallocate (dqR_prim_dz_n(i)%vf(l)%sf) + @:DEALLOCATE(dqL_prim_dz_n(i)%vf(l)%sf) + @:DEALLOCATE(dqR_prim_dz_n(i)%vf(l)%sf) end do end if end if - deallocate (dqL_prim_dx_n(i)%vf) - deallocate (dqL_prim_dy_n(i)%vf) - deallocate (dqL_prim_dz_n(i)%vf) - deallocate (dqR_prim_dx_n(i)%vf) - deallocate (dqR_prim_dy_n(i)%vf) - deallocate (dqR_prim_dz_n(i)%vf) + @:DEALLOCATE(dqL_prim_dx_n(i)%vf) + @:DEALLOCATE(dqL_prim_dy_n(i)%vf) + @:DEALLOCATE(dqL_prim_dz_n(i)%vf) + @:DEALLOCATE(dqR_prim_dx_n(i)%vf) + @:DEALLOCATE(dqR_prim_dy_n(i)%vf) + @:DEALLOCATE(dqR_prim_dz_n(i)%vf) end do end if - deallocate (dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n) - deallocate (dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n) + @:DEALLOCATE(dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n) + @:DEALLOCATE(dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n) if (any(Re_size > 0) .and. cyl_coord) then do i = 1, num_dims - deallocate (tau_Re_vf(cont_idx%end + i)%sf) + @:DEALLOCATE(tau_Re_vf(cont_idx%end + i)%sf) end do - deallocate (tau_Re_vf(E_idx)%sf) - deallocate (tau_Re_vf) + @:DEALLOCATE(tau_Re_vf(E_idx)%sf) + @:DEALLOCATE(tau_Re_vf) end if do i = num_dims, 1, -1 @@ -2675,23 +2635,23 @@ subroutine s_finalize_rhs_module() ! ----------------------------------- do l = 1, sys_size nullify (flux_n(i)%vf(l)%sf) nullify (flux_src_n(i)%vf(l)%sf) - deallocate (flux_gsrc_n(i)%vf(l)%sf) + @:DEALLOCATE(flux_gsrc_n(i)%vf(l)%sf) end do else do l = 1, sys_size - deallocate (flux_n(i)%vf(l)%sf) - deallocate (flux_gsrc_n(i)%vf(l)%sf) + @:DEALLOCATE(flux_n(i)%vf(l)%sf) + @:DEALLOCATE(flux_gsrc_n(i)%vf(l)%sf) end do if (any(Re_size > 0)) then do l = mom_idx%beg, E_idx - deallocate (flux_src_n(i)%vf(l)%sf) + @:DEALLOCATE(flux_src_n(i)%vf(l)%sf) end do end if if (riemann_solver == 1) then do l = adv_idx%beg + 1, adv_idx%end - deallocate (flux_src_n(i)%vf(l)%sf) + @:DEALLOCATE(flux_src_n(i)%vf(l)%sf) end do else do l = adv_idx%beg + 1, adv_idx%end @@ -2699,13 +2659,13 @@ subroutine s_finalize_rhs_module() ! ----------------------------------- end do end if - deallocate (flux_src_n(i)%vf(adv_idx%beg)%sf) + @:DEALLOCATE(flux_src_n(i)%vf(adv_idx%beg)%sf) end if - deallocate (flux_n(i)%vf, flux_src_n(i)%vf, flux_gsrc_n(i)%vf) + @:DEALLOCATE(flux_n(i)%vf, flux_src_n(i)%vf, flux_gsrc_n(i)%vf) end do - deallocate (flux_n, flux_src_n, flux_gsrc_n) + @:DEALLOCATE(flux_n, flux_src_n, flux_gsrc_n) s_riemann_solver => null() s_convert_to_mixture_variables => null() diff --git a/src/simulation/m_riemann_solvers.fpp b/src/simulation/m_riemann_solvers.fpp index c8c264366c..253dfdb3e1 100644 --- a/src/simulation/m_riemann_solvers.fpp +++ b/src/simulation/m_riemann_solvers.fpp @@ -37,7 +37,6 @@ module m_riemann_solvers s_riemann_solver, & s_hll_riemann_solver, & s_hllc_riemann_solver, & - s_convert_species_to_mixture_variables_riemann_acc, & s_finalize_riemann_solvers_module abstract interface ! ======================================================= @@ -2673,69 +2672,6 @@ contains end subroutine s_hllc_riemann_solver - subroutine s_convert_species_to_mixture_variables_riemann_acc(rho_K, & - gamma_K, pi_inf_K, & - alpha_K, alpha_rho_K, Re_K, k, l, r) -!$acc routine seq - - real(kind(0d0)), intent(INOUT) :: rho_K, gamma_K, pi_inf_K - - real(kind(0d0)), dimension(:), intent(INOUT) :: alpha_rho_K, alpha_K !< - real(kind(0d0)), dimension(:), intent(OUT) :: Re_K - !! Partial densities and volume fractions - - integer, intent(IN) :: k, l, r - - integer :: i, j !< Generic loop iterators - real(kind(0d0)) :: alpha_K_sum - - ! Constraining the partial densities and the volume fractions within - ! their physical bounds to make sure that any mixture variables that - ! are derived from them result within the limits that are set by the - ! fluids physical parameters that make up the mixture - rho_K = 0d0 - gamma_K = 0d0 - pi_inf_K = 0d0 - - alpha_K_sum = 0d0 - - if (mpp_lim) then - do i = 1, num_fluids - alpha_rho_K(i) = max(0d0, alpha_rho_K(i)) - alpha_K(i) = min(max(0d0, alpha_K(i)), 1d0) - alpha_K_sum = alpha_K_sum + alpha_K(i) - end do - - alpha_K = alpha_K/max(alpha_K_sum, sgm_eps) - - end if - - do i = 1, num_fluids - rho_K = rho_K + alpha_rho_K(i) - gamma_K = gamma_K + alpha_K(i)*gammas(i) - pi_inf_K = pi_inf_K + alpha_K(i)*pi_infs(i) - end do - - if (any(Re_size > 0)) then - - do i = 1, 2 - Re_K(i) = dflt_real - - if (Re_size(i) > 0) Re_K(i) = 0d0 - - do j = 1, Re_size(i) - Re_K(i) = alpha_K(Re_idx(i, j))/Res(i, j) & - + Re_K(i) - end do - - Re_K(i) = 1d0/max(Re_K(i), sgm_eps) - - end do - end if - - end subroutine s_convert_species_to_mixture_variables_riemann_acc ! ---------------- - - !! @param l Third coordinate index !> The computation of parameters, the allocation of memory, !! the association of pointers and/or the execution of any diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 15f9c59773..b03c0e628f 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -97,7 +97,7 @@ contains polytropic, thermal, & integral, integral_wrt, num_integrals, & polydisperse, poly_sigma, qbmm, & - R0_type, DEBUG + R0_type ! Checking that an input file has been provided by the user. If it ! has, then the input file is read in, otherwise, simulation exits. @@ -218,6 +218,9 @@ contains elseif (model_eqns == 2 .and. bubbles .and. bubble_model == 1) then print '(A)', 'The 5-equation bubbly flow model requires bubble_model = 2 (Keller--Miksis)' call s_mpi_abort() + elseif (bubbles .and. nb < 1) then + print '(A)', 'The Ensemble-Averaged Bubble Model requires nb >= 1' + call s_mpi_abort() elseif (bubbles .and. bubble_model == 3 .and. (polytropic .neqv. .true.)) then print '(A)', 'RP bubbles require polytropic compression' call s_mpi_abort() @@ -728,11 +731,7 @@ contains dimension(sys_size), & intent(INOUT) :: q_cons_vf -#ifndef MFC_MPI - - print '(A)', '[m_start_up] s_read_parallel_data_files not supported without MPI.' - -#else +#ifdef MFC_MPI real(kind(0d0)), allocatable, dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb diff --git a/src/simulation/m_time_steppers.f90 b/src/simulation/m_time_steppers.fpp similarity index 89% rename from src/simulation/m_time_steppers.f90 rename to src/simulation/m_time_steppers.fpp index d4dd514dc5..5399053166 100644 --- a/src/simulation/m_time_steppers.f90 +++ b/src/simulation/m_time_steppers.fpp @@ -2,6 +2,8 @@ !! @file m_time_steppers.f90 !! @brief Contains module m_time_steppers +#:include 'macros.fpp' + !> @brief The following module features a variety of time-stepping schemes. !! Currently, it includes the following Runge-Kutta (RK) algorithms: !! 1) 1st Order TVD RK @@ -25,7 +27,7 @@ module m_time_steppers use m_fftw - use nvtx + use m_nvtx ! ========================================================================== implicit none @@ -87,85 +89,76 @@ subroutine s_initialize_time_steppers_module() ! ----------------------- end if ! Allocating the cell-average conservative variables - allocate (q_cons_ts(1:num_ts)) + @:ALLOCATE(q_cons_ts(1:num_ts)) do i = 1, num_ts - allocate (q_cons_ts(i)%vf(1:sys_size)) -!$acc enter data create(q_cons_ts(i)%vf(1:sys_size)) + @:ALLOCATE(q_cons_ts(i)%vf(1:sys_size)) end do do i = 1, num_ts do j = 1, sys_size - allocate (q_cons_ts(i)%vf(j)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_cons_ts(i)%vf(j)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_cons_ts(i)%vf(j)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do end do ! Allocating the cell-average primitive ts variables if (any(com_wrt) .or. any(cb_wrt) .or. probe_wrt) then - allocate (q_prim_ts(0:3)) + @:ALLOCATE(q_prim_ts(0:3)) do i = 0, 3 - allocate (q_prim_ts(i)%vf(1:sys_size)) -!$acc enter data create(q_prim_ts(i)%vf(1:sys_size)) + @:ALLOCATE(q_prim_ts(i)%vf(1:sys_size)) end do do i = 0, 3 do j = 1, sys_size - allocate (q_prim_ts(i)%vf(j)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_prim_ts(i)%vf(j)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_prim_ts(i)%vf(j)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do end do end if ! Allocating the cell-average primitive variables - allocate (q_prim_vf(1:sys_size)) + @:ALLOCATE(q_prim_vf(1:sys_size)) do i = 1, adv_idx%end - allocate (q_prim_vf(i)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_prim_vf(i)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_prim_vf(i)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do if (bubbles) then do i = bub_idx%beg, bub_idx%end - allocate (q_prim_vf(i)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_prim_vf(i)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_prim_vf(i)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do end if if (hypoelasticity) then do i = stress_idx%beg, stress_idx%end - allocate (q_prim_vf(i)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_prim_vf(i)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_prim_vf(i)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do end if if (model_eqns == 3) then do i = internalEnergies_idx%beg, internalEnergies_idx%end - allocate (q_prim_vf(i)%sf(ix%beg:ix%end, & + @:ALLOCATE(q_prim_vf(i)%sf(ix%beg:ix%end, & iy%beg:iy%end, & iz%beg:iz%end)) -!$acc enter data create(q_prim_vf(i)%sf(ix%beg:ix%end,iy%beg:iy%end,iz%beg:iz%end)) end do end if ! Allocating the cell-average RHS variables - allocate (rhs_vf(1:sys_size)) + @:ALLOCATE(rhs_vf(1:sys_size)) do i = 1, sys_size - allocate (rhs_vf(i)%sf(0:m, 0:n, 0:p)) -!$acc enter data create(rhs_vf(i)%sf(0:m, 0:n, 0:p)) + @:ALLOCATE(rhs_vf(i)%sf(0:m, 0:n, 0:p)) end do ! Opening and writing the header of the run-time information file @@ -192,12 +185,18 @@ subroutine s_1st_order_tvd_rk(t_step, time_avg) ! ------------------------------ call nvtxStartRange("Time_Step") call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, t_step) - if (DEBUG) print *, 'got rhs' + +#ifdef DEBUG + print *, 'got rhs' +#endif if (run_time_info) then call s_write_run_time_information(q_prim_vf, t_step) end if - if (DEBUG) print *, 'wrote runtime info' + +#ifdef DEBUG + print *, 'wrote runtime info' +#endif if (any(com_wrt) .or. any(cb_wrt) .or. probe_wrt) then call s_time_step_cycling(t_step) @@ -481,57 +480,57 @@ subroutine s_finalize_time_steppers_module() ! ------------------------- do i = 1, num_ts do j = 1, sys_size - deallocate (q_cons_ts(i)%vf(j)%sf) + @:DEALLOCATE(q_cons_ts(i)%vf(j)%sf) end do - deallocate (q_cons_ts(i)%vf) + @:DEALLOCATE(q_cons_ts(i)%vf) end do - deallocate (q_cons_ts) + @:DEALLOCATE(q_cons_ts) ! Deallocating the cell-average primitive ts variables if (any(com_wrt) .or. any(cb_wrt) .or. probe_wrt) then do i = 0, 3 do j = 1, sys_size - deallocate (q_prim_ts(i)%vf(j)%sf) + @:DEALLOCATE(q_prim_ts(i)%vf(j)%sf) end do - deallocate (q_prim_ts(i)%vf) + @:DEALLOCATE(q_prim_ts(i)%vf) end do - deallocate (q_prim_ts) + @:DEALLOCATE(q_prim_ts) end if ! Deallocating the cell-average primitive variables do i = 1, adv_idx%end - deallocate (q_prim_vf(i)%sf) + @:DEALLOCATE(q_prim_vf(i)%sf) end do if (hypoelasticity) then do i = stress_idx%beg, stress_idx%end - deallocate (q_prim_vf(i)%sf) + @:DEALLOCATE(q_prim_vf(i)%sf) end do end if if (bubbles) then do i = bub_idx%beg, bub_idx%end - deallocate (q_prim_vf(i)%sf) + @:DEALLOCATE(q_prim_vf(i)%sf) end do end if if (model_eqns == 3) then do i = internalEnergies_idx%beg, internalEnergies_idx%end - deallocate (q_prim_vf(i)%sf) + @:DEALLOCATE(q_prim_vf(i)%sf) end do end if - deallocate (q_prim_vf) + @:DEALLOCATE(q_prim_vf) ! Deallocating the cell-average RHS variables do i = 1, sys_size - deallocate (rhs_vf(i)%sf) + @:DEALLOCATE(rhs_vf(i)%sf) end do - deallocate (rhs_vf) + @:DEALLOCATE(rhs_vf) ! Writing the footer of and closing the run-time information file if (proc_rank == 0 .and. run_time_info) then diff --git a/src/simulation/m_viscous.f90 b/src/simulation/m_viscous.fpp similarity index 99% rename from src/simulation/m_viscous.f90 rename to src/simulation/m_viscous.fpp index 778f211a6d..ad43e7cef3 100644 --- a/src/simulation/m_viscous.f90 +++ b/src/simulation/m_viscous.fpp @@ -2,6 +2,8 @@ !! @file m_viscous.f90 !! @brief Contains module m_viscous +#:include 'macros.fpp' + !> @brief The module contains the subroutines used to compute viscous terms. module m_viscous @@ -32,9 +34,7 @@ module m_viscous subroutine s_initialize_viscous_module() integer :: i, j !< generic loop iterators - - - allocate (Res(1:2, 1:maxval(Re_size))) + @:ALLOCATE(Res(1:2, 1:maxval(Re_size))) do i = 1, 2 do j = 1, Re_size(i) @@ -1462,7 +1462,7 @@ subroutine s_reconstruct_cell_boundary_values_visc_deriv(v_vf, vL_x, vL_y, vL_z, end subroutine s_reconstruct_cell_boundary_values_visc_deriv ! -------------------- subroutine s_finalize_viscous_module() - deallocate (Res) + @:DEALLOCATE(Res) end subroutine s_finalize_viscous_module end module m_viscous diff --git a/src/simulation/m_weno.fpp b/src/simulation/m_weno.fpp index d3ed7ff346..5e5df9e65b 100644 --- a/src/simulation/m_weno.fpp +++ b/src/simulation/m_weno.fpp @@ -2,6 +2,8 @@ !! @file m_weno.f90 !! @brief Contains module m_weno +#:include 'macros.fpp' + !> @brief Weighted essentially non-oscillatory (WENO) reconstruction scheme !! that is supplemented with monotonicity preserving bounds (MPWENO) !! and a mapping function that boosts the accuracy of the non-linear @@ -143,20 +145,20 @@ contains is3%end = p - is3%beg - allocate (poly_coef_cbL_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbL_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (poly_coef_cbR_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbR_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (d_cbL_x(0:weno_polyn, is1%beg + weno_polyn:is1%end - weno_polyn)) - allocate (d_cbR_x(0:weno_polyn, is1%beg + weno_polyn:is1%end - weno_polyn)) + @:ALLOCATE(d_cbL_x(0:weno_polyn, is1%beg + weno_polyn:is1%end - weno_polyn)) + @:ALLOCATE(d_cbR_x(0:weno_polyn, is1%beg + weno_polyn:is1%end - weno_polyn)) - allocate (beta_coef_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(beta_coef_x(is1%beg + weno_polyn:is1%end - weno_polyn, 0:weno_polyn, & 0:2*(weno_polyn - 1))) call s_compute_weno_coefficients(1, is1) - allocate (v_rs_ws_x(is1%beg:is1%end, & + @:ALLOCATE(v_rs_ws_x(is1%beg:is1%end, & is2%beg:is2%end, is3%beg:is3%end, 1:sys_size)) ! ================================================================== @@ -175,20 +177,20 @@ contains is3%end = p - is3%beg - allocate (poly_coef_cbL_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbL_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (poly_coef_cbR_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbR_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (d_cbL_y(0:weno_polyn, is2%beg + weno_polyn:is2%end - weno_polyn)) - allocate (d_cbR_y(0:weno_polyn, is2%beg + weno_polyn:is2%end - weno_polyn)) + @:ALLOCATE(d_cbL_y(0:weno_polyn, is2%beg + weno_polyn:is2%end - weno_polyn)) + @:ALLOCATE(d_cbR_y(0:weno_polyn, is2%beg + weno_polyn:is2%end - weno_polyn)) - allocate (beta_coef_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(beta_coef_y(is2%beg + weno_polyn:is2%end - weno_polyn, 0:weno_polyn, & 0:2*(weno_polyn - 1))) call s_compute_weno_coefficients(2, is2) - allocate (v_rs_ws_y(is2%beg:is2%end, & + @:ALLOCATE(v_rs_ws_y(is2%beg:is2%end, & is1%beg:is1%end, is3%beg:is3%end, 1:sys_size)) ! ================================================================== @@ -200,20 +202,20 @@ contains is1%beg = -buff_size; is1%end = m - is1%beg is3%beg = -buff_size; is3%end = p - is3%beg - allocate (poly_coef_cbL_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbL_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (poly_coef_cbR_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(poly_coef_cbR_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & 0:weno_polyn - 1)) - allocate (d_cbL_z(0:weno_polyn, is3%beg + weno_polyn:is3%end - weno_polyn)) - allocate (d_cbR_z(0:weno_polyn, is3%beg + weno_polyn:is3%end - weno_polyn)) + @:ALLOCATE(d_cbL_z(0:weno_polyn, is3%beg + weno_polyn:is3%end - weno_polyn)) + @:ALLOCATE(d_cbR_z(0:weno_polyn, is3%beg + weno_polyn:is3%end - weno_polyn)) - allocate (beta_coef_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & + @:ALLOCATE(beta_coef_z(is3%beg + weno_polyn:is3%end - weno_polyn, 0:weno_polyn, & 0:2*(weno_polyn - 1))) call s_compute_weno_coefficients(3, is3) - allocate (v_rs_ws_z(is3%beg:is3%end, & + @:ALLOCATE(v_rs_ws_z(is3%beg:is3%end, & is2%beg:is2%end, is1%beg:is1%end, 1:sys_size)) ! ================================================================== @@ -1289,34 +1291,34 @@ subroutine s_weno(v_vf, vL_rs_vf_x, vL_rs_vf_y, vL_rs_vf_z, vR_rs_vf_x, vR_rs_vf ! Deallocating the WENO-stencil of the WENO-reconstructed variables !deallocate(vL_rs_vf_x, vR_rs_vf_x) - deallocate (v_rs_ws_x) + @:DEALLOCATE(v_rs_ws_x) ! Deallocating WENO coefficients in x-direction ==================== - deallocate (poly_coef_cbL_x, poly_coef_cbR_x) - deallocate (d_cbL_x, d_cbR_x) - deallocate (beta_coef_x) + @:DEALLOCATE(poly_coef_cbL_x, poly_coef_cbR_x) + @:DEALLOCATE(d_cbL_x, d_cbR_x) + @:DEALLOCATE(beta_coef_x) ! ================================================================== ! Deallocating WENO coefficients in y-direction ==================== if (n == 0) return !deallocate(vL_rs_vf_y, vR_rs_vf_y) - deallocate (v_rs_ws_y) + @:DEALLOCATE(v_rs_ws_y) - deallocate (poly_coef_cbL_y, poly_coef_cbR_y) - deallocate (d_cbL_y, d_cbR_y) - deallocate (beta_coef_y) + @:DEALLOCATE(poly_coef_cbL_y, poly_coef_cbR_y) + @:DEALLOCATE(d_cbL_y, d_cbR_y) + @:DEALLOCATE(beta_coef_y) ! ================================================================== ! Deallocating WENO coefficients in z-direction ==================== if (p == 0) return !deallocate(vL_rs_vf_z, vR_rs_vf_z) - deallocate (v_rs_ws_z) + @:DEALLOCATE(v_rs_ws_z) - deallocate (poly_coef_cbL_z, poly_coef_cbR_z) - deallocate (d_cbL_z, d_cbR_z) - deallocate (beta_coef_z) + @:DEALLOCATE(poly_coef_cbL_z, poly_coef_cbR_z) + @:DEALLOCATE(d_cbL_z, d_cbR_z) + @:DEALLOCATE(beta_coef_z) ! ================================================================== end subroutine s_finalize_weno_module ! -------------------------------- diff --git a/src/simulation/p_main.fpp b/src/simulation/p_main.fpp index 7c258ac489..06c8330858 100644 --- a/src/simulation/p_main.fpp +++ b/src/simulation/p_main.fpp @@ -56,11 +56,8 @@ program p_main use openacc #endif - use nvtx + use m_nvtx -#ifdef _OPENACC - use openacc -#endif ! ========================================================================== implicit none @@ -105,11 +102,8 @@ program p_main #endif devtype = acc_get_device_type() - devNum = acc_get_num_devices(devtype) - dev = mod(local_rank, devNum) - - ! Note: I0 (Iw, where w=0) is a Fortran 95 feature. - print '("Global Rank "I0" (/"I0") - Local Rank "I0" (/"I0") - OpenACC (Local) Device "I0" (/"I0")")', proc_rank, num_procs, local_rank, local_size, dev, devNum + devNum = acc_get_num_devices(devtype) + dev = mod(local_rank, devNum) call acc_set_device_num(dev, devtype) #endif @@ -122,6 +116,8 @@ program p_main call s_assign_default_values_to_user_inputs() call s_read_input_file() call s_check_input_file() + + print '(" Simulating a "I0"x"I0"x"I0" case on "I0" rank(s)")', m, n, p, num_procs end if ! Broadcasting the user inputs to all of the processors and performing the @@ -228,18 +224,26 @@ program p_main ! Time-stepping Loop ======================================================= do if (proc_rank == 0) then - print *, '------ Time step ', t_step, 'of', t_step_stop, '----' + print '(" ["I3"%] Time step "I8" of "I0" @ t_step = "I0"")', & + int(100*(real(t_step + 1)/(t_step_stop - t_step_start + 1))), & + t_step - t_step_start + 1, & + t_step_stop - t_step_start + 1, & + t_step end if + mytime = mytime + dt if (probe_wrt) then do i = 1, sys_size -!$acc update host(q_cons_ts(1)%vf(i)%sf) + !$acc update host(q_cons_ts(1)%vf(i)%sf) end do end if call s_compute_derived_variables(t_step) - if (DEBUG) print *, 'Computed derived vars' + +#ifdef DEBUG + print *, 'Computed derived vars' +#endif ! Total-variation-diminishing (TVD) Runge-Kutta (RK) time-steppers if (time_stepper == 1) then diff --git a/toolchain/dependencies/CMakeLists.txt b/toolchain/dependencies/CMakeLists.txt index da69717ea9..f1ba32aad8 100644 --- a/toolchain/dependencies/CMakeLists.txt +++ b/toolchain/dependencies/CMakeLists.txt @@ -3,8 +3,11 @@ cmake_minimum_required(VERSION 3.12) -project(MFC-Dependencies) +project(MFC-Dependencies LANGUAGES Fortran) +if (MFC_SILO OR NOT CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + enable_language(C CXX) +endif() # Imports include(GNUInstallDirs) @@ -12,43 +15,52 @@ include(ExternalProject) # Options -option(MFC_BUILD_FFTW "Build the FFTW3 dependency" OFF) -option(MFC_BUILD_HDF5 "Build the HDF5 dependency" OFF) -option(MFC_BUILD_SILO "Build the SILO dependency" OFF) +option(MFC_FFTW "Build the FFTW3 dependency" OFF) +option(MFC_HDF5 "Build the HDF5 dependency" OFF) +option(MFC_SILO "Build the SILO dependency" OFF) # FFTW3 -if (MFC_BUILD_FFTW) - ExternalProject_Add(fftw - URL "http://www.fftw.org/fftw-3.3.10.tar.gz" - CMAKE_ARGS -DBUILD_TESTS=OFF - -DBUILD_SHARED_LIBS=OFF - "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" - ) +if (MFC_FFTW) + if (NOT CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + ExternalProject_Add(fftw + URL "http://www.fftw.org/fftw-3.3.10.tar.gz" + CMAKE_ARGS -DBUILD_TESTS=OFF + -DBUILD_SHARED_LIBS=OFF + "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + ) + else() + message(WARNING "The Fortran compiler vendor is Cray so FFTW3 will not be built. We will use cray-fftw instead.") + add_custom_target(fftw) + endif() endif() # HDF5 -find_package(HDF5) -if (MFC_BUILD_HDF5 OR (MFC_BUILD_SILO AND NOT HDF5_FOUND)) - ExternalProject_Add(hdf5 - GIT_REPOSITORY "https://github.com/HDFGroup/hdf5" - GIT_TAG hdf5-1_12_2 - GIT_SHALLOW ON - GIT_PROGRESS ON - CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" - -DBUILD_SHARED_LIBS=OFF - -DFORTRAN_LIBRARIES=ON - -DBUILD_TESTING=OFF - -DHDF5_BUILD_UTILS=OFF - -DHDF5_BUILD_TOOLS=OFF - -DHDF5_BUILD_EXAMPLES=OFF - ) +if (MFC_HDF5) + if (NOT CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + ExternalProject_Add(hdf5 + GIT_REPOSITORY "https://github.com/HDFGroup/hdf5" + GIT_TAG hdf5-1_12_2 + GIT_SHALLOW ON + GIT_PROGRESS ON + CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DFORTRAN_LIBRARIES=ON + -DBUILD_TESTING=OFF + -DHDF5_BUILD_UTILS=OFF + -DHDF5_BUILD_TOOLS=OFF + -DHDF5_BUILD_EXAMPLES=OFF + ) + else() + message(WARNING "The Fortran compiler vendor is Cray so HDF5 will not be built. We will use cray-hdf5 instead.") + add_custom_target(hdf5) + endif() endif() # SILO -if (MFC_BUILD_SILO) +if (MFC_SILO) ExternalProject_Add(silo GIT_REPOSITORY "https://github.com/henryleberre/Silo" GIT_TAG 964d993039b467e3d472d3a70afd03380fe2c320 @@ -57,12 +69,13 @@ if (MFC_BUILD_SILO) CMAKE_ARGS -DSILO_ENABLE_SHARED=OFF -DSILO_ENABLE_SILOCK=OFF -DSILO_ENABLE_BROWSER=OFF - -DHDF5_USE_STATIC_LIBRARIES=ON - "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + -DHDF5_USE_STATIC_LIBRARIES=ON + "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" $<$:"-DSILO_HDF5_DIR=\"${CMAKE_INSTALL_PREFIX}\""> ) - if (MFC_BUILD_HDF5) + if (MFC_HDF5 AND NOT CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") add_dependencies(silo hdf5) endif() endif() + diff --git a/toolchain/mfc.py b/toolchain/mfc.py index b0cd6796da..36cfb94dce 100644 --- a/toolchain/mfc.py +++ b/toolchain/mfc.py @@ -1,114 +1,69 @@ #!/usr/bin/env python3 -import os -import signal -import getpass -import platform -import itertools - -from mfc import args -from mfc import build -from mfc.cfg import user -from mfc.cfg import lock -from mfc.run import run -from mfc.tests import tests - -from mfc.util.common import MFC_LOGO, MFCException, quit -from mfc.util.common import delete_directory, format_list_to_string -from mfc.util.common import does_command_exist -from mfc.util.printer import cons - - -class MFCState: - def __init__(self) -> None: - self.user = user.MFCUser() - self.lock = lock.MFCLock(self.user) - self.test = tests.MFCTest(self) - self.args = args.parse(self) - self.run = run.MFCRun(self) - - self.__handle_mode() - self.__print_greeting() - self.__checks() - self.__run() - - - def __handle_mode(self): - # Handle mode change - if self.args["mode"] == self.lock.mode and self.args["mpi"] == self.lock.mpi: - return - - cons.print(f"[bold yellow]Switching to [bold magenta]{self.args['mode']}[/bold magenta] mode from [bold magenta]{self.lock.mode}[/bold magenta] mode:[/bold yellow]") - - self.lock.mode = self.args["mode"] - self.lock.mpi = self.args["mpi"] - self.lock.write() - - for target_name in build.get_mfc_target_names(): - t = build.get_target(target_name) - dirpath = build.get_build_dirpath(t) - cons.print(f"[bold red] - Removing {os.path.relpath(dirpath)}[/bold red]") - delete_directory(dirpath) - - - def __print_greeting(self): - MFC_LOGO_LINES = MFC_LOGO.splitlines() - max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ]) +import signal, getpass, platform, itertools, dataclasses - host_line = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]" +from mfc import args, lock, build, bench, state +from mfc.state import ARG +from mfc.run import run +from mfc.test import test +from mfc.common import MFC_LOGO, MFCException, quit, format_list_to_string, does_command_exist +from mfc.printer import cons - targets_line = \ - f"[bold]--targets: {format_list_to_string([ f'[magenta]{target}[/magenta]' for target in self.args['targets']], 'None')}[/bold]" - MFC_SIDEBAR_LINES = [ - "", - f"[bold]{host_line}[/bold]", - '-' * len(host_line), - "", - "", - f"[bold]--jobs: [magenta]{self.args['jobs']}[/magenta][/bold]", - f"[bold]--mode: [magenta]{self.lock.mode}[/magenta][/bold]", - targets_line if self.args["command"] != "test" else "", - "", - "", - "[yellow]$ ./mfc.sh \[build, run, test, clean] --help[/yellow]", - ] +def __print_greeting(): + MFC_LOGO_LINES = MFC_LOGO.splitlines() + max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ]) + host_line = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]" + targets_line = f"[bold]--targets {format_list_to_string([ f'[magenta]{target}[/magenta]' for target in ARG('targets')], 'None')}[/bold]" - for a, b in itertools.zip_longest(MFC_LOGO_LINES, MFC_SIDEBAR_LINES): - lhs = a.ljust(max_logo_line_length) - rhs = b if b is not None else '' - cons.print( - f"[bold blue] {lhs} [/bold blue] {rhs}", - highlight=False - ) + MFC_SIDEBAR_LINES = [ + "", + f"[bold]{host_line}[/bold]", + '-' * len(host_line), + "", + f"[bold]--jobs [magenta]{ARG('jobs')}[/magenta][/bold]" + ] + [ + f"[bold]--{'' if getattr(state.gCFG, field.name) else 'no-'}{field.name}[/bold]" for field in dataclasses.fields(state.gCFG) + ] + [ + targets_line if ARG("command") != "test" else "", + "", + "[yellow]$ ./mfc.sh \[build, run, test, clean] --help[/yellow]", + ] - cons.print() + for a, b in itertools.zip_longest(MFC_LOGO_LINES, MFC_SIDEBAR_LINES): + lhs = a.ljust(max_logo_line_length) + rhs = b if b is not None else '' + cons.print( + f"[bold blue] {lhs} [/bold blue] {rhs}", + highlight=False + ) + cons.print() - def __checks(self): - if not does_command_exist("cmake"): - raise MFCException("CMake is required to build MFC but couldn't be located on your system. Please ensure it installed and discoverable (e.g in your system's $PATH).") - if not does_command_exist("mpif90") and self.args["mpi"]: - raise MFCException("mpif90 couldn't be located on your system. We therefore assume MPI is not available on your system. It is required to build MFC. Please ensure it is installed and discoverable (e.g in your system's $PATH).") +def __checks(): + if not does_command_exist("cmake"): + raise MFCException("CMake is required to build MFC but couldn't be located on your system. Please ensure it installed and discoverable (e.g in your system's $PATH).") - def __run(self): - if self.args["command"] == "test": - self.test.execute() - elif self.args["command"] == "run": - self.run.run() - elif self.args["command"] == "build": - build.build(self) - elif self.args["command"] == "clean": - for target in self.args["targets"]: - build.clean_target(self, target) +def __run(): + {"test": test.test, "run": run.run, "build": build.build, + "clean": build.clean, "bench": bench.bench + }[ARG("command")]() if __name__ == "__main__": try: - MFCState() + lock.init() + state.gARG = args.parse(state.gCFG) + + lock.switch(state.MFCConfig.from_dict(state.gARG)) + + __print_greeting() + __checks() + __run() + except MFCException as exc: cons.reset() cons.print(f"""\ @@ -129,4 +84,3 @@ def __run(self): """) quit(signal.SIGTERM) - diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 6ddffaed43..95b70c8f39 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -1,13 +1,11 @@ import argparse +from .build import get_mfc_target_names, get_target_names, get_dependencies_names +from .common import format_list_to_string +from .test.test import CASES as TEST_CASES -from .build import get_mfc_target_names -from .build import get_target_names -from .build import get_dependencies_names -from .util.common import format_list_to_string - -def parse(mfc): +def parse(config): from .run.engines import ENGINES from .run.mpi_bins import BINARIES @@ -21,27 +19,34 @@ def parse(mfc): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - mode_names = [ e.name for e in mfc.user.modes ] - parsers = parser.add_subparsers(dest="command") run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) clean = parsers.add_parser(name="clean", help="Clean build artifacts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + def add_common_arguments(p, mask = None): + if mask is None: + mask = "" - def add_common_arguments(p, mask=""): if "t" not in mask: p.add_argument("-t", "--targets", metavar="TARGET", nargs="+", type=str.lower, choices=get_target_names(), default=get_mfc_target_names(), help=f"Space separated list of targets to act upon. Allowed values are: {format_list_to_string(get_target_names())}.") if "m" not in mask: - p.add_argument("-m", "--mode", metavar="MODE", type=str.lower, choices=mode_names, default=mfc.lock.mode, - help=f"Change MFC's mode. Allowed values are: {format_list_to_string(mode_names)}") + p.add_argument( "--mpi", action="store_true", help=f"Build with MPI.") + p.add_argument( "--no-mpi", action="store_false", dest="mpi", help=f"Build without MPI.") + p.add_argument( "--gpu", action="store_true", help=f"Build with GPU Acceleration.") + p.add_argument( "--no-gpu", action="store_false", dest="gpu", help=f"Build without GPU Acceleration.") + p.add_argument( "--debug", action="store_true", help=f"Build in Debug mode.") + p.add_argument("--no-debug", action="store_false", dest="debug", help=f"Build in Release mode.") + + p.set_defaults(mpi=config.mpi, gpu=config.gpu, debug=config.debug) if "j" not in mask: - p.add_argument("-j", "--jobs", metavar="JOBS", type=int, default=int(mfc.user.build.threads), - help="Allows for JOBS concurrent jobs.") + p.add_argument("-j", "--jobs", metavar="JOBS", type=int, default=1, help="Allows for JOBS concurrent jobs.") if "v" not in mask: p.add_argument("-v", "--verbose", action="store_true", help="Enables verbose compiler & linker output.") @@ -50,11 +55,6 @@ def add_common_arguments(p, mask=""): for name in get_dependencies_names(): p.add_argument(f"--no-{name}", action="store_true", help=f"Do not build the {name} dependency. Use the system's instead.") - p.add_argument(f"--mpi", action="store_true", help="Build with MPI.") - p.add_argument(f"--no-mpi", action="store_false", dest="mpi", help="Build without MPI.") - - p.set_defaults(mpi=mfc.lock.mpi) - # === BUILD === add_common_arguments(build) @@ -65,10 +65,10 @@ def add_common_arguments(p, mask=""): # === TEST === add_common_arguments(test, "t") - test.add_argument("-g", "--generate", action="store_true", help="Generate golden files.") + test.add_argument( "--generate", action="store_true", help="Generate golden files.") test.add_argument("-l", "--list", action="store_true", help="List all available tests.") - test.add_argument("-f", "--from", default=mfc.test.cases[0].get_uuid(), type=str, help="First test UUID to run.") - test.add_argument("-t", "--to", default=mfc.test.cases[-1].get_uuid(), type=str, help="Last test UUID to run.") + test.add_argument("-f", "--from", default=TEST_CASES[0].get_uuid(), type=str, help="First test UUID to run.") + test.add_argument("-t", "--to", default=TEST_CASES[-1].get_uuid(), type=str, help="Last test UUID to run.") test.add_argument("-o", "--only", nargs="+", type=str, default=[], metavar="L", help="Only run tests with UUIDs or hashes L.") test.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Serial) Override MPI execution binary") test.add_argument("-r", "--relentless", action="store_true", default=False, help="Run all tests, even if multiple fail.") @@ -78,22 +78,24 @@ def add_common_arguments(p, mask=""): engines = [ e.slug for e in ENGINES ] add_common_arguments(run) - run.add_argument("input", metavar="INPUT", type=str, help="Input file to run.") - run.add_argument("-e", "--engine", choices=engines, type=str, default=engines[0], help="Job execution/submission engine choice.") - run.add_argument("-p", "--partition", metavar="PARTITION", type=str, default=mfc.user.run.partition, help="(Batch) Partition for job submission.") - run.add_argument("-N", "--nodes", metavar="NODES", type=int, default=mfc.user.run.nodes, help="(Batch) Number of nodes.") - run.add_argument("-n", "--cpus-per-node", metavar="CPUS", type=int, default=mfc.user.run.cpus_per_node, help="Number of tasks per node.") - run.add_argument("-g", "--gpus-per-node", metavar="GPUS", type=int, default=mfc.user.run.gpus_per_node, help="(Batch) Number of GPUs per node.") - run.add_argument("-w", "--walltime", metavar="WALLTIME", type=str, default=mfc.user.run.walltime, help="(Batch) Walltime.") - run.add_argument("-a", "--account", metavar="ACCOUNT", type=str, default=mfc.user.run.account, help="(Batch) Account to charge.") - run.add_argument("-@", "--email", metavar="EMAIL", type=str, default=mfc.user.run.email, help="(Batch) Email for job notification.") - run.add_argument("-#", "--name", metavar="NAME", type=str, default=mfc.user.run.name, help="(Batch) Job name.") - run.add_argument("-f", "--flags", metavar="FLAGS", nargs="+", type=str, default=mfc.user.run.flags, help="(Batch) Additional batch options.") - run.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Interactive) Override MPI execution binary") - run.add_argument("-s", "--scratch", action="store_true", default=False, help="Build from scratch.") - run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.") - run.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") - run.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.") + run.add_argument("input", metavar="INPUT", type=str, help="Input file to run.") + run.add_argument("-e", "--engine", choices=engines, type=str, default=engines[0], help="Job execution/submission engine choice.") + run.add_argument("-p", "--partition", metavar="PARTITION", type=str, default="", help="(Batch) Partition for job submission.") + run.add_argument("-N", "--nodes", metavar="NODES", type=int, default=1, help="(Batch) Number of nodes.") + run.add_argument("-n", "--tasks-per-node", metavar="TASKS", type=int, default=1, help="Number of tasks per node.") + run.add_argument("-w", "--walltime", metavar="WALLTIME", type=str, default="01:00:00", help="(Batch) Walltime.") + run.add_argument("-a", "--account", metavar="ACCOUNT", type=str, default="", help="(Batch) Account to charge.") + run.add_argument("-@", "--email", metavar="EMAIL", type=str, default="", help="(Batch) Email for job notification.") + run.add_argument("-#", "--name", metavar="NAME", type=str, default="MFC", help="(Batch) Job name.") + run.add_argument("-f", "--flags", metavar="FLAGS", nargs="+", type=str, default=[], help="(Batch) Additional batch options.") + run.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Interactive) Override MPI execution binary") + run.add_argument("-s", "--scratch", action="store_true", default=False, help="Build from scratch.") + run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.") + run.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") + run.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.") + + # === BENCH === + add_common_arguments(bench, "t") args: dict = vars(parser.parse_args()) @@ -105,7 +107,8 @@ def append_defaults_to_data(name: str, parser): if not key in args: args[key] = val - for a, b in [("run", run), ("test", test), ("build", build), ("clean", clean)]: + for a, b in [("run", run ), ("test", test ), ("build", build), + ("clean", clean), ("bench", bench)]: append_defaults_to_data(a, b) if args["command"] is None: diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py new file mode 100644 index 0000000000..55ba99f45d --- /dev/null +++ b/toolchain/mfc/bench.py @@ -0,0 +1,58 @@ +import os, json, time, typing, datetime, subprocess + +import rich.table + +from .printer import cons +from .state import ARG +from .build import build_targets +from .common import system, MFC_SUBDIR +from . import sched + +def bench(): + build_targets(["pre_process", "simulation"]) + + cons.print("[bold]Benchmarking [magenta]simulation[/magenta]:[/bold]") + cons.indent() + + CASES = ["1D_bubblescreen", "1D_exercise_WENO", "1D_kapilashocktube"] + RESULTS = [] + + table = rich.table.Table(show_lines=False, show_edge=False) + table.add_column("Case") + table.add_column("(Simulation) Runtime (s)") + + def __worker(case: str): + nonlocal RESULTS + + system(["./mfc.sh", "run", f"examples/{case}/case.py", "--no-build", "-t", "pre_process"], stdout=subprocess.DEVNULL) + start = time.monotonic() + system(["./mfc.sh", "run", f"examples/{case}/case.py", "--no-build", "-t", "simulation"], stdout=subprocess.DEVNULL) + end = time.monotonic() + runtime = datetime.timedelta(seconds=end - start).total_seconds() + + RESULTS.append({ + "name": f"Simulation: {case}", + "unit": "seconds", + "value": runtime + }) + + table.add_row(case, str(runtime)) + + tasks: typing.List[sched.Task] = [ + sched.Task(1, __worker, [ case ]) for case in CASES + ] + + cons.print() + sched.sched(tasks, 1 if ARG('case_optimization') else ARG('jobs')) + cons.print() + cons.unindent() + cons.print("[bold]Benchmark Results:[/bold]") + cons.print() + cons.raw.print(table) + cons.print() + + filepath = os.path.join(MFC_SUBDIR, "bench.json") + with open(filepath, "w") as f: + json.dump(RESULTS, f) + + cons.print(f"[bold green]✓[/bold green] Saved results to [magenta]{filepath}[/magenta].") diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 1f7644566b..f171f060f3 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -1,36 +1,39 @@ -from .util.printer import cons +import os, typing, dataclasses -from .util import common as common - -import os -import typing -import dataclasses +from .state import ARG +from .printer import cons +from . import common @dataclasses.dataclass class MFCTarget: + @dataclasses.dataclass + class Dependencies: + all: typing.List[str] + cpu: typing.List[str] + gpu: typing.List[str] + + def compute(self) -> typing.List[str]: + r = self.all[:] + r += self.gpu[:] if ARG("gpu") else self.cpu[:] + + return r + name: str flags: typing.List[str] - requires: typing.List[str] isDependency: bool isDefault: bool + requires: Dependencies TARGETS: typing.List[MFCTarget] = [ - MFCTarget(name='fftw', flags=['-DMFC_BUILD_FFTW=ON'], - isDependency=True, isDefault=False, requires=[]), - MFCTarget(name='hdf5', flags=['-DMFC_BUILD_HDF5=ON'], - isDependency=True, isDefault=False, requires=[]), - MFCTarget(name='silo', flags=['-DMFC_BUILD_SILO=ON'], - isDependency=True, isDefault=False, requires=["hdf5"]), - MFCTarget(name='pre_process', flags=['-DMFC_BUILD_PRE_PROCESS=ON'], - isDependency=False, isDefault=True, requires=[]), - MFCTarget(name='simulation', flags=['-DMFC_BUILD_SIMULATION=ON'], - isDependency=False, isDefault=True, requires=["fftw"]), - MFCTarget(name='post_process', flags=['-DMFC_BUILD_POST_PROCESS=ON'], - isDependency=False, isDefault=True, requires=['fftw', 'silo']), - MFCTarget(name="doc", flags=['-DMFC_BUILD_DOC=ON'], - isDependency=False, isDefault=False, requires=[]) + MFCTarget('fftw', ['-DMFC_FFTW=ON'], True, False, MFCTarget.Dependencies([], [], [])), + MFCTarget('hdf5', ['-DMFC_HDF5=ON'], True, False, MFCTarget.Dependencies([], [], [])), + MFCTarget('silo', ['-DMFC_SILO=ON'], True, False, MFCTarget.Dependencies(["hdf5"], [], [])), + MFCTarget('pre_process', ['-DMFC_PRE_PROCESS=ON'], False, True, MFCTarget.Dependencies([], [], [])), + MFCTarget('simulation', ['-DMFC_SIMULATION=ON'], False, True, MFCTarget.Dependencies([], ["fftw"], [])), + MFCTarget('post_process', ['-DMFC_POST_PROCESS=ON'], False, True, MFCTarget.Dependencies(['fftw', 'silo'], [], [])), + MFCTarget('documentation', ['-DMFC_DOCUMENTATION=ON'], False, False, MFCTarget.Dependencies([], [], [])) ] @@ -61,7 +64,9 @@ def get_build_dirpath(target: MFCTarget) -> str: # Get the directory that contains the target's CMakeLists.txt def get_cmake_dirpath(target: MFCTarget) -> str: - return os.sep.join([os.getcwd(), ["", "toolchain/dependencies"][int(target.isDependency)]]) + subdir = ["", os.sep.join(["toolchain", "dependencies"])][int(target.isDependency)] + + return os.sep.join([os.getcwd(), subdir]) def get_install_dirpath() -> str: @@ -69,16 +74,15 @@ def get_install_dirpath() -> str: def is_target_configured(target: MFCTarget) -> bool: - build_dirpath = get_build_dirpath(target) - return os.path.isdir(build_dirpath) + cmake_cachepath = os.sep.join([get_build_dirpath(target), "CMakeCache.txt"]) + return os.path.isfile(cmake_cachepath) -def clean_target(mfc, name: str): - cons.print(f"Cleaning [bold magenta]{name}[/bold magenta]:") +def clean_target(name: str): + cons.print(f"[bold]Cleaning [magenta]{name}[/magenta]:[/bold]") cons.indent() target = get_target(name) - mode = mfc.user.get_mode(mfc.args["mode"]) build_dirpath = get_build_dirpath(target) @@ -88,9 +92,9 @@ def clean_target(mfc, name: str): return clean = ["cmake", "--build", build_dirpath, "--target", "clean", - "--config", mode.type] + "--config", "Debug" if ARG("debug") else "Release" ] - if mfc.args["verbose"]: + if ARG("verbose"): clean.append("--verbose") common.system(clean, exception_text=f"Failed to clean the [bold magenta]{name}[/bold magenta] target.") @@ -98,14 +102,23 @@ def clean_target(mfc, name: str): cons.unindent() -def build_target(mfc, name: str, history: typing.List[str] = None): +def clean_targets(targets: typing.List[str]): + for target in targets: + clean_target(target) + + +def clean(): + clean_targets(ARG("targets")) + + +def build_target(name: str, history: typing.List[str] = None): cons.print(f"[bold]Building [magenta]{name}[/magenta]:[/bold]") cons.indent() if history is None: history = [] - if mfc.args["no_build"]: + if ARG("no_build"): cons.print("--no-build specified, skipping...") cons.unindent() return @@ -118,9 +131,8 @@ def build_target(mfc, name: str, history: typing.List[str] = None): history.append(name) target = get_target(name) - mode = mfc.user.get_mode(mfc.args["mode"]) - if target.isDependency and mfc.args[f"no_{target.name}"]: + if target.isDependency and ARG(f"no_{target.name}"): cons.print(f"--no-{target.name} given, skipping...") cons.unindent() return @@ -129,7 +141,7 @@ def build_target(mfc, name: str, history: typing.List[str] = None): cmake_dirpath = get_cmake_dirpath(target) install_dirpath = get_install_dirpath() - flags: list = target.flags.copy() + mode.flags + [ + flags: list = target.flags.copy() + [ # Disable CMake warnings intended for developers (us). # See: https://cmake.org/cmake/help/latest/manual/cmake.1.html. f"-Wno-dev", @@ -139,10 +151,10 @@ def build_target(mfc, name: str, history: typing.List[str] = None): f"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", # Set build type (e.g Debug, Release, etc.). # See: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html - f"-DCMAKE_BUILD_TYPE={mode.type}", + f"-DCMAKE_BUILD_TYPE={'Debug' if ARG('debug') else 'Release'}", # Used by FIND_PACKAGE (/FindXXX) to search for packages, with the # second heighest level of priority, still letting users manually - # specify _ROOT, which has precedent over CMAKE_PREFIX_PATH. + # specify _ROOT, which has precedence over CMAKE_PREFIX_PATH. # See: https://cmake.org/cmake/help/latest/command/find_package.html. f"-DCMAKE_PREFIX_PATH={install_dirpath}", # First directory that FIND_LIBRARY searches. @@ -153,41 +165,41 @@ def build_target(mfc, name: str, history: typing.List[str] = None): f"-DCMAKE_INSTALL_PREFIX={install_dirpath}", ] - # Use the faster and more modern Ninja generator if present - if common.does_command_exist("ninja"): - flags.append("-GNinja") - - if not mfc.args["mpi"]: - flags.append("-DMFC_WITH_MPI=OFF") + if not target.isDependency: + flags.append(f"-DMFC_MPI={ 'ON' if ARG('mpi') else 'OFF'}") + flags.append(f"-DMFC_OpenACC={'ON' if ARG('gpu') else 'OFF'}") configure = ["cmake"] + flags + ["-S", cmake_dirpath, "-B", build_dirpath] - build = ["cmake", "--build", build_dirpath, "--target", name, - "-j", mfc.args["jobs"], "--config", mode.type] - if mfc.args['verbose']: + build = ["cmake", "--build", build_dirpath, + "--target", name, + "-j", ARG("jobs"), + "--config", 'Debug' if ARG('debug') else 'Release'] + if ARG('verbose'): build.append("--verbose") install = ["cmake", "--install", build_dirpath] - # Only configure the first time if not is_target_configured(target): - for dependency_name in target.requires: - build_target(mfc, dependency_name, history) + for dependency_name in target.requires.compute(): + build_target(dependency_name, history) + common.delete_directory(build_dirpath) common.create_directory(build_dirpath) if common.system(configure, no_exception=True) != 0: - common.delete_directory(build_dirpath) + raise common.MFCException("Failed to configure the [bold magenta]{name}[/bold magenta] target.") - raise common.MFCException(f"Failed to configure the [bold magenta]{name}[/bold magenta] target.") - - common.system(build, exception_text=f"Failed to build the [bold magenta]{name}[/bold magenta] target.") - common.system(install, exception_text=f"Failed to install the [bold magenta]{name}[/bold magenta] target.") + common.system(build, exception_text="Failed to build the [bold magenta]{name}[/bold magenta] target.") + common.system(install, exception_text="Failed to install the [bold magenta]{name}[/bold magenta] target.") + cons.print(no_indent=True) - cons.unindent() -def build(mfc): - for target_name in mfc.args["targets"]: - build_target(mfc, target_name) +def build_targets(targets): + for target in targets: + build_target(target) + +def build(): + build_targets(ARG("targets")) diff --git a/toolchain/mfc/case.py b/toolchain/mfc/case.py new file mode 100644 index 0000000000..5eb9c1ba0a --- /dev/null +++ b/toolchain/mfc/case.py @@ -0,0 +1,23 @@ +import json, copy, dataclasses + +@dataclasses.dataclass(init=False) +class Case: + params: dict + + def __init__(self, params: dict) -> None: + self.params = copy.deepcopy(params) + + def get_parameters(self) -> str: + return self.params.keys() + + def has_parameter(self, key: str)-> bool: + return key in self.get_parameters() + + def gen_json_dict_str(self) -> str: + return json.dumps(self.params, indent=4) + + def __getitem__(self, key: str) -> str: + return self.params[key] + + def __setitem__(self, key: str, val: str): + self.params[key] = val diff --git a/toolchain/mfc/cfg/lock.py b/toolchain/mfc/cfg/lock.py deleted file mode 100644 index 7731629282..0000000000 --- a/toolchain/mfc/cfg/lock.py +++ /dev/null @@ -1,43 +0,0 @@ -from ..util import common -from ..cfg.user import MFCUser - -import os -import dataclasses - -MFC_LOCK_CURRENT_VERSION: int = 2 - -@dataclasses.dataclass -class MFCLock: - mode: str - mpi: bool - version: int - - def __init__(self, user: MFCUser): - if not os.path.exists(common.MFC_LOCK_FILEPATH): - common.create_file(common.MFC_LOCK_FILEPATH) - common.file_dump_yaml(common.MFC_LOCK_FILEPATH, { - "mpi": True, - "mode": user.modes[0].name, - "version": MFC_LOCK_CURRENT_VERSION - }) - - data: dict = common.file_load_yaml(common.MFC_LOCK_FILEPATH) - - self.version = int(data.get("version", "0")) - - # 0 is the default version in order to accommodate versions of mfc.sh - # prior to the introduction of the "version" attribute to the lock file. - - if self.version < MFC_LOCK_CURRENT_VERSION: - raise common.MFCException(f"""\ -There has been a breaking change to the MFC build system. Please delete your \ -build/ directory and run MFC again. (v{self.version} -> v{MFC_LOCK_CURRENT_VERSION}).\ -""") - - self.mode = data["mode"] - self.mpi = data['mpi'] - - - def write(self): - common.file_dump_yaml(common.MFC_LOCK_FILEPATH, dataclasses.asdict(self)) - diff --git a/toolchain/mfc/cfg/user.py b/toolchain/mfc/cfg/user.py deleted file mode 100644 index 180432ab18..0000000000 --- a/toolchain/mfc/cfg/user.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import typing -import dataclasses - -from ..util.common import MFC_USER_FILEPATH, MFCException, create_file, \ - file_load_yaml - - -@dataclasses.dataclass -class Mode: - name: str - type: str - flags: typing.List[str] - - def __init__(self, data: dict) -> None: - self.name = data["name"] - self.type = data["type"] - self.flags = data.get("flags", []) - - -@dataclasses.dataclass -class Build: - threads: int - - def __init__(self, data: dict) -> None: - self.threads = data.get("threads", 1) - - -@dataclasses.dataclass -class Run: - nodes: int - partition: str - cpus_per_node: int - gpus_per_node: int - walltime: str - account: str - email: str - name: str - flags: list - - def __init__(self, data: dict) -> None: - self.nodes = int(data.get("nodes", "")) - self.partition = data.get("partition", "") - self.cpus_per_node = int(data.get("cpus-per-node", "")) - self.gpus_per_node = int(data.get("gpus-per-node", "")) - self.walltime = data.get("walltime", "") - self.account = data.get("account", "") - self.email = data.get("email", "") - self.name = data.get("name", "") - self.flags = data.get("flags", []) - - -@dataclasses.dataclass -class MFCUser: - modes: list - build: Build - run: Run - - def __init__(self) -> None: - if not os.path.exists(MFC_USER_FILEPATH): - create_file(MFC_USER_FILEPATH) - - data: dict = file_load_yaml(MFC_USER_FILEPATH) - - self.build = Build(data["build"]) - self.run = Run (data["run"]) - self.modes = [ Mode(e) for e in data["modes"] ] - - def get_mode(self, name: str) -> Mode: - for mode in self.modes: - if mode.name == name: - return mode - - raise MFCException(f'MFCConf: Mode "{mode}" doesn\'t exist') diff --git a/toolchain/mfc/util/common.py b/toolchain/mfc/common.py similarity index 95% rename from toolchain/mfc/util/common.py rename to toolchain/mfc/common.py index 763e25b45d..3b15210e65 100644 --- a/toolchain/mfc/util/common.py +++ b/toolchain/mfc/common.py @@ -1,15 +1,11 @@ -import os -import yaml -import typing -import shutil -import subprocess +import os, yaml, typing, shutil, subprocess, dataclasses from .printer import cons from os.path import abspath, normpath, dirname, realpath -MFC_ROOTDIR = normpath(f"{dirname(realpath(__file__))}/../../..") +MFC_ROOTDIR = normpath(f"{dirname(realpath(__file__))}/../..") MFC_TESTDIR = abspath(f"{MFC_ROOTDIR}/tests") MFC_SUBDIR = abspath(f"{MFC_ROOTDIR}/build") MFC_DEV_FILEPATH = abspath(f"{MFC_ROOTDIR}/toolchain/mfc.dev.yaml") @@ -38,9 +34,13 @@ class MFCException(Exception): def system(command: typing.List[str], no_exception: bool = False, exception_text=None, on_error=lambda: None, cwd=None, stdout=None, stderr=None) -> int: cmd = [ str(x) for x in command if not isspace(str(x)) ] - cons.print(no_indent=True) + if stdout != subprocess.DEVNULL: + cons.print(no_indent=True) + cons.print(f"$ {' '.join(cmd)}") - cons.print(no_indent=True) + + if stdout != subprocess.DEVNULL: + cons.print(no_indent=True) r = subprocess.run(cmd, cwd=cwd, stdout=stdout, stderr=stderr) diff --git a/toolchain/mfc/lock.py b/toolchain/mfc/lock.py new file mode 100644 index 0000000000..b43ee5500a --- /dev/null +++ b/toolchain/mfc/lock.py @@ -0,0 +1,75 @@ +import os, dataclasses + +from . import build, state, common +from .state import MFCConfig +from .printer import cons + + +MFC_LOCK_CURRENT_VERSION: int = 3 + + +@dataclasses.dataclass +class MFCLockData: + config: MFCConfig + version: int + + +data: MFCLockData = None + + +def init(): + global data + + if not os.path.exists(common.MFC_LOCK_FILEPATH): + config = MFCConfig() + data = MFCLockData(config, MFC_LOCK_CURRENT_VERSION) + state.gCFG = config + + common.create_file(common.MFC_LOCK_FILEPATH) + write() + else: + load() + + +def load(): + global data + + d = common.file_load_yaml(common.MFC_LOCK_FILEPATH) + + # 0 is the default version in order to accommodate versions of mfc.sh + # prior to the introduction of the "version" attribute to the lock file. + + if d["version"] < MFC_LOCK_CURRENT_VERSION: + raise common.MFCException(f"""\ +There has been a breaking change to the MFC build system. Please delete your \ +build/ directory and run MFC again. (v{d["version"]} -> v{MFC_LOCK_CURRENT_VERSION}).\ +""") + + config = MFCConfig.from_dict(d["config"]) + data = MFCLockData(config, d["version"]) + state.gCFG = config + + +def write(): + global data + + common.file_dump_yaml(common.MFC_LOCK_FILEPATH, dataclasses.asdict(data)) + + +def switch(to: MFCConfig): + global data + + if to == data.config: + return + + cons.print(f"[bold yellow]Switching from {data.config} to {to}[/bold yellow]") + cons.print("") + + data.config = to + state.gCFG = to + write() + + for target_name in build.get_target_names(): + dirpath = build.get_build_dirpath(build.get_target(target_name)) + cons.print(f"[bold red]Removing {os.path.relpath(dirpath)}[/bold red]") + common.delete_directory(dirpath) diff --git a/toolchain/mfc/util/printer.py b/toolchain/mfc/printer.py similarity index 97% rename from toolchain/mfc/util/printer.py rename to toolchain/mfc/printer.py index 2786c0f765..91eda7f1e1 100644 --- a/toolchain/mfc/util/printer.py +++ b/toolchain/mfc/printer.py @@ -1,7 +1,6 @@ import typing -import rich -import rich.console +import rich, rich.console class MFCPrinter: diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index dfcb69ca73..f36872ac99 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -1,4 +1,5 @@ -from ..util import common +from .. import common +from ..state import ARG COMMON = [ @@ -63,7 +64,7 @@ SIMULATION = COMMON + [ - 'run_time_info', 't_step_old', 't_tol', 'debug', 'dt', 't_step_start', + 'run_time_info', 't_step_old', 't_tol', 'dt', 't_step_start', 't_step_stop', 't_step_save', 'time_stepper', 'weno_vars', 'weno_eps', 'char_decomp', 'mapped_weno', 'mp_weno', 'weno_avg', 'weno_Re_flux', 'riemann_solver', 'wave_speeds', 'avg_state', 'commute_err', 'split_err', @@ -148,7 +149,7 @@ CASE_OPTIMIZATION = [ "nb", "weno_order" ] -def get_input_dict_keys(target_name: str, args: list) -> list: +def get_input_dict_keys(target_name: str) -> list: result = None if target_name == "pre_process": result = PRE_PROCESS.copy() if target_name == "simulation": result = SIMULATION.copy() @@ -157,7 +158,7 @@ def get_input_dict_keys(target_name: str, args: list) -> list: if result == None: raise common.MFCException(f"[INPUT DICTS] Target {target_name} doesn't have an input dict.") - if not args["case_optimization"] or target_name != "simulation": + if not ARG("case_optimization") or target_name != "simulation": return result return [ x for x in result if x not in CASE_OPTIMIZATION ] diff --git a/toolchain/mfc/run/engines.py b/toolchain/mfc/run/engines.py index 49786cf4f7..698530155c 100644 --- a/toolchain/mfc/run/engines.py +++ b/toolchain/mfc/run/engines.py @@ -1,22 +1,9 @@ -import re -import os -import time -import copy -import queue -import typing -import datetime -import subprocess -import dataclasses -import multiprocessing - -from ..util.printer import cons - -from .. import build -from ..util import common - -from ..run import queues -from ..run import mpi_bins +import re, os, time, copy, queue, typing, datetime, subprocess, dataclasses, multiprocessing +from ..state import ARG, ARGS +from ..printer import cons +from .. import build, common +from ..run import queues, mpi_bins from ..run.input import MFCInputFile @@ -25,8 +12,7 @@ class Engine: name: str slug: str - def init(self, mfc, input: MFCInputFile) -> None: - self.mfc = mfc + def init(self, input: MFCInputFile) -> None: self.input = input self._init() @@ -40,9 +26,6 @@ def get_args(self) -> typing.List[str]: def run(self, names: typing.List[str]) -> None: raise common.MFCException(f"MFCEngine::run: not implemented for {self.name}.") - def validate_job_options(self) -> None: - raise common.MFCException(f"MFCEngine::validate_job_options: not implemented for {self.name}.") - def get_binpath(self, target: str) -> str: return os.sep.join([build.get_install_dirpath(), "bin", target]) @@ -56,27 +39,26 @@ def __init__(self) -> None: super().__init__("Interactive", "interactive") def _init(self) -> None: - self.mpibin = mpi_bins.get_binary(self.mfc.args) + self.mpibin = mpi_bins.get_binary() # If using MPI, we don't know yet whether this engine works - self.bKnowWorks = not self.mfc.args["mpi"] + self.bKnowWorks = not ARG("mpi") def get_args(self) -> str: return f"""\ -Nodes (-N) {self.mfc.args['nodes']} -CPUs (/node) (-n) {self.mfc.args['cpus_per_node']} -GPUs (/node) (-g) {self.mfc.args["gpus_per_node"]} +Nodes (-N) {ARG('nodes')} +Tasks (/node) (-n) {ARG('tasks_per_node')} MPI Binary (-b) {self.mpibin.bin}\ """ def get_exec_cmd(self, target_name: str) -> typing.List[str]: binpath = self.get_binpath(target_name) - if not self.mfc.args["mpi"]: + if not ARG("mpi"): return [binpath] - flags = self.mfc.args["flags"][:] + flags = ARG("flags")[:] - return [self.mpibin.bin] + self.mpibin.gen_params(self.mfc.args) + flags + [binpath] + return [self.mpibin.bin] + self.mpibin.gen_params() + flags + [binpath] def run(self, names: typing.List[str]) -> None: @@ -84,7 +66,7 @@ def run(self, names: typing.List[str]) -> None: # Fix MFlowCode/MFC#21: Check whether attempting to run a job will hang # forever. This can happen when using the wrong queue system. - work_timeout = 10 + work_timeout = 30 cons.print(f"Ensuring the [bold magenta]Interactive Engine[/bold magenta] works ({work_timeout}s timeout):") @@ -95,7 +77,7 @@ def run(self, names: typing.List[str]) -> None: p = multiprocessing.Process( target=_interactive_working_worker, args=( - [self.mpibin.bin] + self.mpibin.gen_params(self.mfc.args) + test_cmd, + [self.mpibin.bin] + self.mpibin.gen_params() + test_cmd, q, )) @@ -121,7 +103,7 @@ def run(self, names: typing.List[str]) -> None: cons.print(f"[bold]Running [magenta]{name}[/magenta][/bold]:") cons.indent() - if not self.mfc.args["dry_run"]: + if not ARG("dry_run"): start_time = time.monotonic() common.system(self.get_exec_cmd(name), cwd=self.input.case_dirpath) end_time = time.monotonic() @@ -131,10 +113,6 @@ def run(self, names: typing.List[str]) -> None: cons.unindent() - def validate_job_options(self, mfc) -> None: - if mfc.args["nodes"] != 1: - raise common.MFCException("InteractiveEngine: Only node can be used with the interactive engine.") - class BatchEngine(Engine): def __init__(self) -> None: @@ -142,13 +120,12 @@ def __init__(self) -> None: def get_args(self) -> str: return f"""\ -Nodes (-N) {self.mfc.args['nodes']} -CPUs (/node) (-n) {self.mfc.args['cpus_per_node']} -GPUs (/node) (-g) {self.mfc.args["gpus_per_node"]} -Walltime (-w) {self.mfc.args["walltime"]} -Partition (-p) {self.mfc.args["partition"]} -Account (-a) {self.mfc.args["account"]} -Email (-@) {self.mfc.args["email"]} +Nodes (-N) {ARG('nodes')} +Tasks (/node) (-n) {ARG('tasks_per_node')} +Walltime (-w) {ARG("walltime")} +Partition (-p) {ARG("partition")} +Account (-a) {ARG("account")} +Email (-@) {ARG("email")} """ def run(self, names: typing.List[str]) -> None: @@ -160,7 +137,7 @@ def run(self, names: typing.List[str]) -> None: self.__create_batch_file(system, names) - if not self.mfc.args["dry_run"]: + if not ARG("dry_run"): self.__execute_batch_file(system, names) cons.print("[bold yellow]INFO:[/bold yellow] Batch file submitted! Please check your queue system for the job status.") @@ -173,7 +150,7 @@ def __get_batch_dirpath(self) -> str: return copy.copy(self.input.case_dirpath) def __get_batch_filename(self, names: typing.List[str]) -> str: - return f"{self.mfc.args['name']}.sh" + return f"{ARG('name')}.sh" def __get_batch_filepath(self, names: typing.List[str]): return os.path.abspath(os.sep.join([ @@ -199,17 +176,15 @@ def __generate_prologue(self, system: queues.QueueSystem, names: typing.List[str TABLE_TITLE_FORMAT="| %8s %-96s |\\n" TABLE_CONTENT=$(cat <<-END $(printf "$TABLE_FORMAT_LINE" "Start-time:" "$(date +%T)" "Start-date:" "$(date +%T)") -$(printf "$TABLE_FORMAT_LINE" "Partition:" "{self.mfc.args["partition"]}" "Walltime:" "{self.mfc.args["walltime"]}") -$(printf "$TABLE_FORMAT_LINE" "Account:" "{self.mfc.args["account"]}" "Nodes:" "{self.mfc.args["nodes"]}") -$(printf "$TABLE_FORMAT_LINE" "CPUs (/node):" "{self.mfc.args["cpus_per_node"]}" "GPUs (/node):" "{self.mfc.args["gpus_per_node"]}") -$(printf "$TABLE_FORMAT_LINE" "Job Name:" "{self.mfc.args["name"]}" "Engine" "{self.mfc.args["engine"]}") -$(printf "$TABLE_FORMAT_LINE" "Queue System:" "{system.name}" "Mode:" "{self.mfc.args["mode"]}") -$(printf "$TABLE_FORMAT_LINE" "Email:" "{self.mfc.args["email"]}" "" "") +$(printf "$TABLE_FORMAT_LINE" "Partition:" "{ARG("partition")}" "Walltime:" "{ARG("walltime")}") +$(printf "$TABLE_FORMAT_LINE" "Account:" "{ARG("account")}" "Nodes:" "{ARG("nodes")}") +$(printf "$TABLE_FORMAT_LINE" "Job Name:" "{ARG("name")}" "Engine" "{ARG("engine")}") +$(printf "$TABLE_FORMAT_LINE" "Queue System:" "{system.name}" "Email:" "{ARG("email")}") END ) printf "$TABLE_HEADER" -printf "$TABLE_TITLE_FORMAT" "Starting" "{self.mfc.args["name"]} from {self.mfc.args["input"]}:" +printf "$TABLE_TITLE_FORMAT" "Starting" "{ARG("name")} from {ARG("input")}:" printf "$TABLE_CONTENT\\n" printf "$TABLE_FOOTER\\n" @@ -229,7 +204,7 @@ def __generate_epilogue(self) -> str: t_stop="$(date +%s)" printf "\\n$TABLE_HEADER" -printf "$TABLE_TITLE_FORMAT" "Finished" "{self.mfc.args["name"]}:" +printf "$TABLE_TITLE_FORMAT" "Finished" "{ARG("name")}:" printf "$TABLE_FORMAT_LINE" "Total-time:" "$(expr $t_stop - $t_start)s" "Exit Code:" "$code" printf "$TABLE_FORMAT_LINE" "End-time:" "$(date +%T)" "End-date:" "$(date +%T)" printf "$TABLE_FOOTER" @@ -237,37 +212,14 @@ def __generate_epilogue(self) -> str: exit $code """ - def __evaluate_variable(self, var: str) -> str: - v: str = var.strip() - if v in self.mfc.args: - return str(self.mfc.args[v]) - - return None - def __evaluate_expression(self, expr: str) -> str: - expr_original = expr[:] - - evaluated = self.__evaluate_variable(expr) - if evaluated is not None: - if not common.isspace(evaluated): - return evaluated - else: - # The expression is valid but it is empty - return None - - # It may be an expression. Try and parse it - for var_candidate in re.split(r"[\*,\ ,\+,\-,\/,\\,\%,\,,\.,\^,\',\",\[,\],\(,\),\=]", expr): - evaluated = self.__evaluate_variable(var_candidate) - - if evaluated is not None and not common.isspace(evaluated): - expr = expr.replace(var_candidate, evaluated) - # See if it computable try: # We assume eval is safe because we control the expression. - return str(eval(expr)) + r = str(eval(expr, ARGS())) + return r if not common.isspace(r) else None except Exception as exc: - raise common.MFCException(f"BatchEngine: {expr_original} (interpreted as {expr}) is not a valid expression in the template file. Please check your spelling.") + raise common.MFCException(f"BatchEngine: '{expr}' is not a valid expression in the template file. Please check your spelling.") def __batch_evaluate(self, s: str, system: queues.QueueSystem, names: typing.List[str]): replace_list = [ @@ -293,7 +245,7 @@ def __batch_evaluate(self, s: str, system: queues.QueueSystem, names: typing.Lis # If not specified, then remove the line it appears on s = re.sub(f"^.*\{match}.*$\n", "", s, flags=re.MULTILINE) - cons.print(f"> > [bold yellow]Warning:[/bold yellow] [magenta]{match[1:-1]}[/magenta] was not specified. Thus, any line it figures on will be discarded.") + cons.print(f"> > [bold yellow]Warning:[/bold yellow] [magenta]{match[1:-1]}[/magenta] was not specified. Thus, any line it appears on will be discarded.") return s @@ -315,9 +267,6 @@ def __execute_batch_file(self, system: queues.QueueSystem, names: typing.List[st if common.system(cmd, cwd=self.__get_batch_dirpath()) != 0: raise common.MFCException(f"Submitting batch file for {system.name} failed. It can be found here: {self.__get_batch_filepath(target_name)}. Please check the file for errors.") - def validate_job_options(self, mfc) -> None: - pass - ENGINES = [ InteractiveEngine(), BatchEngine() ] diff --git a/toolchain/mfc/run/input.py b/toolchain/mfc/run/input.py index 45500bd7f0..31abdeace6 100644 --- a/toolchain/mfc/run/input.py +++ b/toolchain/mfc/run/input.py @@ -1,22 +1,18 @@ +import os, json, dataclasses -import os -import json -import dataclasses - -from ..util.printer import cons -from ..util import common -from . import case_dicts +from ..printer import cons +from .. import common +from ..state import ARG +from . import case_dicts @dataclasses.dataclass class MFCInputFile: filename: str case_dirpath: str case_dict: dict - args: dict - def __generate_inp(self, target_name: str) -> None: - MASTER_KEYS: list = case_dicts.get_input_dict_keys(target_name, self.args) + MASTER_KEYS: list = case_dicts.get_input_dict_keys(target_name) # Create Fortran-style input file content string dict_str = "" @@ -38,7 +34,7 @@ def __generate_inp(self, target_name: str) -> None: def __generate_fpp(self, target_name: str) -> None: # === case.fpp === - use_case_optimization = self.args["case_optimization"] and target_name == "simulation" + use_case_optimization = ARG("case_optimization") and target_name == "simulation" filepath = os.path.join(os.getcwd(), "src", target_name, "case.fpp") content = f"""\ @@ -89,8 +85,8 @@ def generate(self, target_name: str) -> None: # Load the input file -def load(args: dict) -> MFCInputFile: - filename: str = args["input"].strip() +def load() -> MFCInputFile: + filename: str = ARG("input").strip() cons.print(f"Acquiring [bold magenta]{filename}[/bold magenta]...") @@ -115,5 +111,4 @@ def load(args: dict) -> MFCInputFile: except Exception as exc: raise common.MFCException(f"Input file {filename} did not produce valid JSON. It should only print the case dictionary.\n\n{exc}\n") - return MFCInputFile(filename, dirpath, dictionary, args) - + return MFCInputFile(filename, dirpath, dictionary) diff --git a/toolchain/mfc/run/mpi_bins.py b/toolchain/mfc/run/mpi_bins.py index d404245b36..e14e5ffa91 100644 --- a/toolchain/mfc/run/mpi_bins.py +++ b/toolchain/mfc/run/mpi_bins.py @@ -1,7 +1,7 @@ -import typing -import dataclasses +import typing, dataclasses -from ..util import common +from .. import common +from ..state import ARG # Note: This file is now only used when running # in serial mode. @@ -14,7 +14,7 @@ class MPIBinary: def is_present(self) -> bool: return common.does_command_exist(self.bin) - def gen_params(self, args: list) -> typing.List[str]: + def gen_params(self) -> typing.List[str]: raise common.MFCException(f"MPIBinary::gen_params <{self.name}> not implemented.") @@ -22,17 +22,17 @@ class JSRUN(MPIBinary): def __init__(self): super().__init__("IBM's JSRUN", "jsrun") - def gen_params(self, args: list) -> typing.List[str]: + def gen_params(self) -> typing.List[str]: # ORNL Summit: https://docs.olcf.ornl.gov/systems/summit_user_guide.html?highlight=lsf#launching-a-job-with-jsrun # We create one resource-set per CPU(Core)/GPU pair. - nrs=args["cpus_per_node"]*args["nodes"] - cpus_per_rs=1 - gpus_per_rs=min(args["gpus_per_node"], 1) + nrs=ARG("tasks_per_node")*ARG("nodes") + cores_per_rs=1 + gpus_per_rs=min(ARG("tasks_per_node"), 1) tasks_per_rs=1 arguments=[ '--nrs', nrs, - '--cpu_per_rs', cpus_per_rs, + '--cpu_per_rs', cores_per_rs, '--gpu_per_rs', gpus_per_rs, '--tasks_per_rs', tasks_per_rs ] @@ -47,24 +47,21 @@ class SRUN(MPIBinary): def __init__(self): super().__init__("SLURM's SRUN", "srun") - def gen_params(self, args: list) -> typing.List[str]: - params = ['--ntasks-per-node', args["cpus_per_node"]] + def gen_params(self) -> typing.List[str]: + params = ['--ntasks-per-node', ARG("tasks_per_node")] - if args["nodes"] != 1: - params += ['-N', args["nodes"]] + if ARG("nodes") != 1: + params += ['-N', ARG("nodes")] # MFC binds its GPUs on its own, as long as they have been allocated # by the system's scheduler, or are present on your local machine, # if running in serial mode. - # - # if args["gpus_per_node"] != 0: - # params += f' -G {args["gpus_per_node"]}' - if not common.isspace(args["account"]): - params += ['-A', args["account"]] + if not common.isspace(ARG("account")): + params += ['-A', ARG("account")] - if not common.isspace(args["partition"]): - params += ['-p', args["partition"]] + if not common.isspace(ARG("partition")): + params += ['-p', ARG("partition")] return params @@ -73,22 +70,22 @@ class MPIEXEC(MPIBinary): def __init__(self): super().__init__("MPIEXEC", "mpiexec") - def gen_params(self, args: list) -> str: - return ["-np", args["cpus_per_node"]*args["nodes"]] + def gen_params(self) -> str: + return ["-np", ARG("tasks_per_node")*ARG("nodes")] class MPIRUN(MPIBinary): def __init__(self): super().__init__("MPIRUN", "mpirun") - def gen_params(self, args: list) -> str: - return ["-np", args["cpus_per_node"]*args["nodes"]] + def gen_params(self) -> str: + return ["-np", ARG("tasks_per_node")*ARG("nodes")] # In descending order of priority (if no override present) BINARIES: list = [ JSRUN(), SRUN(), MPIRUN(), MPIEXEC() ] -def get_binary(args: list, exclude: typing.List[str] = None) -> MPIBinary: +def get_binary(exclude: typing.List[str] = None) -> MPIBinary: if exclude is None: exclude = [] @@ -100,14 +97,13 @@ def get_binary(args: list, exclude: typing.List[str] = None) -> MPIBinary: raise common.MFCException("No MPI binary found.") # Handle user override - if args["binary"] is not None: + if ARG("binary") is not None: for binary in binaries: binary: MPIBinary - if binary.bin == args["binary"]: + if binary.bin == ARG("binary"): return binary - raise common.MFCException(f"MPI Binary <{args['binary']}> not found.") + raise common.MFCException(f"MPI Binary <{ARG('binary')}> not found.") return binaries[0] - diff --git a/toolchain/mfc/run/queues.py b/toolchain/mfc/run/queues.py index 8a1477e48a..f9c23b797c 100644 --- a/toolchain/mfc/run/queues.py +++ b/toolchain/mfc/run/queues.py @@ -1,8 +1,6 @@ -import os -import typing -import dataclasses +import os, typing, dataclasses -from ..util import common +from .. import common @dataclasses.dataclass @@ -62,4 +60,3 @@ def get_system() -> QueueSystem: return system raise common.MFCException(f"Failed to detect a queue system.") - diff --git a/toolchain/mfc/run/run.py b/toolchain/mfc/run/run.py index ff13426d5d..7139d35226 100644 --- a/toolchain/mfc/run/run.py +++ b/toolchain/mfc/run/run.py @@ -1,68 +1,62 @@ -import re +import re, typing -from ..util.printer import cons +from ..printer import cons +from ..state import ARG -from . import engines -from . import input +from . import engines, input +from .. import common, build -from ..util import common -from .. import build +def validate_job_options() -> None: + if ARG("nodes") <= 0: + raise common.MFCException("RUN: At least one node must be requested.") + if ARG("tasks_per_node") <= 0: + raise common.MFCException("RUN: At least one task per node must be requested.") -class MFCRun: - def __init__(self, mfc): - self.mfc = mfc + if not common.isspace(ARG("email")): + # https://stackoverflow.com/questions/8022530/how-to-check-for-valid-email-address + if not re.match(r"\"?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)\"?", ARG("email")): + raise common.MFCException(f'RUN: {ARG("email")} is not a valid e-mail address.') - def validate_job_options(self) -> None: - if self.mfc.args["cpus_per_node"] != self.mfc.args["gpus_per_node"] \ - and self.mfc.args["gpus_per_node"] != 0: - raise common.MFCException("RUN: Conflicting job execution parameters. If using GPUs, CPUs per node and GPUs per node must match.") - if self.mfc.args["nodes"] <= 0: - raise common.MFCException("RUN: At least one node must be requested.") +def run_targets(targets: typing.List[str]): + build.build_targets(targets) + + cons.print("[bold]Run[/bold]") + cons.indent() - if self.mfc.args["cpus_per_node"] <= 0: - raise common.MFCException("RUN: At least one CPU per node must be requested.") + if len(ARG("targets")) == 0: + cons.print(f"> No target selected.") + return - if not common.isspace(self.mfc.args["email"]): - # https://stackoverflow.com/questions/8022530/how-to-check-for-valid-email-address - if not re.match(r"\"?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)\"?", self.mfc.args["email"]): - raise common.MFCException(f'RUN: {self.mfc.args["email"]} is not a valid e-mail address.') + input_file = input.load() - engines.get_engine(self.mfc.args["engine"]).validate_job_options(self.mfc) + engine = engines.get_engine(ARG("engine")) + engine.init(input_file) - def run(self) -> None: - for name in self.mfc.args["targets"]: - build.build_target(self.mfc, name) - - cons.print("[bold]Run[/bold]") - cons.indent() - - if len(self.mfc.args["targets"]) == 0: - cons.print(f"> No target selected.") - return - - input_file = input.load(self.mfc.args) - - engine = engines.get_engine(self.mfc.args["engine"]) - engine.init(self.mfc, input_file) - - cons.print(f"Configuration:") - cons.indent() - cons.print(f"""\ -Input {self.mfc.args['input']} -Job Name (-#) {self.mfc.args['name']} -Engine (-e) {self.mfc.args['engine']} + cons.print(f"Configuration:") + cons.indent() + cons.print(f"""\ +Input {ARG('input')} +Job Name (-#) {ARG('name')} +Engine (-e) {ARG('engine')} {engine.get_args()}\ """) - cons.unindent() + cons.unindent() + + validate_job_options() + + cons.print("Generating input files...") + for name in ARG("targets"): + input_file.generate(name) - self.validate_job_options() + engine.run(ARG("targets")) + - cons.print("Generating input files...") - for name in self.mfc.args["targets"]: - input_file.generate(name) +def run_target(target: str): + run_targets([target]) - engine.run(self.mfc.args["targets"]) +def run() -> None: + run_targets(ARG("targets")) diff --git a/toolchain/mfc/sched.py b/toolchain/mfc/sched.py new file mode 100644 index 0000000000..3966a43c3c --- /dev/null +++ b/toolchain/mfc/sched.py @@ -0,0 +1,91 @@ +import time, typing, threading, dataclasses + +from .printer import cons + +import rich, rich.progress + + +class WorkerThread(threading.Thread): + def __init__(self, *args, **kwargs): + self.exc = None + + threading.Thread.__init__(self, *args, **kwargs) + + def run(self): + try: + if self._target: + self._target(*self._args, **self._kwargs) + except Exception as exc: + self.exc = exc + + +@dataclasses.dataclass +class WorkerThreadHolder: + thread: threading.Thread + ppn: int + + +@dataclasses.dataclass +class Task: + ppn: int + func: typing.Callable + args: typing.List[typing.Any] + + +def sched(tasks: typing.List[Task], nThreads: int): + nAvailable: int = nThreads + threads: typing.List[WorkerThreadHolder] = [] + + + def join_first_dead_thread(progress, complete_tracker) -> None: + nonlocal threads, nAvailable + + for threadID, threadHolder in enumerate(threads): + if not threadHolder.thread.is_alive(): + if threadHolder.thread.exc != None: + raise threadHolder.thread.exc + + nAvailable += threadHolder.ppn + progress.advance(complete_tracker) + + del threads[threadID] + + break + + + with rich.progress.Progress(console=cons.raw, transient=True) as progress: + queue_tracker = progress.add_task("Queued ", total=len(tasks)) + complete_tracker = progress.add_task("Completed", total=len(tasks)) + + # Queue Tests + for task in tasks: + # Wait until there are threads available + while nAvailable < task.ppn: + # This is important if "-j 1" is used (the default) since there + # are test cases that require test.ppn=2 + if task.ppn > nThreads and nAvailable > 0: + break + + # Keep track of threads that are done + join_first_dead_thread(progress, complete_tracker) + + # Do not overwhelm this core with this loop + time.sleep(0.05) + + # Launch Thread + progress.advance(queue_tracker) + + thread = WorkerThread(target=task.func, args=tuple(task.args)) + thread.start() + + threads.append(WorkerThreadHolder(thread, task.ppn)) + nAvailable -= task.ppn + + + # Wait for the lasts tests to complete + while len(threads) != 0: + # Keep track of threads that are done + join_first_dead_thread(progress, complete_tracker) + + # Do not overwhelm this core with this loop + time.sleep(0.05) diff --git a/toolchain/mfc/state.py b/toolchain/mfc/state.py new file mode 100644 index 0000000000..6c2a394cff --- /dev/null +++ b/toolchain/mfc/state.py @@ -0,0 +1,46 @@ +import typing, dataclasses + + +@dataclasses.dataclass +class MFCConfig: + mpi: bool = True + gpu: bool = False + debug: bool = False + + def from_dict(d: dict): + r = MFCConfig() + + for key in d: + setattr(r, key, d[key]) + + return r + + def __eq__(self, other) -> bool: + for field in dataclasses.fields(self): + if getattr(self, field.name) != getattr(other, field.name): + return False + + return True + + def __str__(self) -> str: + m = { False: "No", True: "Yes" } + r = ' & '.join([ f"{field.name}={m[getattr(self, field.name)]}" for field in dataclasses.fields(self) ]) + + return r + + +gCFG: MFCConfig = MFCConfig() +gARG: dict = {} + + +def ARG(arg: str) -> typing.Any: + global gARG + return gARG[arg] + +def ARGS() -> dict: + global gARG + return gARG + +def CFG() -> MFCConfig: + global gCFG + return gCFG diff --git a/toolchain/mfc/cfg/__init__.py b/toolchain/mfc/test/__init__.py similarity index 100% rename from toolchain/mfc/cfg/__init__.py rename to toolchain/mfc/test/__init__.py diff --git a/toolchain/mfc/tests/case.py b/toolchain/mfc/test/case.py similarity index 75% rename from toolchain/mfc/tests/case.py rename to toolchain/mfc/test/case.py index edab45fb18..df4839900b 100644 --- a/toolchain/mfc/tests/case.py +++ b/toolchain/mfc/test/case.py @@ -1,11 +1,7 @@ -import os -import json -import hashlib -import binascii -import subprocess -import dataclasses +import os, hashlib, binascii, subprocess, dataclasses -from ..util import common +from .. import case, common +from ..state import ARG Tend = 0.25 Nt = 50 @@ -97,47 +93,33 @@ } @dataclasses.dataclass(init=False) -class Case: - trace: str - params: dict - ppn: int +class TestCase(case.Case): + ppn: int + trace: str def __init__(self, trace: str, mods: dict, ppn: int = None) -> None: - self.trace = trace - self.params = {**BASE_CFG.copy(), **mods} - self.ppn = ppn if ppn is not None else 1 + self.trace = trace + self.ppn = ppn if ppn is not None else 1 + super().__init__({**BASE_CFG.copy(), **mods}) - def run(self, args: dict) -> subprocess.CompletedProcess: + def run(self) -> subprocess.CompletedProcess: filepath = f'"{self.get_dirpath()}/case.py"' - mode = f'-m "{args["mode"]}"' - tasks = f"-n {self.ppn} " - jobs = f"-j {args['jobs']}" if args["case_optimization"] else "" - gpus = f"-g {self.ppn}" if "gpu" in args["mode"] else "" - binary_option = f"-b {args['binary']}" if args["binary"] is not None else "" - case_optimization = "--case-optimization" if args["case_optimization"] else "--no-build" - no_mpi = f"--no-mpi" if not args["mpi"] else "" + tasks = f"-n {self.ppn}" + jobs = f"-j {ARG('jobs')}" if ARG("case_optimization") else "" + binary_option = f"-b {ARG('binary')}" if ARG("binary") is not None else "" + case_optimization = "--case-optimization" if ARG("case_optimization") else "--no-build" mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh" - + command: str = f'''\ -{mfc_script} run {filepath} {mode} {tasks} {binary_option} {case_optimization} \ -{jobs} {gpus} {no_mpi} -t pre_process simulation 2>&1\ +{mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \ +{jobs} -t pre_process simulation 2>&1\ ''' return subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) - - def get_keys(self) -> str: - return self.params.keys() - - def has_parameter(self, key: str)-> bool: - return key in self.get_keys() - - def gen_json_dict_str(self) -> str: - return json.dumps(self.params, indent=4) - def get_uuid(self) -> str: return hex(binascii.crc32(hashlib.sha1(str(self.trace).encode()).digest())).upper()[2:].zfill(8) @@ -149,7 +131,9 @@ def create_directory(self): content = f"""\ #!/usr/bin/env python3 + import json + print(json.dumps({self.gen_json_dict_str()})) """ @@ -157,20 +141,10 @@ def create_directory(self): common.file_write(f"{dirpath}/case.py", content) - def __getitem__(self, key: str) -> str: - if key not in self.params: - raise common.MFCException(f"Case {self.trace}: Parameter {key} does not exist.") - - return self.params[key] - - def __setitem__(self, key: str, val: str): - self.params[key] = val - def __str__(self) -> str: return f"tests/[bold magenta]{self.get_uuid()}[/bold magenta]: {self.trace}" - @dataclasses.dataclass class CaseGeneratorStack: trace: list # list of strs @@ -190,7 +164,7 @@ def pop(self) -> None: return (self.mods.pop(), self.trace.pop()) -def create_case(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: int = None) -> Case: +def create_case(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: int = None) -> TestCase: mods: dict = {} for dict in stack.mods: @@ -205,4 +179,4 @@ def create_case(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: in if not common.isspace(trace): traces.append(trace) - return Case(' -> '.join(traces), mods, ppn) + return TestCase(' -> '.join(traces), mods, ppn) diff --git a/toolchain/mfc/tests/cases.py b/toolchain/mfc/test/cases.py similarity index 98% rename from toolchain/mfc/tests/cases.py rename to toolchain/mfc/test/cases.py index c056849141..f404a4662f 100644 --- a/toolchain/mfc/tests/cases.py +++ b/toolchain/mfc/test/cases.py @@ -1,11 +1,7 @@ import typing -import itertools -from ..util import common - -from .case import Case -from .case import create_case -from .case import CaseGeneratorStack +from .. import common +from .case import TestCase, create_case, CaseGeneratorStack def get_bc_mods(bc: int, dimInfo): @@ -71,7 +67,7 @@ def get_dimensions(): return r -def generate_cases() -> typing.List[Case]: +def generate_cases() -> typing.List[TestCase]: stack, cases = CaseGeneratorStack(), [] def alter_bcs(dimInfo, dimParams): diff --git a/toolchain/mfc/tests/pack.py b/toolchain/mfc/test/pack.py similarity index 93% rename from toolchain/mfc/tests/pack.py rename to toolchain/mfc/test/pack.py index c707fa8d47..d666805d77 100644 --- a/toolchain/mfc/tests/pack.py +++ b/toolchain/mfc/test/pack.py @@ -1,17 +1,11 @@ -import os -import re -import math -import dataclasses +import os, re, math, dataclasses from pathlib import Path +from . import case +from .. import common -from ..util import common - -from . import case -from . import tests - -from ..util.common import MFCException +from ..common import MFCException @dataclasses.dataclass(repr=False) @@ -100,7 +94,7 @@ def load(filepath: str) -> Pack: return Pack(entries) -def generate(case: case.Case) -> Pack: +def generate(case: case.TestCase) -> Pack: entries = [] case_dir = case.get_dirpath() @@ -144,9 +138,7 @@ def is_close(error: Error, tolerance: Tolerance) -> bool: return False -def check_tolerance(case: case.Case, candidate: Pack, golden: Pack, tol: float) -> Error: - uuid = case.get_uuid() - +def check_tolerance(case: case.TestCase, candidate: Pack, golden: Pack, tol: float) -> Error: # Keep track of the average error avg_err = AverageError() diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py new file mode 100644 index 0000000000..41f5accacc --- /dev/null +++ b/toolchain/mfc/test/test.py @@ -0,0 +1,170 @@ +import os + +from ..printer import cons +from .. import common +from ..state import ARG +from .case import TestCase +from .cases import generate_cases +from .. import sched +from ..common import MFCException +from ..build import build_targets +from . import pack as packer + +import rich, rich.table + + +CASES = generate_cases() +nFAIL = 0 + +def __filter(): + global CASES + + # Check "--from" and "--to" exist and are in the right order + bFoundFrom, bFoundTo = (False, False) + from_i = -1 + for i, case in enumerate(CASES): + if case.get_uuid() == ARG("from"): + from_i = i + bFoundFrom = True + # Do not "continue" because "--to" might be the same as "--from" + if bFoundFrom and case.get_uuid() == ARG("to"): + CASES = CASES[from_i:i+1] + bFoundTo = True + break + + if not bFoundTo: + raise MFCException("Testing: Your specified range [--from,--to] is incorrect. Please ensure both IDs exist and are in the correct order.") + + if len(ARG("only")) > 0: + for i, case in enumerate(CASES[:]): + case: TestCase + + doKeep = False + for o in ARG("only"): + if str(o) == case.get_uuid(): + doKeep = True + break + + if not doKeep: + CASES.remove(case) + + if not ARG("mpi"): + for case in CASES[:]: + if case.ppn > 1: + CASES.remove(case) + + +def test(): + global CASES, nFAIL + + # Delete UUIDs that are not in the list of CASES from tests/ + if ARG("generate"): + dir_uuids = set([name for name in os.listdir(".") if os.path.isdir(name)]) + new_uuids = set([case.get_uuid() for case in CASES]) + + for old_uuid in dir_uuids - new_uuids: + common.delete_directory(f"{common.MFC_TESTDIR}/{old_uuid}") + + __filter() + + if ARG("list"): + table = rich.table.Table(title="MFC Test Cases", box=rich.table.box.SIMPLE) + + table.add_column("UUID", style="bold magenta", justify="center") + table.add_column("Trace") + + for case in CASES: + table.add_row(case.get_uuid(), case.trace) + + rich.print(table) + + return + + build_targets(["pre_process", "simulation"]) + + range_str = f"from [bold magenta]{ARG('from')}[/bold magenta] to [bold magenta]{ARG('to')}[/bold magenta]" + + if len(ARG("only")) > 0: + range_str = "Only " + common.format_list_to_string([ + f"[bold magenta]{uuid}[/bold magenta]" for uuid in ARG("only") + ], "Nothing to run") + + + cons.print(f"[bold]Test[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})") + cons.indent() + + + # Run CASES with multiple threads (if available) + cons.print() + cons.print(f" tests/[bold magenta]UUID[/bold magenta] Summary") + cons.print() + + # Select the correct number of threads to use to launch test CASES + # We can't use ARG("jobs") when the --case-optimization option is set + # because running a test case may cause it to rebuild, and thus + # interfere with the other test CASES. It is a niche feature so we won't + # engineer around this issue (for now). + nThreads = ARG("jobs") if not ARG("case_optimization") else 1 + tasks = [ + sched.Task(ppn=case.ppn, func=handle_case, args=[ case ]) for case in CASES + ] + sched.sched(tasks, nThreads) + + cons.print() + if nFAIL == 0: + cons.print(f"Tested [bold green]✓[/bold green]") + cons.unindent() + else: + if nFAIL == 1: + raise MFCException(f"Testing: There was [bold red]1[/bold red] failure.") + else: + raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.") + + +def handle_case(test: TestCase): + global nFAIL + + try: + test.create_directory() + + if test.params.get("qbmm", 'F') == 'T': + tol = 1e-10 + elif test.params.get("bubbles", 'F') == 'T': + tol = 1e-10 + elif test.params.get("hypoelasticity", 'F') == 'T': + tol = 1e-7 + else: + tol = 1e-12 + + cmd = test.run() + + out_filepath = os.path.join(test.get_dirpath(), "out.txt") + + common.file_write(out_filepath, cmd.stdout) + + if cmd.returncode != 0: + cons.print(cmd.stdout) + raise MFCException(f"""Test {test}: Failed to execute MFC. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""") + + pack = packer.generate(test) + pack.save(os.path.join(test.get_dirpath(), "pack.txt")) + + golden_filepath = os.path.join(test.get_dirpath(), "golden.txt") + if ARG("generate"): + common.delete_file(golden_filepath) + pack.save(golden_filepath) + else: + if not os.path.isfile(golden_filepath): + raise MFCException(f"Test {test}: Golden file doesn't exist! To generate golden files, use the '-g' flag.") + + packer.check_tolerance(test, pack, packer.load(golden_filepath), tol) + + cons.print(f" [bold magenta]{test.get_uuid()}[/bold magenta] {test.trace}") + except Exception as exc: + nFAIL = nFAIL + 1 + + if not ARG("relentless"): + raise exc + + cons.print(f"[bold red]Failed test {test}.[/bold red]") + cons.print(f"{exc}") diff --git a/toolchain/mfc/tests/__init__.py b/toolchain/mfc/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/toolchain/mfc/tests/tests.py b/toolchain/mfc/tests/tests.py deleted file mode 100644 index 41fa39b7c5..0000000000 --- a/toolchain/mfc/tests/tests.py +++ /dev/null @@ -1,175 +0,0 @@ -import os - -import rich - -from ..util.printer import cons - -from ..util import common - -from .case import Case -from .cases import generate_cases -from .threads import MFCTestThreadManager - -from ..util.common import MFCException - -from ..build import build_target - -from . import pack as packer - - -import rich -import rich.table - - -class MFCTest: - def __init__(self, mfc): - self.mfc = mfc - self.sched = MFCTestThreadManager(1) - self.cases = generate_cases() - self.nFail = 0 - - def __filter_tests(self): - # Check "--from" and "--to" exist and are in the right order - bFoundFrom, bFoundTo = (False, False) - from_i = -1 - for i, case in enumerate(self.cases): - if case.get_uuid() == self.mfc.args["from"]: - from_i = i - bFoundFrom = True - # Do not "continue" because "--to" might be the same as "--from" - if bFoundFrom and case.get_uuid() == self.mfc.args["to"]: - self.cases = self.cases[from_i:i+1] - bFoundTo = True - break - - if not bFoundTo: - raise MFCException("Testing: Your specified range [--from,--to] is incorrect. Please ensure both IDs exist and are in the correct order.") - - if len(self.mfc.args["only"]) > 0: - for i, case in enumerate(self.cases[:]): - case: Case - - doKeep = False - for o in self.mfc.args["only"]: - if str(o) == case.get_uuid(): - doKeep = True - break - - if not doKeep: - self.cases.remove(case) - - if not self.mfc.args["mpi"]: - for case in self.cases[:]: - if case.ppn > 1: - self.cases.remove(case) - - - def execute(self): - # Select the correct number of threads to use to launch test cases - # We can't use args["jobs"] when the --case-optimization option is set - # because running a test case may cause it to rebuild, and thus - # interfere with the other test cases. It is a niche feature so we won't - # engineer around this issue (for now). - self.sched.nAvailable = self.mfc.args["jobs"] if not self.mfc.args["case_optimization"] else 1 - - # Delete UUIDs that are not in the list of cases from tests/ - if self.mfc.args["generate"]: - dir_uuids = set([name for name in os.listdir(".") if os.path.isdir(name)]) - new_uuids = set([case.get_uuid() for case in self.cases]) - - for old_uuid in dir_uuids - new_uuids: - common.delete_directory(f"{common.MFC_TESTDIR}/{old_uuid}") - - self.__filter_tests() - - if self.mfc.args["list"]: - table = rich.table.Table(title="MFC Test Cases", box=rich.table.box.SIMPLE) - - table.add_column("UUID", style="bold magenta", justify="center") - table.add_column("Trace") - - for case in self.cases: - table.add_row(case.get_uuid(), case.trace) - - rich.print(table) - - return - - build_target(self.mfc, "pre_process") - build_target(self.mfc, "simulation") - - range_str = f"from [bold magenta]{self.mfc.args['from']}[/bold magenta] to [bold magenta]{self.mfc.args['to']}[/bold magenta]" - - if len(self.mfc.args["only"]) > 0: - range_str = "Only " + common.format_list_to_string([ - f"[bold magenta]{uuid}[/bold magenta]" for uuid in self.mfc.args["only"] - ], "Nothing to run") - - - cons.print(f"[bold]Test[/bold] | {range_str} ({len(self.cases)} test{'s' if len(self.cases) != 1 else ''})") - cons.indent() - - - # Run cases with multiple threads (if available) - cons.print() - cons.print(f" tests/[bold magenta]UUID[/bold magenta] Summary") - cons.print() - self.sched.run(self.cases, self.handle_case) - - cons.print() - if self.nFail == 0: - cons.print(f"Tested [bold green]✓[/bold green]") - cons.unindent() - else: - if self.nFail == 1: - raise MFCException(f"Testing: There was [bold red]1[/bold red] failure.") - else: - raise MFCException(f"Testing: There were [bold red]{self.nFail}[/bold red] failures.") - - - def handle_case(self, test: Case): - try: - test.create_directory() - - if test.params.get("qbmm", 'F') == 'T': - tol = 1e-10 - elif test.params.get("bubbles", 'F') == 'T': - tol = 1e-10 - elif test.params.get("hypoelasticity", 'F') == 'T': - tol = 1e-7 - else: - tol = 1e-12 - - cmd = test.run(self.mfc.args) - - out_filepath = os.path.join(test.get_dirpath(), "out.txt") - - common.file_write(out_filepath, cmd.stdout) - - if cmd.returncode != 0: - cons.print(cmd.stdout) - raise MFCException(f"""Test {test}: Failed to execute MFC. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""") - - pack = packer.generate(test) - pack.save(os.path.join(test.get_dirpath(), "pack.txt")) - - golden_filepath = os.path.join(test.get_dirpath(), "golden.txt") - if self.mfc.args["generate"]: - common.delete_file(golden_filepath) - pack.save(golden_filepath) - else: - if not os.path.isfile(golden_filepath): - raise MFCException(f"Test {test}: Golden file doesn't exist! To generate golden files, use the '-g' flag.") - - packer.check_tolerance(test, pack, packer.load(golden_filepath), tol) - - cons.print(f" [bold magenta]{test.get_uuid()}[/bold magenta] {test.trace}") - except Exception as exc: - self.nFail = self.nFail + 1 - - if not self.mfc.args["relentless"]: - raise exc - - cons.print(f"[bold red]Failed test {test}.[/bold red]") - cons.print(f"{exc}") - diff --git a/toolchain/mfc/tests/threads.py b/toolchain/mfc/tests/threads.py deleted file mode 100644 index ca7c4dccf5..0000000000 --- a/toolchain/mfc/tests/threads.py +++ /dev/null @@ -1,92 +0,0 @@ -import time -import threading -import dataclasses - -from ..util.printer import cons - -from .case import Case - - -import rich -import rich.progress - - -class TestThread(threading.Thread): - def __init__(self, *args, **kwargs): - self.exc = None - - threading.Thread.__init__(self, *args, **kwargs) - - def run(self): - try: - if self._target: - self._target(*self._args, **self._kwargs) - except Exception as exc: - self.exc = exc - - -@dataclasses.dataclass -class TestThreadHolder: - thread: threading.Thread - ppn: int - - -class MFCTestThreadManager: - def __init__(self, nThreads: int) -> None: - self.threads = [] - self.nThreads = nThreads - self.nAvailable = self.nThreads - - def join_first_dead_thread(self, progress, complete_tracker) -> None: - for threadID, threadHolder in enumerate(self.threads): - threadHolder: TestThreadHolder - - if not threadHolder.thread.is_alive(): - if threadHolder.thread.exc != None: - raise threadHolder.thread.exc - - self.nAvailable += threadHolder.ppn - progress.advance(complete_tracker) - - del self.threads[threadID] - - break - - def run(self, cases: list, handle_case) -> None: - with rich.progress.Progress(console=cons.raw, transient=True) as progress: - queue_tracker = progress.add_task("Queued ", total=len(cases)) - complete_tracker = progress.add_task("Completed", total=len(cases)) - - # Queue Tests - for i, test in enumerate(cases): - test: Case - - # Wait until there are threads available - while self.nAvailable < test.ppn: - # This is important if "-j 1" is used (the default) since there - # are test cases that require test.ppn=2 - if test.ppn > self.nThreads and self.nAvailable > 0: - break - - # Keep track of threads that are done - self.join_first_dead_thread(progress, complete_tracker) - - # Do not overwhelm this core with this loop - time.sleep(0.05) - - # Launch Thread - progress.advance(queue_tracker) - - thread = TestThread(target=handle_case, args=(test,)) - thread.start() - - self.threads.append(TestThreadHolder(thread, test.ppn)) - self.nAvailable -= test.ppn - - # Wait for the lasts tests to complete - while len(self.threads) != 0: - # Keep track of threads that are done - self.join_first_dead_thread(progress, complete_tracker) - - # Do not overwhelm this core with this loop - time.sleep(0.05) diff --git a/toolchain/mfc/util/__init__.py b/toolchain/mfc/util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/toolchain/requirements.txt b/toolchain/requirements.txt new file mode 100644 index 0000000000..afee88b5af --- /dev/null +++ b/toolchain/requirements.txt @@ -0,0 +1,7 @@ +rich +fypp +wheel +typing +PyYAML +argparse +dataclasses diff --git a/toolchain/scripts/format.sh b/toolchain/scripts/format.sh deleted file mode 100644 index 6381483673..0000000000 --- a/toolchain/scripts/format.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -if [ ! -f "$(pwd)/toolchain/util.sh" ]; then - echo "" - echo "[toolchain/scripts/prettify.sh] Please execute this script from the top-level MFC directory (currently in: $(pwd))." - - exit 1 -fi - -source "$(pwd)/toolchain/util.sh" - -if ! command -v fprettify; then - pip3 install --upgrade fprettify -fi - -fprettify src --exclude "src/*/autogen" --recursive \ - --indent 4 --c-relations --enable-replacements --enable-decl \ - --whitespace-comma 1 --whitespace-multdiv 0 --whitespace-plusminus 1 \ - --case 1 1 1 1 --strict-indent -ret="$?" - -if [ "$ret" != "0" ]; then - echo -en "$RED" - echo "[mfc.sh] Error: failed to execute fprettify." - echo "[mfc.sh] Error: MFC has not been fprettify'ied." - echo -en "$COLOR_RESET" - - exit 1 -fi - -echo -en "$GREEN" -echo "[mfc.sh] MFC has been fprettify'ied." -echo -en "$COLOR_RESET" diff --git a/toolchain/scripts/load.sh b/toolchain/scripts/load.sh deleted file mode 100755 index 195d770231..0000000000 --- a/toolchain/scripts/load.sh +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env bash - -# Load /toolchain/util.sh -if [ ! -f "$(pwd)/toolchain/util.sh" ]; then - echo "[toolchain/scripts/load.sh]: This script must be source from within MFC's root directory." - return -fi - -source "$(pwd)/toolchain/util.sh" - -echo -en "$YELLOW" -echo -e "Please source this script:\n" -echo -e " - $(whoami) $ . ./mfc.sh load" -echo -e " - $(whoami) $ source ./mfc.sh load\n" -echo -en "$COLOR_RESET" - -on_error() { - echo -en "\n$YELLOW" - echo -e "We would welcome your contribution:" - echo -e " - Github: https://github.com/MFlowCode/MFC" - echo -en "$COLOR_RESET" - return -} - -ORNL="$GREEN""ORNL$COLOR_RESET" -SUMMIT="$ORNL/Summit" -ASCENT="$ORNL/Ascent" -WOMBAT="$ORNL/Wombat" - -XSEDE="$CYAN""XSEDE$COLOR_RESET" -BRIDGES2="$XSEDE/Bridges2" -EXPANSE="$XSEDE/Expanse" - -GT="$YELLOW""GATECH$COLOR_RESET" -PHOENIX="$GT/Phoenix" - -CALTECH=$RED"CALTECH$COLOR_RESET" -RICHARDSON="$CALTECH/Richardson" - -C_ORNL=$GREEN"s$COLOR_RESET/$GREEN""a$COLOR_RESET/$GREEN""w$COLOR_RESET" -C_XSEDE=$CYAN"b$COLOR_RESET/$CYAN""e$COLOR_RESET" -C_GT=$YELLOW"p$COLOR_RESET" -C_OTHER=$RED"r$COLOR_RESET" - -# Reset u_computer & u_cg to known values since this script is run with "source" -# Therefore, values of variables defined here are kept when the script runs again. -u_computer="" -u_cg="" - -# If there are command-line arguments, parse them: -while [[ $# -gt 0 ]]; do - case $1 in - -c|--computer) - u_computer="$2" - shift; shift - ;; - -m|--mode) - u_cg="$2" - shift; shift - ;; - -*|--*) - echo "Unknown option $1" - on_error - return - ;; - esac -done - -# Get computer (if not supplied in command-line) -if [ -v $u_computer ]; then - echo -e "$MAGENTA[Q 1/2]$COLOR_RESET Which computer would you like to load submodules for?" - echo -e " | $SUMMIT ($GREEN""s$COLOR_RESET) | $BRIDGES2 ($CYAN""b$COLOR_RESET) | $RICHARDSON ($RED""r$COLOR_RESET)" - echo -e " | $ASCENT ($GREEN""a$COLOR_RESET) | $EXPANSE ($CYAN""e$COLOR_RESET) |" - echo -e " | $WOMBAT ($GREEN""w$COLOR_RESET) | $PHOENIX ($YELLOW""p$COLOR_RESET) |" - echo -en "($C_ORNL/$C_XSEDE/$C_GT/$C_OTHER): " - read u_computer -fi - -# Get CPU/GPU (if not supplied in command-line) -if [ -v $u_cg ]; then - echo -e "$MAGENTA[Q 2/2]$COLOR_RESET Would you like to run solely on CPUs or GPUs as well?" - echo -e " - CPU (c) - GPU (g)" - echo -n "(c/g): " - read u_cg -fi - -# User input to lowercase -u_computer=$(echo "$u_computer" | tr '[:upper:]' '[:lower:]') -u_cg=$(echo "$u_cg" | tr '[:upper:]' '[:lower:]') - -COMPUTER="" -CG="" -declare -a MODULES - -if [ "$u_cg" == "c" ]; then # for CPU - CG="CPU" -elif [ "$u_cg" == "g" ]; then # For GPU - CG="GPU" -else - echo -e $RED"Error: Invalid choice \"$u_cg\" for Q2."$COLOR_RESET - - on_error - return -fi - -if [ "$u_computer" == "s" ]; then # For Summit - COMPUTER="$SUMMIT" - - if [ "$u_cg" == "c" ]; then - MODULES=("gcc/12.1.0") - elif [ "$u_cg" == "g" ]; then - MODULES=("nvhpc/22.5" "cuda/nvhpc") - fi - - MODULES=("${MODULES[@]}" "python/3.8.10" "darshan-runtime/3.3.1-lite" - "hsi/5.0.2.p5" "xalt/1.2.1" "lsf-tools/2.0" - "cmake/3.23.1" "ninja/1.10.2" "spectrum-mpi/10.4.0.3-20210112") -elif [ "$u_computer" == "b" ]; then # Bridges2 - COMPUTER="$BRIDGES2" - - if [ "$u_cg" == "c" ]; then - MODULES=("intel/2021.3.0" "intelmpi/2021.3.0-intel2021.3.0") - - echo "" - echo -e "$YELLOW""WARNING: Please set CC=icc, CXX=icx, and FC=ifort.$COLOR_RESET" - echo "" - elif [ "$u_cg" == "g" ]; then - MODULES=("openmpi/4.0.5-nvhpc21.7") - fi - - MODULES=("${MODULES[@]}" "python/3.8.6") -elif [ "$u_computer" == "a" ]; then # For Ascent - COMPUTER="$ASCENT" - - if [ "$u_cg" == "c" ]; then - MODULES=("gcc/11.1.0" "spectrum-mpi" "cuda") - elif [ "$u_cg" == "g" ]; then - MODULES=("nvhpc/21.11" "spectrum-mpi" "cuda/nvhpc" - "nsight-compute" "nsight-systems") - fi - - MODULES=("${MODULES[@]}" "python" "cmake/3.22.2") -elif [ "$u_computer" == "r" ]; then # Richardson - COMPUTER="$RICHARDSON" - - if [ "$u_cg" == "c" ]; then - MODULES=("gcc/9.3.0" "openmpi-2.0/gcc-9.3.0") - elif [ "$u_cg" == "g" ]; then - echo -e $RED"Error: GPU not supported on Richardson."$COLOR_RESET - on_error - return - fi - - MODULES=("${MODULES[@]}" "python/3.7") -elif [ "$u_computer" == "w" ]; then # For Wombat - COMPUTER="$WOMBAT" - - if [ "$u_cg" == "c" ]; then - MODULES=("gcc/11.1.0" "openmpi/4.0.5_gcc") - elif [ "$u_cg" == "g" ]; then - MODULES=("cuda/11.5.1" "/sw/wombat/Nvidia_HPC_SDK/modulefiles/nvhpc/22.1") - fi - - MODULES=("${MODULES[@]}" "cmake/3.22.1" "python/3.9.9") -elif [ "$u_computer" == "e" ]; then # Expanse - COMPUTER="$EXPANSE" - - if [ "$u_cg" == "c" ]; then - echo "" - echo -e "$YELLOW""WARNING: Please set CC=icc, CXX=icx, and FC=ifort.$COLOR_RESET" - echo "" - - MODULES=("cpu/0.15.4" "gcc/9.2.0" "openmpi/4.1.1" "cmake/3.18.2") - elif [ "$u_cg" == "g" ]; then - MODULES=("gpu/0.15.4" "cuda/11.0.2" "nvhpc/22.2" "openmpi/4.0.5" "cmake/3.19.8") - fi - - MODULES=("${MODULES[@]}" "python/3.8.5") -elif [ "$u_computer" == "p" ]; then # Phoenix - COMPUTER="$PHOENIX" - - if [ "$u_cg" == "c" ]; then - MODULES=("intel/19.0.5" "mvapich2/2.3.2") - elif [ "$u_cg" == "g" ]; then - MODULES=("cuda/11.2" "nvhpc/22.1") - fi - - MODULES=("${MODULES[@]}" "python/3.7.4" "cmake/3.20.3") -else - echo -e $RED"Error: Requested system $u_computer is not supported (yet!)"$COLOR_RESET - - on_error - return -fi - -echo -e "$MAGENTA[ MFC ]$COLOR_RESET Loading modules for $COMPUTER ($CG mode):" - -# Reset modules to default system configuration -if [ "$u_computer" != "p" ]; then - module reset > /dev/null 2>&1 - code="$?" - - - # Purge if reset is not available - if [ "$code" -ne "0" ]; then - module purge > /dev/null 2>&1 - fi -else - module purge > /dev/null 2>&1 -fi - -# Find length of longest module_name in $MODULES for $COMPUTER -max_module_length="0" -for module_name in ${MODULES[@]}; do - module_length="${#module_name}" - - if [ "$module_length" -gt "$max_module_length" ]; then - max_module_length="$module_length" - fi -done - -# Load modules for $COMPUTER with $MODULES -for module_name in ${MODULES[@]}; do - echo -en " - Loading $CYAN$module_name$COLOR_RESET " - - # Add padding spaces - module_length="${#module_name}" - delta="$((max_module_length-module_length-2))" - if [ "$delta" -ne "-2" ]; then - printf "%0.s-" $(seq 0 $delta) - echo -n " " - fi - - # Load the module - module load "$module_name" > /dev/null 2>&1 - - # Handle Success / Failure - code=$? - if [ "$code" == "0" ]; then - echo -e "[$GREEN""OK$COLOR_RESET]" - else - echo -e "[$RED""NOT OK$COLOR_RESET]" - - # Run load again to show error message - module load "$module_name" - - on_error - return - fi -done - -echo -e "$MAGENTA[ MFC ]$COLOR_RESET You should now be able to build MFC on $COMPUTER ($CG mode)!" diff --git a/toolchain/templates/lsf.sh b/toolchain/templates/lsf.sh index 554d5f23a5..3c052bfa0e 100644 --- a/toolchain/templates/lsf.sh +++ b/toolchain/templates/lsf.sh @@ -30,7 +30,7 @@ #BSUB -nnodes {nodes} #BSUB -N #BSUB -P {account} -#BSUB -W {"walltime"[:-3]} +#BSUB -W {walltime[:-3]} #> #> Note: The above expression for the walltime converts #> the expression "hh:mm:ss" to the appropriate @@ -70,11 +70,12 @@ for binpath in {MFC::BINARIES}; do echo -e ":) Running $binpath:" echo "" - jsrun {'--smpiargs="-gpu"' if gpus_per_node > 0 else ''} \ - --nrs {cpus_per_node*nodes} \ - --cpu_per_rs 1 \ - --gpu_per_rs {min(gpus_per_node, 1)} \ - --tasks_per_rs 1 \ + jsrun \ + {'--smpiargs="-gpu"' if gpu else ''} \ + --nrs {tasks_per_node*nodes} \ + --cpu_per_rs 1 \ + --gpu_per_rs {1 if gpu else 0} \ + --tasks_per_rs 1 \ "$binpath" echo "" diff --git a/toolchain/templates/pbs.sh b/toolchain/templates/pbs.sh index f49819aabc..fdbd256abe 100644 --- a/toolchain/templates/pbs.sh +++ b/toolchain/templates/pbs.sh @@ -27,7 +27,7 @@ #> intended to be modified by users. #> #PBS -N {name} -#PBS -l nodes={nodes}:ppn={cpus_per_node} +#PBS -l nodes={nodes}:ppn={tasks_per_node} #PBS -A {account} #PBS -l walltime={walltime} #PBS -q {partition} @@ -65,21 +65,21 @@ for binpath in {MFC::BINARIES}; do echo -e ":) Running $binpath:" - srun \ - --nodes={nodes} \ - --ntasks-per-node {cpus_per_node} \ + srun \ + --nodes={nodes} \ + --ntasks-per-node {tasks_per_node} \ "$binpath" -done + #> + #> srun --mpi=pmix \ + #> "$binpath" + #> + #> mpirun \ + #> -np {tasks_per_node*nodes} \ + #> "$binpath" + #> -#> -#> srun --mpi=pmix \ -#> "{MFC::BIN}" -#> -#> mpirun \ -#> -np {cpus_per_node*nodes} \ -#> "{MFC::BIN}" -#> +done {MFC::EPILOGUE} #> diff --git a/toolchain/templates/slurm.sh b/toolchain/templates/slurm.sh index ed0ede89b5..f3cdb4eed9 100644 --- a/toolchain/templates/slurm.sh +++ b/toolchain/templates/slurm.sh @@ -28,8 +28,10 @@ #> #SBATCH --job-name="{name}" #SBATCH --nodes={nodes} -#SBATCH --ntasks-per-node={cpus_per_node} +#SBATCH --ntasks-per-node={tasks_per_node} #SBATCH --cpus-per-task=1 +#SBATCH --gpu-bind=verbose,closest +#SBATCH --gpus=v100-16:{(1 if gpu else 0)*tasks_per_node*nodes} #SBATCH --time={walltime} #SBATCH --partition="{partition}" #SBATCH --output="{name}.out" @@ -45,8 +47,7 @@ #> #> #SBATCH --mem=... #> #SBATCH --constraint="lustre" -#> #SBATCH --gpus=v100-16:{gpus_per_node*nodes} -#> +#> #SBATCH --gpus-per-task={1 if gpu else 0} #> @@ -63,6 +64,16 @@ {MFC::PROLOGUE} +#> +#> Note: This MPI executable might not be well supported +#> on your system - if at all. {MFC::BIN} refers to +#> the path the MFC executable. +#> + +for binpath in {MFC::BINARIES}; do + + echo -e ":) Running $binpath:" + #> #> Note: This MPI executable might not be well supported #> on your system - if at all. {MFC::BIN} refers to @@ -78,13 +89,7 @@ #> "{MFC::BIN}" #> -for binpath in {MFC::BINARIES}; do - - echo -e ":) Running $binpath:" - - mpirun \ - -np {cpus_per_node*nodes} \ - "$binpath" + mpirun -np {nodes*tasks_per_node} "$binpath" done diff --git a/toolchain/util.sh b/toolchain/util.sh index 0837eaf6f3..5e32eee2f9 100644 --- a/toolchain/util.sh +++ b/toolchain/util.sh @@ -3,9 +3,13 @@ if [ -t 1 ]; then RED="\x1B[31m"; CYAN="\x1B[36m"; GREEN="\x1B[32m" YELLOW="\x1B[33m"; MAGENTA="\x1B[35m"; COLOR_RESET="\033[m" + + R=$RED; C=$CYAN; G=$GREEN + Y=$YELLOW; M=$MAGENTA; CR=$COLOR_RESET; W=$CR fi -log() { echo -e "$CYAN"mfc"$COLOR_RESET: $1$COLOR_RESET"; } -ok() { log "$GREEN""OK$COLOR_RESET > $1"; } -warning() { log "$YELLOW""WARNING$COLOR_RESET > $1"; } -error() { log "$RED""ERROR$COLOR_RESET > $1"; } +log() { echo -e "$CYAN"mfc"$COLOR_RESET: $1$COLOR_RESET"; } +log_n() { echo -en "$CYAN"mfc"$COLOR_RESET: $1$COLOR_RESET"; } +ok() { log "$GREEN""OK$COLOR_RESET > $1"; } +warn() { log "$YELLOW""WARNING$COLOR_RESET > $1"; } +error() { log "$RED""ERROR$COLOR_RESET > $1"; }