From 9da3ba0a267860c32d5d1e3ea00dec9f98c8f783 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 3 May 2021 10:10:14 -0400 Subject: [PATCH 1/4] Add script to build a single library --- build_stack.sh | 20 +------ build_stack_lib.sh | 131 +++++++++++++++++++++++++++++++++++++++++++++ setup_modules.sh | 18 +++---- stack_helpers.sh | 14 +++++ 4 files changed, 154 insertions(+), 29 deletions(-) create mode 100755 build_stack_lib.sh diff --git a/build_stack.sh b/build_stack.sh index e68cd66f..545be5e2 100755 --- a/build_stack.sh +++ b/build_stack.sh @@ -12,7 +12,6 @@ set -eu export HPC_STACK_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # ============================================================================== - usage() { set +x echo @@ -29,8 +28,6 @@ usage() { # ============================================================================== -[[ $# -eq 0 ]] && usage - # Defaults: export PREFIX="$HOME/opt" config="${HPC_STACK_ROOT}/config/config_custom.sh" @@ -58,12 +55,10 @@ while getopts ":p:c:y:mh" opt; do done # ============================================================================== - # Source helper functions source "${HPC_STACK_ROOT}/stack_helpers.sh" #=============================================================================== - # Source the config file if [[ -e $config ]]; then source $config @@ -81,21 +76,10 @@ else fi # ============================================================================== - -compilerName=$(echo $HPC_COMPILER | cut -d/ -f1) -compilerVersion=$(echo $HPC_COMPILER | cut -d/ -f2) - -mpiName=$(echo $HPC_MPI | cut -d/ -f1) -mpiVersion=$(echo $HPC_MPI | cut -d/ -f2) - -echo "Compiler: $compilerName/$compilerVersion" -echo "MPI: $mpiName/$mpiVersion" - # install with root permissions? [[ $USE_SUDO =~ [yYtT] ]] && export SUDO="sudo" || export SUDO="" # ============================================================================== - # create build directory if needed pkgdir=${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} mkdir -p $pkgdir @@ -105,7 +89,6 @@ logdir=$HPC_STACK_ROOT/${LOGDIR:-"log"} mkdir -p $logdir # ============================================================================== - # start with a clean slate if $MODULES; then source $MODULESHOME/init/bash @@ -118,7 +101,8 @@ else fi # ============================================================================== -# Echo build information +# Echo compiler, mpi and build information +compilermpi_info build_info # ============================================================================== diff --git a/build_stack_lib.sh b/build_stack_lib.sh new file mode 100755 index 00000000..c1a2f694 --- /dev/null +++ b/build_stack_lib.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# The purpose of this script is to build a single library in the +# software stack using the compiler/MPI combination +# +# sample usage: +# build_stack_lib.sh -p "prefix" -c "config.sh" -y "stack.yaml" -m -l "library_name" +# build_stack_lib.sh -h + +set -eu + +# root directory for the repository +export HPC_STACK_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# ============================================================================== + +usage() { + set +x + echo + echo "Usage: $0 -l | -p | -c | -y -m -h" + echo + echo " -l library to install DEFAULT: NONE, REQUIRED" + echo " -p installation prefix DEFAULT: $HOME/opt" + echo " -c use configuration file DEFAULT: config/config_custom.sh" + echo " -y use yaml file DEFAULT: config/stack_custom.yaml" + echo " -m use modules DEFAULT: NO" + echo " -h display this message and quit" + echo + exit 1 +} + +# ============================================================================== +# Defaults: +unset LIBRARY +export PREFIX="$HOME/opt" +config="${HPC_STACK_ROOT}/config/config_custom.sh" +yaml="${HPC_STACK_ROOT}/config/stack_custom.yaml" +export MODULES=false + +while getopts ":l:p:c:y:mh" opt; do + case $opt in + l) + LIBRARY=$OPTARG + ;; + p) + export PREFIX=$OPTARG + ;; + c) + config=$OPTARG + ;; + y) + yaml=$OPTARG + ;; + m) + export MODULES=true + ;; + h|\?|:) + usage + ;; + esac +done + +# Make sure required arguments are provided +shift "$(( OPTIND - 1 ))" +set +u +if [ -z "$LIBRARY" ]; then + echo 'Missing -l &2 + usage +fi +set -u + +# ============================================================================== +# Source helper functions +source "${HPC_STACK_ROOT}/stack_helpers.sh" + +#=============================================================================== +# Source the config file +if [[ -e $config ]]; then + source $config +else + echo "ERROR: CONFIG FILE $config DOES NOT EXIST, ABORT!" + exit 1 +fi + +# Source the yaml to determine software and version +if [[ -e $yaml ]]; then + eval $(parse_yaml $yaml "STACK_") +else + echo "ERROR: YAML FILE $yaml DOES NOT EXIST, ABORT!" + exit 1 +fi + +# ============================================================================== +# install with root permissions? +[[ $USE_SUDO =~ [yYtT] ]] && export SUDO="sudo" || export SUDO="" + +# ============================================================================== +# create build directory if needed +pkgdir=${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} +mkdir -p $pkgdir + +# This is for the log files +logdir=$HPC_STACK_ROOT/${LOGDIR:-"log"} +mkdir -p $logdir + +# ============================================================================== +# start with a clean slate +if $MODULES; then + source $MODULESHOME/init/bash + module use $PREFIX/modulefiles/stack + module load hpc +else + no_modules + set_no_modules_path + set_pkg_root +fi + +# ============================================================================== +# Echo compiler, mpi and build information +compilermpi_info +echo "build: $LIBRARY | version: ${STACK_\${LIBRARY}_version}" +# ============================================================================== +# Build desired library +build_lib $LIBRARY + +# ============================================================================== +# optionally clean up +[[ $MAKE_CLEAN =~ [yYtT] ]] && \ + ( $SUDO rm -rf $pkgdir; $SUDO rm -rf $logdir ) + +# ============================================================================== +echo "build_stack_lib.sh: SUCCESS!" diff --git a/setup_modules.sh b/setup_modules.sh index 00bb3558..7b7d3ffb 100755 --- a/setup_modules.sh +++ b/setup_modules.sh @@ -37,8 +37,6 @@ usage() { #=============================================================================== -[[ $# -eq 0 ]] && usage - # Defaults: PREFIX="$HOME/opt" config="${HPC_STACK_ROOT}/config/config_custom.sh" @@ -57,6 +55,10 @@ while getopts ":p:c:h" opt; do esac done +# ============================================================================== +# Source helper functions +source "${HPC_STACK_ROOT}/stack_helpers.sh" + #=============================================================================== # Source the config file @@ -68,16 +70,10 @@ else fi #=============================================================================== +# Echo compiler, mpi and build information +compilermpi_info -compilerName=$(echo $HPC_COMPILER | cut -d/ -f1) -compilerVersion=$(echo $HPC_COMPILER | cut -d/ -f2) - -mpiName=$(echo $HPC_MPI | cut -d/ -f1) -mpiVersion=$(echo $HPC_MPI | cut -d/ -f2) - -echo "Compiler: $compilerName/$compilerVersion" -echo "MPI: $mpiName/$mpiVersion" - +#=============================================================================== # install with root permissions? [[ $USE_SUDO =~ [yYtT] ]] && SUDO="sudo" || SUDO="" diff --git a/stack_helpers.sh b/stack_helpers.sh index ee2198a0..698b25f2 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -247,6 +247,19 @@ function build_info() { echo "==========================" } +function compilermpi_info() { + local compiler=$1 + local mpi=$2 + local compilerName=$(echo $compiler | cut -d/ -f1) + local compilerVersion=$(echo $compiler | cut -d/ -f2) + + local mpiName=$(echo $mpi | cut -d/ -f1) + local mpiVersion=$(echo $mpi | cut -d/ -f2) + + echo "Compiler: $compilerName/$compilerVersion" + echo "MPI: $mpiName/$mpiVersion" +} + export -f update_modules export -f no_modules export -f set_pkg_root @@ -255,3 +268,4 @@ export -f build_lib export -f build_nceplib export -f parse_yaml export -f build_info +export -f compilermpi_info From 7097ee4a10e64ba338692139941b89e39b624f14 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 3 May 2021 10:15:42 -0400 Subject: [PATCH 2/4] fix errors in stack_helpers.sh --- stack_helpers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack_helpers.sh b/stack_helpers.sh index 698b25f2..14eaaa2e 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -248,8 +248,8 @@ function build_info() { } function compilermpi_info() { - local compiler=$1 - local mpi=$2 + local compiler=$HPC_COMPILER + local mpi=$HPC_MPI local compilerName=$(echo $compiler | cut -d/ -f1) local compilerVersion=$(echo $compiler | cut -d/ -f2) From c24c7b676a4a62d969eb6ffe9a36d6fcad243e41 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 3 May 2021 10:35:36 -0400 Subject: [PATCH 3/4] remove single library script and consolidate with build_stack.sh. There is no need to maintain 2 scripts --- build_stack.sh | 17 +++++- build_stack_lib.sh | 131 --------------------------------------------- 2 files changed, 15 insertions(+), 133 deletions(-) delete mode 100755 build_stack_lib.sh diff --git a/build_stack.sh b/build_stack.sh index 545be5e2..5545ab56 100755 --- a/build_stack.sh +++ b/build_stack.sh @@ -3,7 +3,7 @@ # the compiler/MPI combination # # sample usage: -# build_stack.sh -p "prefix" -c "config.sh" -y "stack.yaml" -m +# build_stack.sh -p "prefix" -c "config.sh" -y "stack.yaml" -l "library" -m # build_stack.sh -h set -eu @@ -15,12 +15,13 @@ export HPC_STACK_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 usage() { set +x echo - echo "Usage: $0 -p | -c | -y -m -h" + echo "Usage: $0 -p | -c | -y | -l -m -h" echo echo " -p installation prefix DEFAULT: $HOME/opt" echo " -c use configuration file DEFAULT: config/config_custom.sh" echo " -y use yaml file DEFAULT: config/stack_custom.yaml" echo " -m use modules DEFAULT: NO" + echo " -l library to install DEFAULT: ALL" echo " -h display this message and quit" echo exit 1 @@ -29,6 +30,7 @@ usage() { # ============================================================================== # Defaults: +library="" export PREFIX="$HOME/opt" config="${HPC_STACK_ROOT}/config/config_custom.sh" yaml="${HPC_STACK_ROOT}/config/stack_custom.yaml" @@ -45,6 +47,9 @@ while getopts ":p:c:y:mh" opt; do y) yaml=$OPTARG ;; + l) + library=$OPTARG + ;; m) export MODULES=true ;; @@ -105,6 +110,14 @@ fi compilermpi_info build_info +# ============================================================================== +# Is this a single library build or the entire stack? +if [ -n "${library:-""}" ]; then + build_lib $library + echo "build_stack.sh: SUCCESS!" + exit 0 +fi + # ============================================================================== #---------------------- # Compiler and MPI diff --git a/build_stack_lib.sh b/build_stack_lib.sh deleted file mode 100755 index c1a2f694..00000000 --- a/build_stack_lib.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# The purpose of this script is to build a single library in the -# software stack using the compiler/MPI combination -# -# sample usage: -# build_stack_lib.sh -p "prefix" -c "config.sh" -y "stack.yaml" -m -l "library_name" -# build_stack_lib.sh -h - -set -eu - -# root directory for the repository -export HPC_STACK_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# ============================================================================== - -usage() { - set +x - echo - echo "Usage: $0 -l | -p | -c | -y -m -h" - echo - echo " -l library to install DEFAULT: NONE, REQUIRED" - echo " -p installation prefix DEFAULT: $HOME/opt" - echo " -c use configuration file DEFAULT: config/config_custom.sh" - echo " -y use yaml file DEFAULT: config/stack_custom.yaml" - echo " -m use modules DEFAULT: NO" - echo " -h display this message and quit" - echo - exit 1 -} - -# ============================================================================== -# Defaults: -unset LIBRARY -export PREFIX="$HOME/opt" -config="${HPC_STACK_ROOT}/config/config_custom.sh" -yaml="${HPC_STACK_ROOT}/config/stack_custom.yaml" -export MODULES=false - -while getopts ":l:p:c:y:mh" opt; do - case $opt in - l) - LIBRARY=$OPTARG - ;; - p) - export PREFIX=$OPTARG - ;; - c) - config=$OPTARG - ;; - y) - yaml=$OPTARG - ;; - m) - export MODULES=true - ;; - h|\?|:) - usage - ;; - esac -done - -# Make sure required arguments are provided -shift "$(( OPTIND - 1 ))" -set +u -if [ -z "$LIBRARY" ]; then - echo 'Missing -l &2 - usage -fi -set -u - -# ============================================================================== -# Source helper functions -source "${HPC_STACK_ROOT}/stack_helpers.sh" - -#=============================================================================== -# Source the config file -if [[ -e $config ]]; then - source $config -else - echo "ERROR: CONFIG FILE $config DOES NOT EXIST, ABORT!" - exit 1 -fi - -# Source the yaml to determine software and version -if [[ -e $yaml ]]; then - eval $(parse_yaml $yaml "STACK_") -else - echo "ERROR: YAML FILE $yaml DOES NOT EXIST, ABORT!" - exit 1 -fi - -# ============================================================================== -# install with root permissions? -[[ $USE_SUDO =~ [yYtT] ]] && export SUDO="sudo" || export SUDO="" - -# ============================================================================== -# create build directory if needed -pkgdir=${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} -mkdir -p $pkgdir - -# This is for the log files -logdir=$HPC_STACK_ROOT/${LOGDIR:-"log"} -mkdir -p $logdir - -# ============================================================================== -# start with a clean slate -if $MODULES; then - source $MODULESHOME/init/bash - module use $PREFIX/modulefiles/stack - module load hpc -else - no_modules - set_no_modules_path - set_pkg_root -fi - -# ============================================================================== -# Echo compiler, mpi and build information -compilermpi_info -echo "build: $LIBRARY | version: ${STACK_\${LIBRARY}_version}" -# ============================================================================== -# Build desired library -build_lib $LIBRARY - -# ============================================================================== -# optionally clean up -[[ $MAKE_CLEAN =~ [yYtT] ]] && \ - ( $SUDO rm -rf $pkgdir; $SUDO rm -rf $logdir ) - -# ============================================================================== -echo "build_stack_lib.sh: SUCCESS!" From bae349699aa6ecdc78bf647951edf4bfeb760bc9 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 3 May 2021 10:50:56 -0400 Subject: [PATCH 4/4] add missing l to while getopts --- build_stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_stack.sh b/build_stack.sh index 5545ab56..a57d81f2 100755 --- a/build_stack.sh +++ b/build_stack.sh @@ -36,7 +36,7 @@ config="${HPC_STACK_ROOT}/config/config_custom.sh" yaml="${HPC_STACK_ROOT}/config/stack_custom.yaml" export MODULES=false -while getopts ":p:c:y:mh" opt; do +while getopts ":p:c:y:l:mh" opt; do case $opt in p) export PREFIX=$OPTARG