configure

#!/bin/bash
###############################################################################################
#   Rayleigh Configuration Script.
#   This script is built around functions as much as possible.
#   The "actual" script is contained with the function "main,"
#   defined at the bottom of this file.
###############################################################################################
function compiler_version {
  #$1 --version
  VSTR=$($1 --version)
  VTMP="unknown"
  # List Cray first because other compilers might have Cray company name in their version string.
  if [[ $VSTR == *"Cray"* ]]
  then
    VTMP="CRAY"
  fi
  if [[ $VSTR == *"Intel"* ]]
  then
    VTMP="INTEL"
  fi
  if [[ $VSTR == *"GCC"* ]]
  then
    VTMP="GNU"
  fi
  if [[ $VSTR == *"gcc"* ]]
  then
    VTMP="GNU"
  fi
  if [[ $VSTR == *"GNU"* ]]
  then
    VTMP="GNU"
  fi
  if [[ $VSTR == *"AOCC"* ]]
  then
    VTMP="AOCC"
  fi
  echo $VTMP
}

function opt_flags {
  #$1 --version
  if [[ $1 == "INTEL" ]]
  then
    # the value for heap arrays below was chosen to mimic gnu behavior
    echo '-FR -fpp -r8 -O3 -shared-intel -heap-arrays 65'
  fi

  if [[ $1 == "GNU" ]]
  then
    echo '-O3 -ffixed-line-length-132'
  fi

  if [[ $1 == "AOCC" ]]
  then
    echo '-O3 -ffixed-line-length-132'
  fi

  if [[ $1 == "CRAY" ]]
  then
    echo '-O3'
  fi
}

function dbg_flags {
  #$1 --version
  if [[ $1 == "INTEL" ]]
  then
    # the value for heap arrays below was chosen to mimic gnu behavior
    echo '-FR -fpp -r8 -O0 -g -CB -traceback -shared-intel -heap-arrays 65'
  fi
  if [[ $1 == "GNU" ]]
  then
    echo '-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132'
  fi
  if [[ $1 == "AOCC" ]]
  then
    echo '-O0 -g -ffixed-line-length-132'
  fi
  if [[ $1 == "CRAY" ]]
  then
    echo '-O0 -g'
  fi
}

function displayhelp {
    echo ""
    echo "            Rayleigh Configuration Help"
    echo "            ================================"
    echo " "
    echo "  usage: ./configure [options]"
    echo ""
    echo ""
    echo "  Options passed to this script can be used to control the Fortran and C"
    echo "  compiler commands, including the optimization, include, and linking "
    echo "  flags passed to the Makefile."
    echo ""
    echo "  This script generates the file /src/build/Machine_Definitions,"
    echo "  which is included by the Makefile and used to control the compilation" 
    echo "  process.  If configure fails to automatically produce a viable 'definitions'"
    echo "  file, you may need to specify one or more of the options below."
    echo ""
    echo "  The configure/make/make-install process will generate two rayleigh executables "
    echo "  by default, compiled as follows."
    echo ""
    echo "  rayleigh.opt: "
    echo "      \$FC \$FFLAGS_OPT \$INCLUDE -o  rayleigh.opt {rayleigh object files} \$LIBFLAGS"
    echo ""
    echo "  rayleigh.dbg: "
    echo "      \$FC \$FFLAGS_DBG \$INCLUDE -o  rayleigh.dbg {rayleigh object files} \$LIBFLAGS"    
    echo ""
    echo "  All variables used in the compilation syntax above may be manipulated"
    echo "  by specifying options of the same name, described below.  Several other"
    echo "  options may also be specified, which are primarily designed to fine-tune"
    echo "  the automatic values of these variables generated by the configure script."
    echo ""
    echo ""
    echo "  OPTIONS: "
    echo ""
    echo "  --blank: "
    echo "      Causes configure to produce a blank definitions file in"
    echo "      <rayleigh root>/src/build/Machine_Definitions"
    echo ""

    echo "  --CC=<C compiler name> "
    echo "      Specifies the C-compiler command to be invoked by Make."
    echo "      CC must be in your PATH, or else configure will fail. If CC"
    echo "      is not specified, configure will check your environment"
    echo "      and use the value of the \$CC environment variable, if defined." 
    echo "      The C-compiler command DOES NOT need to be MPI-capable."
    echo ""

    echo "  --devel / -devel "
    echo "      Forces Rayleigh to build with debugging flags and no."
    echo "      vectorization (useful for quick development builds)."
    echo ""

    echo "  --FC=<fortran compiler name> "
    echo "      Specifies the Fortran compiler command to be invoked by Make."
    echo "      FC must be in your PATH, or else configure will fail.  FC must"
    echo "      refer to an MPI-capable, Fortran 2003-compliant compiler. If FC"
    echo "      is not specified, configure will check your environment"
    echo "      and use the value of the \$FC environment variable, if defined." 
    echo ""

    echo "  --FFLAGS=<compilation flags>:"
    echo "      Used to specify both optimization and debugging flags.  This is"
    echo "      necessary on machines that do not use the GNU or Intel compilers."
    echo "      If FFLAGS is set, it will override any optimization or debugging"
    echo "      flags automatically generated by configure.  FFLAGS will also "
    echo "      override the values of FFLAGS_OPT and FFLAGS_DBG, if set."
    echo ""

    echo "  --FFLAGS_DBG=<debugging flags>:"
    echo "      Specifies debugging flags used to compile rayleigh.dbg.  This"
    echo "      is necessary on machines that do not use the GNU or Intel compilers."
    echo "      If FFLAGS_DBG is set, it will override any debugging flags that are"
    echo "      automatically generated by configure."
    echo ""

    echo "  --FFLAGS_OPT=<optimization flags>:"
    echo "      Specifies optimization flags used to compile rayleigh.opt.  This"
    echo "      is necessary on machines that do not use the GNU or Intel compilers."
    echo "      If FFLAGS_OPT is set, it will override any optimization flags that are"
    echo "      automatically generated by configure."
    echo ""

    echo "  -h ; -help ; --help:" 
    echo "      Display this message."
    echo ""

    echo "  --INCLUDE=<include flags>"
    echo "      Set this flag to manually override the include flags generated by"
    echo "      invoking configure with --with-fftw, -mkl, or --with-mkl."
    echo ""

    echo "  --LIBFLAGS=<library linking flags>:"
    echo "      Set this flag to manually specify the linking flags invoked by the"
    echo "      compiler.  Setting this value will override any values set using the"
    echo "      --with-blas, --with-lapack, --with-fftw, --with-mkl, and -mkl flags."
    echo ""
    
    echo "  -mkl:     " 
    echo "      Link to MKL's FFTW, LAPack, and BLAS routines."
    echo "      This is done automatically when using the Intel Compiler."
    echo "      The MKLROOT environment variable must be defined. If this"
    echo "      variable is undefined, call configure with the --with-mkl flag."
    echo ""

    echo "  -conda-mkl:     " 
    echo "      Use this flag in place of -mkl and, if desired, alonside -with-mkl to account"
    echo "      for the conda-provided MKL directory structure."
    echo "      Usage Example 1:  ./configure -conda-mkl             "
    echo "      Usage Example 2:  ./configure --with-mkl='/my/mkl/root' -conda-mkl "
    echo ""

    echo "  -debian-mkl:     "
    echo "      Use this flag in place of -mkl if you plan to use MKL as provided by"
    echo "      the intel-mkl package in Debian/Ubuntu."
    echo ""

    echo "  --nodirs ; -nodirs: "
    echo "      Causes Make to compile Rayleigh without directory creation support."
    echo "      This is intended ONLY for machines where the make process fails when "
    echo "      linking compiled C-code to Fortran."
    echo ""
    echo "      If Rayleigh is compiled without directory creation support, users "
    echo "      will need to run the make_dirs script (found in rayleigh/etc) within "
    echo "      each run directory they create for Rayleigh."
    echo ""

    echo "  --openblas ; -openblas:"
    echo "      Set this flag to link against the OpenBLAS library"
    echo "      Depending on the value of --static-blas, Rayleigh will be"
    echo "      linked against either <BLAS ROOT>/lib/libopenblas.so or"
    echo "      <BLAS ROOT>/lib/libopenblas.a"
    echo ""

    echo "  --mpi-f77 ; -mpi-f77:"
    echo "      Set this flag to use mpif.h instead of the MPI F90 module."
    echo "      This can be useful on systems where the current compiler does not"
    echo "      match the one used to compile MPI."
    echo ""

    echo "  --mpi-f08 ; -mpi-f08:"
    echo "      Set this flag to use the newer Fortran 2008 MPI module."
    echo "      This needs to be supported by the MPI install."
    echo "      It may help with compiler errors due to type mismatches."
    echo ""

    echo "  --prefix=<RAYLEIGH PREFIX>: "
    echo "      Specifies the directory in which to place Rayleigh's executables"
    echo "      when "make install" is run.  By default, executables are placed in"
    echo "      rayleigh/bin.  If prefix is specified, executables will be placed"
    echo "      in <RAYLEIGH PREFIX>\bin."
    echo ""

    echo "  --static-blas ; -static-blas:"
    echo "      By default, Rayleigh will be dynamically linked to a shared BLAS "
    echo "      or OpenBLAS library.  Set this flag to link statically instead."
    echo ""

    echo "  --static-fftw ; -static-fftw:"
    echo "      By default, Rayleigh will be dynamically linked to a shared FFTW "
    echo "      library.  Set this flag to link statically instead."
    echo ""

    echo "  --static-lapack ; -static-lapack:"
    echo "      By default, Rayleigh will be dynamically linked to a shared LAPack "
    echo "      library.  Set this flag to link statically instead."
    echo ""

    echo "  --ifort-march ; -ifort-march:"
    echo "      ifort only: Use the generic -march options for architecture selection "
    echo "      instead of the -x options. This is needed for optimizing the code for "
    echo "      non-Intel CPUs."
    echo ""

    echo "  --with-mkl=<path to the MKL root directory>"
    echo "      Set this value if the MKLROOT environment variable is not set on your"
    echo "      system, or if you wish to link against an alternative MKL installation."
    echo ""
    echo "      On most Intel-based systems, the \$MKLROOT environment defines the "
    echo "      MKL root directory.  If specifying this flag, be sure that you have"
    echo "      specified the root directory properly (see below)."
    echo ""
    echo "      For Intel-compiler builds, verify that the following file exists:"
    echo "          <MKL ROOT>/include/fftw/fftw3.f "
    echo ""
    echo "      For GNU-compiler builds, verify that the following files exist: "
    echo "          <MKL ROOT>/include/fftw/fftw3.f"
    echo "          <MKL ROOT>/lib/intel64/libmkl_gf_lp64.so"
    echo "          <MKL ROOT>/lib/intel64/libmkl_sequential.so"
    echo "          <MKL ROOT>/lib/intel64/libmkl_core.so"
    echo ""

    echo "  --with-blas=<BLAS ROOT>"
    echo "      Set this flag to specify the BLAS root directory.  To ensure that"
    echo "      the root directory is specified properly, verify that AT LEAST ONE" 
    echo "      of the following files exist: "
    echo "          <BLAS ROOT>/lib/libblas.so"
    echo "          <BLAS ROOT>/lib/libblas.a"
    echo "          <BLAS ROOT>/lib/libopenblas.so"
    echo "          <BLAS ROOT>/lib/libopenblas.a"
    echo ""

    echo "  --with-fftw=<FFTW ROOT>"
    echo "      Set this flag to specify the fftw root directory.  To ensure that"
    echo "      the root directory is specified properly, verify that the following "
    echo "      files exist: "
    echo "          <FFTW ROOT>/include/fftw3.f"
    echo "          <FFTW ROOT>/lib/{ libfftw3.so OR libfftw3.a }"
    echo ""

    echo "  --with-lapack=<LAPACK ROOT>"
    echo "      Set this flag to specify the LAPack root directory.  To ensure that"
    echo "      the root directory is specified properly, verify that AT LEAST ONE" 
    echo "      of the following files exist: "
    echo "          <LAPACK ROOT>/lib/liblapack.so"
    echo "          <LAPACK ROOT>/lib/liblapack.a"
    echo ""

    echo "  --with-custom=<CUSTOM ROOT>"
    echo "      Set this flag to specify the location of custom Rayleigh source files."
    echo "      These files will be copied to the build directory and override any of"
    echo "      the standard Rayleigh source files."
    echo ""

}


###############################################################################################
# MAIN 
#

RAYLEIGHROOT=$PWD
PREFIX=$RAYLEIGHROOT
NEED_HELP=no
USE_MKL="FALSE"
CONDAMKL="FALSE"
DEBIANMKL="FALSE"
CUSTOMROOT="none"


###############################################################################################

#Check options and manage environment variables accordingly

for option
do
    case $option in

    -help | --help | -h)
      displayhelp;
      exit 0;
      ;;

    --with-custom=*)
      export CUSTOMROOT=`expr "x$option" : "x-*with-custom=\(.*\)"`;
      ;;

    -prefix=* | --prefix=*)
      PREFIX=`expr "x$option" : "x-*prefix=\(.*\)"`
      ;;

    --FFLAGS=*)
      RAFFLAGS=`expr "x$option" : "x-*FFLAGS=\(.*\)"`;
      CUSTOMFLAGS="TRUE";
      ;;

    --FFLAGS_OPT=*)
      RAFFLAGS_OPT=`expr "x$option" : "x-*FFLAGS_OPT=\(.*\)"`;
      CUSTOMOPT="TRUE";
      ;;

    --FFLAGS_DBG=*)
      RAFFLAGS_DBG=`expr "x$option" : "x-*FFLAGS_DBG=\(.*\)"`;
      CUSTOMDBG="TRUE";
      ;;


    --LIBFLAGS=*)
      RALIB=`expr "x$option" : "x-*LIBFLAGS=\(.*\)"`;
      CUSTOMLIB="TRUE";
      ;;

    --INCLUDE=*)
      RAINC=`expr "x$option" : "x-*INCLUDE=\(.*\)"`;
      CUSTOMINC="TRUE";
      ;;

    --FC=*)
      export FC=`expr "x$option" : "x-*FC=\(.*\)"`;
      ;;

    --nodirs | -nodirs )
      NODIRS="TRUE";
      ;;

    --devel | -devel)
      DEVELMODE="TRUE";
      ;;

    -conda-mkl)
      if [ ! -z "$MKLROOT" ]
      then
        CONDAMKL="TRUE"
        USE_MKL="TRUE"
        RAMKLROOT=$MKLROOT
      else
        echo "Error Code 5: "
        echo "MKLROOT is not defined"
        echo "MKLROOT must be defined in your environment to use the -mkl flag."
        echo "exiting..."
        exit 5
      fi 
      ;;

    -debian-mkl)
      DEBIANMKL="TRUE"
      USE_MKL="TRUE"
      ;;

    -mkl)
      if [ ! -z "$MKLROOT" ]
      then
        USE_MKL="TRUE";
        RAMKLROOT=$MKLROOT
      else
        echo "Error Code 5: "
        echo "MKLROOT is not defined"
        echo "MKLROOT must be defined in your environment to use the -mkl flag."
        echo "exiting..."
        exit 5
      fi
      ;;

    --with-mkl=*)
      RAMKLROOT=`expr "x$option" : "x-*with-mkl=\(.*\)"`;
      USE_MKL="TRUE";
      ;;

    --with-fftw=*)
      FFTWROOT=`expr "x$option" : "x-*with-fftw=\(.*\)"`;
      ;;

    --with-blas=*)
      BLASROOT=`expr "x$option" : "x-*with-blas=\(.*\)"`;
      ;;

    --openblas | -openblas )
      USEOPENBLAS="TRUE";
      ;;

    --mpi-f77 | -mpi-f77 )
      USEMPIF77="TRUE";
      ;;

    --mpi-f08 | -mpi-f08 )
      USEMPIF08="TRUE";
      ;;

    --with-lapack=*)
      LPROOT=`expr "x$option" : "x-*with-lapack=\(.*\)"`;
      ;;

    --static-blas | -static-blas )
      STATICBLAS="TRUE";
      ;;

    --static-lapack | -static-lapack )
      STATICLAPACK="TRUE";
      ;;

    --static-fftw | -static-fftw )
      STATICFFTW="TRUE";
      ;;

    --ifort-march | -ifort-march )
      IFORTMARCH="TRUE";
      ;;

    --CC=*)
      export CC=`expr "x$option" : "x-*CC=\(.*\)"`;
      ;;

    --blank)
      BLANKMACHINE="TRUE";
      ;;

    *)
      echo "unknown option: $option"
      exit 1
      ;;

    esac
done
echo ""
echo " -----------------------------------------------------------------------------"
VERSION_FILE="VERSION"
VERSION=`cat $VERSION_FILE`
echo "                          Rayleigh v$VERSION Configuration"
echo " -----------------------------------------------------------------------------"
echo ""
FVERSION="unknown"
CVERSION="unknown"
if [[ $BLANKMACHINE == "TRUE" ]]
then
      echo "Creating blank definitions file in $RAYLEIGHROOT/src/build/Machine_Definitions""."
      cp $RAYLEIGHROOT/src/Machinefiles/machine.blank $RAYLEIGHROOT/src/build/Machine_Definitions
      echo "Edit this file as appropriate, and then run 'make' from within $RAYLEIGHROOT""."
      echo "PREFIX = "$PREFIX > make.inc
      if [[ $NODIRS == "TRUE" ]]
      then
        echo "NODIRS = 1" >> make.inc
      fi
      if [[ $CUSTOMROOT == "none" ]]
      then
        echo "CUSTOMROOT = " >> make.inc
      else
        echo "CUSTOMROOT = "$CUSTOMROOT >> make.inc
      fi
      exit 0

fi

if [[ $FC == "" ]]
then
  echo ""
  echo "--- Error Code 1 ---"
  echo "No Fortran compiler specified."
  echo "Please specify a valid Fortran compiler by either: "
  echo "  1)  setting the FC environment variable "
  echo "  2)  calling this script with the --FC={compiler command} option "
  echo ""
  exit 1
else
 
  #Check to see that the file exists (this seems to catch broken softlinks)
  if [[ ! -e "$(which $FC)" ]]
  then
    echo " "
    echo "           --- Error Code 2 ---     "
    echo " Specified Fortran compiler does not exist." 
    echo " FC is set to: "$FC
    echo "               ...exiting..."
    echo " "
    exit 2
  fi


  FCOMP=$FC
  FVERSION=$(compiler_version $FC)

  OPT_FLAGS=$(opt_flags $FVERSION)
  DBG_FLAGS=$(dbg_flags $FVERSION)

  if [ "$CUSTOMOPT" == "TRUE" ]
  then
    echo "  Overriding optimization flags."
    OPT_FLAGS=$RAFFLAGS_OPT
  fi

  if [ "$CUSTOMDBG" == "TRUE" ]
  then
    echo "  Overriding debug flags."
    DBG_FLAGS=$RAFFLAGS_DBG
  fi

  if [ "$CUSTOMFLAGS" == "TRUE" ]
  then
    echo "  Overriding all flags..,"
    OPT_FLAGS=$RAFFLAGS
    DBG_FLAGS=$RAFFLAGS
  fi

  # internally used preprocessor flags should appear after custom flags are set
  if [ "$USEMPIF77" == "TRUE" ]
  then
          OPT_FLAGS="$OPT_FLAGS -DUSE_MPI_F77_BINDINGS"
          DBG_FLAGS="$DBG_FLAGS -DUSE_MPI_F77_BINDINGS"
  fi

  if [ "$USEMPIF08" == "TRUE" ]
  then
          OPT_FLAGS="$OPT_FLAGS -DUSE_MPI_F08_BINDINGS"
          DBG_FLAGS="$DBG_FLAGS -DUSE_MPI_F08_BINDINGS"
  fi

  if [ $FVERSION == "INTEL" ]
  then
      OPT_FLAGS="$OPT_FLAGS -DINTEL_COMPILER"
      DBG_FLAGS="$DBG_FLAGS -DINTEL_COMPILER"
  fi

  if [ $FVERSION == "GNU" ]
  then
      OPT_FLAGS="$OPT_FLAGS -DGNU_COMPILER"
      DBG_FLAGS="$DBG_FLAGS -DGNU_COMPILER"
  fi

  if [ $FVERSION == "AOCC" ]
  then
      OPT_FLAGS="$OPT_FLAGS -DAOCC_COMPILER"
      DBG_FLAGS="$DBG_FLAGS -DAOCC_COMPILER"
  fi

  if [ $FVERSION == "CRAY" ]
  then
      OPT_FLAGS="$OPT_FLAGS -DCRAY_COMPILER"
      DBG_FLAGS="$DBG_FLAGS -DCRAY_COMPILER"
  fi

  echo ' '
  echo " Compiler manufacturer      : " $FVERSION
  echo ' Fortran compiler command   : ' $FCOMP
  echo ' Fortran optimization flags : ' $OPT_FLAGS
  echo ' Fortran debugging flags    : ' $DBG_FLAGS 

  if [ $FVERSION == "INTEL" ]
  then
      if [ ! -z "$MKLROOT" ]
      then
        USE_MKL="TRUE";
        RAMKLROOT=$MKLROOT
        echo " Intel compiler set:  MKLROOT is defined"
      else
        echo "         --- Error Code 5 ---     "
        echo " The MKLROOT environment variable is undefined."
        echo " Please define this environment variable or  "
        echo " rerun configure using the --with-mkl flag."
        echo " e.g., ./configure --with-mkl='/your/path/to/mkl'"
        echo " exiting..."
        echo ""
        exit 5
      fi
  fi
fi

if [[ $CC == "" ]]
then
  echo ""
  echo "--- Error Code 3 ---"
  echo "No C-compiler specified."
  echo "Please specify a valid C-compiler by either: "
  echo "  1)  setting the CC environment variable "
  echo "  2)  calling this script with the --CC={compiler command} option "
  echo ""
  exit 1
else
 
  #Check to see that the file exists (this seems to catch broken softlinks)
  if [[ ! -e "$(which $CC)" ]]
  then
    echo " "
    echo "           --- Error Code 4 ---     "
    echo " Specified C-compiler does not exist." 
    echo " CC is set to: "$CC
    #echo " Ensure that this executable is in your path."
    #echo " If this is a symlink, ensure that it is not broken."
    echo "               ...exiting..."
    echo " "
    exit 2
  fi


  CCOMP=$CC
  CVERSION=$(compiler_version $CC)
  echo " C compiler command         :  "$CC
  #echo " C compiler manufacturer    :  "$CVERSION
  echo ' '
  #Suppress the warning related to mkdir
  if [[ $CVERSION == "GNU" ]]
  then
    CC="$CC -w"
  fi
  if [[ $CVERSION == "INTEL" ]]
  then
    CC="$CC -w"
  fi

fi

DEFAULT=opt
DEBUG=dbg

RVTAGS[0]="DEFAULT"
RVTAGS[1]="DEBUG"
RVALUES[0]=$DEFAULT 
RVALUES[1]=$DEBUG
FFLAGS[0]=$OPT_FLAGS
FFLAGS[1]=$DBG_FLAGS
VECLEVEL[1]=""
nvers=2
MULTIVEC=FALSE

if [[ $DEVELMODE == "TRUE" ]]
then
  echo "************************************************************************"
  echo " Development mode has been activated. Only rayleigh.dbg"
  echo " will be compiled with debugging flags and no vectorization."
  RVTAGS[0]="DEBUG"
  RVALUES[0]=$DEBUG 
  FFLAGS[0]=$DBG_FLAGS
  nvers=1
fi


if [[ $FVERSION == "INTEL" ]] &&  [[ $DEVELMODE != "TRUE" ]]
then

  echo "***********************************************************************"
  echo " Intel compiler detected.  Please indicate desired SIMD instruction set."
  echo ""
  echo " Suggested instruction sets are: "
  echo " -- Westmere cores                  :  SSE4.2"
  echo " -- Sandy Bridge / Ivy Bridge cores :  AVX"
  echo " -- Haswell /Broadwell              :  AVX2 "
  echo " -- SkyLake / Knights Landing       :  AVX512 "
  echo " -- Support for all of the above    :  FAT"
  echo ""
    OPTIONS="SSE4.2 AVX AVX2 AVX512 ALL FAT NONE"
    if [[ $IFORTMARCH == TRUE ]]
    then
            OSSE42="-march=corei7"
            OAVX="-march=corei7-avx"
            OAVX2="-march=core-avx2"
            # There seems to be no direct equivalent for COMMON-AVX512 using the march option.
            OAVX512="-xCOMMON-AVX512 -axCORE-AVX512,MIC-AVX512"
    else
            OSSE42="-xSSE4.2"
            OAVX="-xAVX"
            OAVX2="-xCORE-AVX2"
            OAVX512="-xCOMMON-AVX512 -axCORE-AVX512,MIC-AVX512"
    fi

    select opt in $OPTIONS; do
       if [ "$opt" = "SSE4.2" ]; then
        FFLAGS[0]=$OPT_FLAGS" $OSSE42"
        break
       elif [ "$opt" = "AVX" ]; then
        FFLAGS[0]=$OPT_FLAGS" $OAVX"
        break        
       elif [ "$opt" = "AVX2" ]; then
        FFLAGS[0]=$OPT_FLAGS" $OAVX2"
        break            
       elif [ "$opt" = "AVX512" ]; then
        FFLAGS[0]=$OPT_FLAGS" $OAVX512"
        break      
       elif [ "$opt" = "ALL" ]; then
         DEFAULT='avx2'
         nvers=5
         MULTIVEC=TRUE
         FFLAGS[0]=$OPT_FLAGS" $OAVX2"
         FFLAGS[2]=$OPT_FLAGS" $OAVX"
         FFLAGS[3]=$OPT_FLAGS" $OSSE42"
         FFLAGS[4]=$OPT_FLAGS" $OAVX512"
         RVALUES[0]=$DEFAULT 
         RVALUES[2]=avx
         RVALUES[3]=sse
         RVALUES[4]=avx512
         RVTAGS[2]="AVX"
         RVTAGS[3]="SSE"
         RVTAGS[4]="AVX512"
         break
       elif [ "$opt" = "FAT" ]; then
        FFLAGS[0]=$OPT_FLAGS" $OSSE42 -axAVX,CORE-AVX2,CORE-AVX512,MIC-AVX512"
        break
       elif [ "$opt" == "NONE" ]; then
         FFLAGS[0]=$OPT_FLAGS
         break
       else
        echo "Invalid option.  Choosing no vectorization"
        break
       fi
    done
  echo "***********************************************************************"

fi

if [[ $FVERSION == "GNU" || $FVERSION == "AOCC" ]] &&  [[ $DEVELMODE != "TRUE" ]]
then

  echo "***********************************************************************"
  echo " GNU compiler detected.  Please indicate desired SIMD instruction set."
  echo ""
  echo " Suggested instruction sets are: "
  echo " -- auto-detect (make sure compiling CPU is same as compute) : native"
  echo " -- Westmere cores (SSE4.2)                                  : westmere"
  echo " -- Sandy Bridge / Ivy Bridge cores (AVX)                    : sandybridge/ivybridge"
  echo " -- Haswell / Broadwell (AVX2)                               : haswell/broadwell"
  echo " -- SkyLake (AVX2 only)                                      : skylake"
  echo " -- SkyLake / Knights Landing (AVX512)                       : skylake-avx512/knl"
  echo " -- separate executables for each architecture               : ALL"
  echo " -- no vectorization                                         : NONE"
  echo ""
    CPUTYPES="native westmere sandybridge ivybridge haswell broadwell skylake skylake-avx512 knl"
    OPTIONS="${CPUTYPES} ALL NONE"
    select opt in $OPTIONS; do
       if [ -z "$opt" ]; then
         echo "Invalid option.  Choosing no vectorization"
         break
       elif [ "$opt" = "NONE" ]; then
         FFLAGS[0]=$OPT_FLAGS
         break
       elif [ "$opt" = "ALL" ]; then
         DEFAULT='avx2'
         nvers=5
         MULTIVEC=TRUE
         FFLAGS[0]=$OPT_FLAGS" -march=haswell"
         FFLAGS[2]=$OPT_FLAGS" -march=sandybridge"
         FFLAGS[3]=$OPT_FLAGS" -march=westmere"
         FFLAGS[4]=$OPT_FLAGS" -march=knl"
         RVALUES[0]=$DEFAULT
         RVALUES[2]=avx
         RVALUES[3]=sse
         RVALUES[4]=avx512
         RVTAGS[2]="AVX"
         RVTAGS[3]="SSE"
         RVTAGS[4]="AVX512"
         break
       fi
       FFLAGS[0]="${OPT_FLAGS} -march=${opt}"
       if [[ "$opt" == "native" && $FVERSION == "GNU" ]]; then
               echo "CPU autodetect result: $($FC -march=native -Q --help=target|grep '^\s\+-march'|awk '{print $2}')"
       fi
       break
    done
  echo "***********************************************************************"

fi

###############################################################################################
# Set the include flags
# For now, if MKL is used, so is MKL's FFTW interface
MKLFFTW=$USE_MKL
if [[ $MKLFFTW == "TRUE" ]]
then
    if [[ $CONDAMKL == "TRUE" ]]
    then
        RAYLEIGH_INC="-I$RAMKLROOT/include"
    elif [[ $DEBIANMKL == "TRUE" ]]
    then
        RAYLEIGH_INC="-I/usr/include/mkl/fftw"
    else
        RAYLEIGH_INC="-I$RAMKLROOT/include -I$RAMKLROOT/include/fftw"
    fi
else
    RAYLEIGH_INC=-I$FFTWROOT/include
fi
if [ "$CUSTOMINC" == "TRUE" ]
then
  RAYLEIGH_INC=$RAINC
fi
echo " Include flags              :  "$RAYLEIGH_INC
echo ""

###############################################################################################
# Standard LIBRARIES
if [[ $OSTYPE == *"darwin"* ]]
then
    STD_FLAGS=""
else
    STD_FLAGS=""
fi

###############################################################################################
# BLAS, LAPACK, and FFTW LIBRARIES
if [[ $USE_MKL == "TRUE" ]]
then
    if [[ $FVERSION == "INTEL" ]]
    then
        LIB_FLAGS="-mkl"
    elif [[ $FVERSION == "GNU" || $FVERSION == "AOCC" ]]
    then
        # Start by specifying the directory, unless installed by apt on debian
        # in which case it is in default path
        if [[ $DEBIANMKL == "FALSE" ]]
        then
          LIB_FLAGS="-L${RAMKLROOT}/lib"
        fi

        if [[ $CONDAMKL == "TRUE" ]]
        then
            if [[ $OSTYPE == *"darwin"* ]]
            then
                OS_SPECIFIC_LIB="-Wl,-rpath,${RAMKLROOT}/lib -lmkl_intel_lp64"
            else
                OS_SPECIFIC_LIB="-Wl,--no-as-needed -lmkl_gf_lp64"
            fi
        else
            OS_SPECIFIC_LIB="-Wl,--no-as-needed -lmkl_gf_lp64"
        fi

        # Add OS independent options
        LIB_FLAGS="${LIB_FLAGS} ${OS_SPECIFIC_LIB} -lmkl_sequential -lmkl_core -lpthread -lm -ldl"
    fi
else

    #WE NEED TO RUN CHECKS HERE

    #BLAS
    if [[ $USEOPENBLAS == "TRUE" ]]
    then
    # OPENBLAS already comes with LAPACK
        if [[ $STATICBLAS == "TRUE" ]]
        then
          BLASLINK="-L$BLASROOT/lib/libopenblas.a"
        else
          BLASLINK="-L$BLASROOT/lib -lopenblas"
        fi
    else
        if [[ $STATICBLAS == "TRUE" ]]
        then
          BLASLINK="-L$BLASROOT/lib/libblas.a"
        else
          BLASLINK="-L$BLASROOT/lib -lblas"
        fi
        #LAPACK
        if [[ $STATICLAPACK == "TRUE" ]]
        then
            LAPACKLINK="-L$LPROOT/lib/liblapack.a"
        else
            LAPACKLINK="-L$LPROOT/lib -llapack"
        fi
    fi
    
    #FFTW
    if [[ $STATICFFTW == "TRUE" ]]
    then
        FFTWLINK="-L$FFTWROOT/lib/libfftw3.a"
    else
        FFTWLINK="-L$FFTWROOT/lib -lfftw3"
    fi

    # Combine the flags
    LIB_FLAGS="${LAPACKLINK} ${FFTWLINK} ${BLASLINK}"
fi

if [ "$CUSTOMLIB" == "TRUE" ]
then
  LIB_FLAGS=$RALIB
fi

echo " Library flags              :  "$LIB_FLAGS
echo ""
###############################################################################################
# Create a machine file
# (I) Compiler commands
OFILE=machine.temp

BORDER="################################################################################"
echo $BORDER > $OFILE
echo "#   I.  Compiler Commands" >> $OFILE
echo " " >> $OFILE
echo "FC = "$FC >> $OFILE
echo "CC = "$CC >> $OFILE
echo " " >> $OFILE
echo $BORDER >> $OFILE
###############################################################################################
# Output the executable version info
# (II)  Rayleigh Versions
echo "#   II.  Version Names" >> $OFILE
echo "" >> $OFILE
COUNTER=0
while [ $COUNTER -lt $nvers ]; do
  echo "${RVTAGS[$COUNTER]} = ${RVALUES[$COUNTER]}" >> $OFILE
  let COUNTER=COUNTER+1
done
echo "" >> $OFILE
echo $BORDER >> $OFILE
###############################################################################################
# Create the version list
echo "# III.  Whitespace-Separated List of Version Names" >> $OFILE
echo "" >> $OFILE
VLIST="VERSIONS = "
COUNTER=0
while [ $COUNTER -lt $nvers ]; do
  VLIST="$VLIST \$("${RVTAGS[$COUNTER]}")"
  let COUNTER=COUNTER+1
done
echo $VLIST >> $OFILE
echo $BORDER >> $OFILE

################################################################################
#  IV.  Include flags
echo "# IV.  Include Flags" >> $OFILE
echo "" >> $OFILE
echo "RAYLEIGH_INC = "$RAYLEIGH_INC >> $OFILE
echo "" >> $OFILE
echo $BORDER >> $OFILE
################################################################################
#  V.  Optimization and Library flags
echo "# V.  Optimization and Library Flags" >> $OFILE
echo "" >> $OFILE
COUNTER=0
while [ $COUNTER -lt $nvers ]; do
  VSTR="FFLAGS_\$("${RVTAGS[$COUNTER]}")"
  FSTR=${FFLAGS[$COUNTER]}" \$(RAYLEIGH_INC)" >> $OFILE
  echo "$VSTR = $FSTR" >> $OFILE
  VSTR="LIB_\$("${RVTAGS[$COUNTER]}")"
  echo "$VSTR = $LIB_FLAGS" >> $OFILE
  echo "" >> $OFILE
  let COUNTER=COUNTER+1
done



#################################################################################
# VI.   Generate a header file for inclusion in Run_Parameters.F90


mkdir -p src/build/compiled

COUNTER=0
while [ $COUNTER -lt $nvers ]; do
	HFILE=src/build/Run_Param_Header.${RVALUES[$COUNTER]}.F
	./src/gen_header.sh $HFILE "${RVALUES[$COUNTER]}" "${FFLAGS[$COUNTER]}" "$FULL_LIB"
	let COUNTER=COUNTER+1
done


#################################################################################
# VII.   Prepare the build directory



NOCOMM=src/build/machine.no_comments
MACHDEF=src/build/Machine_Definitions
rm -f $NOCOMM
rm -f $MACHDEF

#First, we strip any user comments out of MACHINEFILE as we copy it to NOCOMM
grep -o '^[^#]*' $OFILE > $NOCOMM

#Next we copy NOCOMM to MACHDEF, removing trailing whitespace in the process
#  This is super important as errors can arise in the make process otherwise.
sed -E 's/[ '$'\t'']+$//' $NOCOMM > $MACHDEF
rm $NOCOMM
rm $OFILE
echo " "


echo "PREFIX = "$PREFIX > make.inc
if [[ $NODIRS == "TRUE" ]]
then
  echo "NODIRS = 1" >> make.inc
fi
if [[ $CUSTOMROOT == "none" ]]
then
  echo "CUSTOMROOT = " >> make.inc
else
  echo "CUSTOMROOT = "$CUSTOMROOT >> make.inc
fi



echo " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Configuration is complete."
echo " A Machine_Definitions files has been placed in $RAYLEIGHROOT/src/build."
echo ""
echo " Typing 'make' will generate the following executables (optimization indicated): "
COUNTER=0
while [ $COUNTER -lt $nvers ]; do
  PROGNAME="rayleigh."${RVALUES[$COUNTER]}
  FSTR=${FFLAGS[$COUNTER]}
  echo " "$PROGNAME"  :  " $FSTR
  let COUNTER=COUNTER+1
done
echo ""
echo " Now run 'make' to compile the code."
echo " Once make completes, run 'make install' to install your executables into:"
echo " "$PREFIX"/bin"