diff --git a/Externals.cfg b/Externals.cfg index 13deca839f..f89fab12a5 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,37 +1,37 @@ # External sub-modules of global-workflow [FV3GFS] -tag = GFS.v16.0.13 +tag = GFS.v16.0.14_port2s4 local_path = sorc/fv3gfs.fd -repo_url = https://github.com/ufs-community/ufs-weather-model.git +repo_url = https://github.com/DavidHuber-NOAA/ufs-weather-model.git protocol = git required = True [GSI] -tag = gfsda.v16.0.0 +tag = gfsda.v16.0.0_port2s4 local_path = sorc/gsi.fd -repo_url = https://github.com/NOAA-EMC/GSI.git +repo_url = https://github.com/DavidHuber-NOAA/GSI.git protocol = git required = True [GLDAS] -tag = gldas_gfsv16_release.v1.11.0 +tag = gldas_gfsv16_release.v1.11.0_port2s4 local_path = sorc/gldas.fd -repo_url = https://github.com/NOAA-EMC/GLDAS.git +repo_url = https://github.com/DavidHuber-NOAA/GLDAS.git protocol = git required = True [EMC_post] -tag = upp_gfsv16_release.v1.1.0 +tag = upp_gfsv16_release.v1.1.0_port2s4 local_path = sorc/gfs_post.fd -repo_url = https://github.com/NOAA-EMC/EMC_post.git +repo_url = https://github.com/DavidHuber-NOAA/EMC_post.git protocol = git required = True [UFS_UTILS] -tag = ops-gfsv16.0.0 +tag = ops-gfsv16.0.0_port2s4 local_path = sorc/ufs_utils.fd -repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git +repo_url = https://github.com/DavidHuber-NOAA/UFS_UTILS.git protocol = git required = True @@ -43,9 +43,9 @@ protocol = git required = True [EMC_gfs_wafs] -tag = gfs_wafs.v6.0.10 +tag = gfs_wafs.v6.0.10_port2s4 local_path = sorc/gfs_wafs.fd -repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git +repo_url = https://github.com/DavidHuber-NOAA/EMC_gfs_wafs.git protocol = git required = False diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter.s4 b/modulefiles/fv3gfs/enkf_chgres_recenter.s4 new file mode 100644 index 0000000000..5a4d8fcf56 --- /dev/null +++ b/modulefiles/fv3gfs/enkf_chgres_recenter.s4 @@ -0,0 +1,14 @@ +#%Module##################################################### +## enkf_chgres_recenter component - s4 +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load nemsio/2.5.2 +module load bacio/2.4.1 +module load w3nco/2.4.1 +module load ip/3.3.3 +module load sp/2.3.3 + +export FC="ifort -march=ivybridge" diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter_nc.s4 b/modulefiles/fv3gfs/enkf_chgres_recenter_nc.s4 new file mode 100644 index 0000000000..62f493a16d --- /dev/null +++ b/modulefiles/fv3gfs/enkf_chgres_recenter_nc.s4 @@ -0,0 +1,16 @@ +#%Module##################################################### +## enkf_chgres_recenter component - hera +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load bacio/2.4.1 +module load w3nco/2.4.1 +module load ip/3.3.3 +module load sp/2.3.3 +module load hdf5/1.10.6 +module load netcdf/4.7.4 + + +export FC="ifort -march=ivybridge" diff --git a/modulefiles/fv3gfs/gaussian_sfcanl.s4 b/modulefiles/fv3gfs/gaussian_sfcanl.s4 new file mode 100644 index 0000000000..25a241db78 --- /dev/null +++ b/modulefiles/fv3gfs/gaussian_sfcanl.s4 @@ -0,0 +1,15 @@ +#%Module##################################################### +## gaussian_sfcanl build module for S4 +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load w3nco/2.4.1 +module load bacio/2.4.1 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load hdf5/1.10.6 +module load netcdf/4.7.4 + +export FCOMP="ifort -march=ivybridge" diff --git a/modulefiles/gfs_bufr.s4 b/modulefiles/gfs_bufr.s4 new file mode 100644 index 0000000000..f1e791260d --- /dev/null +++ b/modulefiles/gfs_bufr.s4 @@ -0,0 +1,29 @@ +#%Module##################################################### +## bufrsnd +############################################################# + +# Loading Intel Compiler Suite +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +# Loading nceplibs modules +module load sigio/2.3.2 +module load bacio/2.4.1 +module load w3nco/2.4.1 +module load bufr/11.4.0 +module load nemsio/2.5.2 +module load miniconda/3.7-base +module load w3emc/2.7.3 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 + + +export myFC=mpiifort +export myFCFLAGS="-march=ivybridge -O3 -convert big_endian -traceback -g -fp-model source -qopenmp" +export myCPP=/lib/cpp +export myCPPFLAGS="-P" +export NETCDF_LIB=$NETCDF_LIBRARIES +export NETCDF_INC=$NETCDF_INCLUDES +export HDF5_LIB=$HDF5_LIBRARIES +export HDF5_INC=$HDF5_INCLUDES diff --git a/modulefiles/gfs_fbwndgfs.s4 b/modulefiles/gfs_fbwndgfs.s4 new file mode 100644 index 0000000000..6d8d50fe75 --- /dev/null +++ b/modulefiles/gfs_fbwndgfs.s4 @@ -0,0 +1,14 @@ +#%Module##################################################### +## Module file for fbwndgfs +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load bacio/2.4.1 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load ip/3.3.3 +module load sp/2.3.3 + +export FFLAGS="-march=ivybridge" diff --git a/modulefiles/module_base.s4 b/modulefiles/module_base.s4 new file mode 100644 index 0000000000..7c5882b63d --- /dev/null +++ b/modulefiles/module_base.s4 @@ -0,0 +1,23 @@ +#%Module###################################################################### +## +## FV3GFS prerequisites +## + +module load license_intel/S4 +module load intel/18.0.3 +module load gempak/7.5.1 + +#Load from official NCEPLIBS +module load emc-hpc-stack/2020-q3 +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load wgrib2/2.0.8 +module load g2tmpl/1.9.1 +module load grib_util/1.1.1 +module load crtm/2.3.0 +module load prod_util/1.1.0 + +module load esmf/8_1_0_beta_snapshot_27 + +# python +module load miniconda/3.7-base diff --git a/modulefiles/modulefile.fv3nc2nemsio.s4 b/modulefiles/modulefile.fv3nc2nemsio.s4 new file mode 100644 index 0000000000..4a51da6123 --- /dev/null +++ b/modulefiles/modulefile.fv3nc2nemsio.s4 @@ -0,0 +1,15 @@ +#%Module##################################################### +## Module file for fv3nc2nemsio +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load bacio/2.4.1 +module load nemsio/2.5.2 +module load w3nco/2.4.1 + +export FCMP="ifort" +export FFLAGS="-march=ivybridge -g -O2 -traceback" diff --git a/modulefiles/modulefile.grib_util.s4 b/modulefiles/modulefile.grib_util.s4 new file mode 100644 index 0000000000..fad5a5cf18 --- /dev/null +++ b/modulefiles/modulefile.grib_util.s4 @@ -0,0 +1,19 @@ +#%Module###################################################################### +# Module file for grib_util +############################################################################## +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load bacio/2.4.1 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load ip/3.3.3 +module load sp/2.3.3 + +module load jasper/2.0.15 +module load png/1.6.35 +module load zlib/1.2.11 + +export FCMP=ifort +export CCMP=icc + diff --git a/modulefiles/modulefile.regrid_nemsio.s4 b/modulefiles/modulefile.regrid_nemsio.s4 new file mode 100644 index 0000000000..5d7d63ba76 --- /dev/null +++ b/modulefiles/modulefile.regrid_nemsio.s4 @@ -0,0 +1,19 @@ +#%Module##################################################### +## Module file for regrid_nemsio +############################################################# + +module load license_intel/S4 +module load intel/18.0.3 +module load emc-hpc-stack/2020-q3 +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load bacio/2.4.1 +module load nemsio/2.5.2 +module load w3nco/2.4.1 +module load sp/2.3.3 + +export FCMP="mpiifort" +export OPTIMIZATION="-O3 -march=ivybridge" +export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz" +export NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz" +export NETCDF_INCLUDE=-I$NETCDF/include diff --git a/modulefiles/modulefile.storm_reloc_v6.0.0.s4 b/modulefiles/modulefile.storm_reloc_v6.0.0.s4 new file mode 100644 index 0000000000..217b5ca0ea --- /dev/null +++ b/modulefiles/modulefile.storm_reloc_v6.0.0.s4 @@ -0,0 +1,19 @@ +#%Module##################################################### +## Module file for storm_reloc_v6.0.0 - S4 +############################################################# + +module load emc-hpc-stack/2020-q3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load bacio/2.4.1 +module load sp/2.3.3 +module load nemsio/2.5.2 +module load nemsiogfs/2.5.3 +module load sigio/2.3.2 +module load jasper/2.0.15 +module load png/1.6.35 +module load zlib/1.2.11 +module load g2/3.4.1 + +export FC="ifort" +export MyFFLAGS="-march=ivybridge" diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh index 5659b53316..2b62254962 100755 --- a/sorc/build_fv3.sh +++ b/sorc/build_fv3.sh @@ -17,6 +17,7 @@ if [ ! -d "../exec" ]; then fi if [ $target = hera ]; then target=hera.intel ; fi +if [ $target = s4 ]; then target=s4.intel ; fi cd fv3gfs.fd/ FV3=$( pwd -P )/FV3 diff --git a/sorc/checkout.sh b/sorc/checkout.sh index a8692e8e50..8d58a2aa77 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -24,9 +24,9 @@ echo $topdir echo fv3gfs checkout ... if [[ ! -d fv3gfs.fd ]] ; then rm -f ${topdir}/checkout-fv3gfs.log - git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 cd fv3gfs.fd - git checkout GFS.v16.0.13 + git checkout GFS.v16.0.14_port2s4 git submodule update --init --recursive cd ${topdir} else @@ -36,9 +36,9 @@ fi echo gsi checkout ... if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log - git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 + git clone --recursive https://github.com/DavidHuber-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 cd gsi.fd - git checkout gfsda.v16.0.0 + git checkout gfsda.v16.0.0_port2s4 git submodule update cd ${topdir} else @@ -48,9 +48,9 @@ fi echo gldas checkout ... if [[ ! -d gldas.fd ]] ; then rm -f ${topdir}/checkout-gldas.log - git clone https://github.com/NOAA-EMC/GLDAS gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/GLDAS gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1 cd gldas.fd - git checkout gldas_gfsv16_release.v1.11.0 + git checkout gldas_gfsv16_release.v1.11.0_port2s4 cd ${topdir} else echo 'Skip. Directory gldas.fd already exists.' @@ -59,9 +59,9 @@ fi echo ufs_utils checkout ... if [[ ! -d ufs_utils.fd ]] ; then rm -f ${topdir}/checkout-ufs_utils.log - git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 cd ufs_utils.fd - git checkout ops-gfsv16.0.0 + git checkout ops-gfsv16.0.0_port2s4 cd ${topdir} else echo 'Skip. Directory ufs_utils.fd already exists.' @@ -70,9 +70,9 @@ fi echo EMC_post checkout ... if [[ ! -d gfs_post.fd ]] ; then rm -f ${topdir}/checkout-gfs_post.log - git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1 cd gfs_post.fd - git checkout upp_gfsv16_release.v1.1.0 + git checkout upp_gfsv16_release.v1.1.0_port2s4 ################################################################################ # checkout_gtg ## yes: The gtg code at NCAR private repository is available for ops. GFS only. @@ -93,9 +93,9 @@ fi echo EMC_gfs_wafs checkout ... if [[ ! -d gfs_wafs.fd ]] ; then rm -f ${topdir}/checkout-gfs_wafs.log - git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1 + git clone --recursive https://github.com/DavidHuber-NOAA/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1 cd gfs_wafs.fd - git checkout gfs_wafs.v6.0.10 + git checkout gfs_wafs.v6.0.10_port2s4 cd ${topdir} else echo 'Skip. Directory gfs_wafs.fd already exists.' diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 2dbe6abd7e..545de21f73 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -175,6 +175,13 @@ export NCEPLIBS=/mnt/lfs3/projects/hfv3gfs/gwv/NCEPLIBS.15X export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src export myFC=mpiifort +elif [[ -d /data/prod ]] ; then + # We are on SSEC's S4 + target=s4 + module purge + module load license_intel/S4 + module load intel/18.0.3 + else echo WARNING: UNKNOWN PLATFORM 1>&2 fi diff --git a/sorc/supvit.fd/makefile b/sorc/supvit.fd/makefile index 288e42beff..223f368a7c 100644 --- a/sorc/supvit.fd/makefile +++ b/sorc/supvit.fd/makefile @@ -7,7 +7,7 @@ LDFLAGS= ##ccs FFLAGS= -O -qflttrap=ov:zero:inv:enable -qcheck -qextchk -qwarn64 -qintsize=$(ISIZE) -qrealsize=$(RSIZE) # FFLAGS= -O2 -check bounds -check format -xHost -fpe0 # DEBUG= -check bounds -check format -FFLAGS= -O2 -g -i$(ISIZE) -r$(RSIZE) +FFLAGS= $(MyFFLAGS) -O2 -g -i$(ISIZE) -r$(RSIZE) supvit: supvit_main.f supvit_modules.o @echo " " diff --git a/sorc/syndat_getjtbul.fd/makefile b/sorc/syndat_getjtbul.fd/makefile index 3ac5730f31..cbce63a316 100755 --- a/sorc/syndat_getjtbul.fd/makefile +++ b/sorc/syndat_getjtbul.fd/makefile @@ -4,7 +4,7 @@ SHELL= /bin/sh FC= ifort #DEBUG = -ftrapuv -check all -fp-stack-check -fstack-protector ##DEBUG = -ftrapuv -fp-stack-check -fstack-protector -FFLAGS= -O3 -g -traceback -assume noold_ldout_format $(DEBUG) +FFLAGS= $(MyFFLAGS) -O3 -g -traceback -assume noold_ldout_format $(DEBUG) LDFLAGS= SRCS= getjtbul.f OBJS= getjtbul.o diff --git a/sorc/syndat_maksynrc.fd/makefile b/sorc/syndat_maksynrc.fd/makefile index 9adcb17e26..9daaec9736 100755 --- a/sorc/syndat_maksynrc.fd/makefile +++ b/sorc/syndat_maksynrc.fd/makefile @@ -3,7 +3,7 @@ SHELL= /bin/sh ##LIBS_SYN_MAK= -L/contrib/nceplibs/nwprod/lib -lw3nco_v2.0.5_4 -lbacio_v2.0.1_4 FC= ifort #DEBUG = -ftrapuv -check all -check nooutput_conversion -fp-stack-check -fstack-protector -FFLAGS= -O3 -g -traceback -assume noold_ldout_format $(DEBUG) +FFLAGS= $(MyFFLAGS) -O3 -g -traceback -assume noold_ldout_format $(DEBUG) LDFLAGS= SRCS= maksynrc.f OBJS= maksynrc.o diff --git a/sorc/syndat_qctropcy.fd/makefile b/sorc/syndat_qctropcy.fd/makefile index d667c26cbe..3f6428b0dc 100755 --- a/sorc/syndat_qctropcy.fd/makefile +++ b/sorc/syndat_qctropcy.fd/makefile @@ -5,7 +5,7 @@ FC= ifort #DEBUG = -ftrapuv -check all -check noarg_temp_created -fp-stack-check -fstack-protector ## if '-check all' enabled, include '-check noarg_temp_created' to avoid warning msgs indicating ## slight performance hit due to chosen method of passing array arguments to w3difdat -FFLAGS= -O3 -g -traceback -r8 -i8 -assume byterecl -assume noold_ldout_format $(DEBUG) +FFLAGS= $(MyFFLAGS) -O3 -g -traceback -r8 -i8 -assume byterecl -assume noold_ldout_format $(DEBUG) LDFLAGS= SRCS= qctropcy.f OBJS= qctropcy.o diff --git a/sorc/tave.fd/makefile b/sorc/tave.fd/makefile index 3ccaf4b87b..0f3be41b63 100755 --- a/sorc/tave.fd/makefile +++ b/sorc/tave.fd/makefile @@ -6,7 +6,7 @@ COMP= ifort ##LIBS= -L/contrib/nceplibs/nwprod/lib -lw3emc_d -lw3nco_d -lg2_d -lbacio_4 -ljasper -lpng -lz LDFLAGS= # DEBUG= -check all -debug all -traceback -FFLAGS= -O2 -g -traceback -I $(INC) -i$(ISIZE) -r$(RSIZE) +FFLAGS= $(MyFFLAGS) -O2 -g -traceback -I $(INC) -i$(ISIZE) -r$(RSIZE) # FFLAGS= -O3 -I $(INC) -i$(ISIZE) -r$(RSIZE) tave: tave.f diff --git a/sorc/vint.fd/makefile b/sorc/vint.fd/makefile index 06647d1fc6..5c0906f593 100755 --- a/sorc/vint.fd/makefile +++ b/sorc/vint.fd/makefile @@ -7,7 +7,7 @@ COMP= ifort LDFLAGS= # FFLAGS= -O3 -I $(INC) -i$(ISIZE) -r$(RSIZE) # DEBUG= -check all -debug all -traceback -FFLAGS= -O2 -g -traceback -I $(INC) -i$(ISIZE) -r$(RSIZE) +FFLAGS= $(MyFFLAGS) -O2 -g -traceback -I $(INC) -i$(ISIZE) -r$(RSIZE) vint: vint.f @echo " "