NOAA-EMC#138 Modifications to build on S4.

DavidHuber-NOAA · Oct 26, 2020 · 389c468 · 389c468
1 parent 58ee9f6
commit 389c468
Show file tree

Hide file tree

Showing 20 changed files with 221 additions and 30 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -1,37 +1,37 @@
 # External sub-modules of global-workflow
 
 [FV3GFS]
-tag = GFS.v16.0.13
+tag = GFS.v16.0.14_port2s4
 local_path = sorc/fv3gfs.fd
-repo_url = https://github.com/ufs-community/ufs-weather-model.git
+repo_url = https://github.com/DavidHuber-NOAA/ufs-weather-model.git
 protocol = git
 required = True
 
 [GSI]
-tag = gfsda.v16.0.0
+tag = gfsda.v16.0.0_port2s4
 local_path = sorc/gsi.fd
-repo_url = https://github.com/NOAA-EMC/GSI.git
+repo_url = https://github.com/DavidHuber-NOAA/GSI.git
 protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.11.0
+tag = gldas_gfsv16_release.v1.11.0_port2s4
 local_path = sorc/gldas.fd
-repo_url = https://github.com/NOAA-EMC/GLDAS.git
+repo_url = https://github.com/DavidHuber-NOAA/GLDAS.git
 protocol = git
 required = True
 
 [EMC_post]
-tag = upp_gfsv16_release.v1.1.0
+tag = upp_gfsv16_release.v1.1.0_port2s4
 local_path = sorc/gfs_post.fd
-repo_url = https://github.com/NOAA-EMC/EMC_post.git
+repo_url = https://github.com/DavidHuber-NOAA/EMC_post.git
 protocol = git
 required = True
 
 [UFS_UTILS]
-tag = ops-gfsv16.0.0
+tag = ops-gfsv16.0.0_port2s4
 local_path = sorc/ufs_utils.fd
-repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
+repo_url = https://github.com/DavidHuber-NOAA/UFS_UTILS.git
 protocol = git
 required = True
 
@@ -43,9 +43,9 @@ protocol = git
 required = True
 
 [EMC_gfs_wafs]
-tag = gfs_wafs.v6.0.10
+tag = gfs_wafs.v6.0.10_port2s4
 local_path = sorc/gfs_wafs.fd
-repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git
+repo_url = https://github.com/DavidHuber-NOAA/EMC_gfs_wafs.git
 protocol = git
 required = False
 

diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter.s4 b/modulefiles/fv3gfs/enkf_chgres_recenter.s4
@@ -0,0 +1,14 @@
+#%Module#####################################################
+## enkf_chgres_recenter component - s4
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load nemsio/2.5.2
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+
+export FC="ifort -march=ivybridge"
diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter_nc.s4 b/modulefiles/fv3gfs/enkf_chgres_recenter_nc.s4
@@ -0,0 +1,16 @@
+#%Module#####################################################
+## enkf_chgres_recenter component - hera
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+
+
+export FC="ifort -march=ivybridge"
diff --git a/modulefiles/fv3gfs/gaussian_sfcanl.s4 b/modulefiles/fv3gfs/gaussian_sfcanl.s4
@@ -0,0 +1,15 @@
+#%Module#####################################################
+## gaussian_sfcanl build module for S4
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load w3nco/2.4.1
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+
+export FCOMP="ifort -march=ivybridge"
diff --git a/modulefiles/gfs_bufr.s4 b/modulefiles/gfs_bufr.s4
@@ -0,0 +1,29 @@
+#%Module#####################################################
+## bufrsnd
+#############################################################
+
+# Loading Intel Compiler Suite
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+# Loading nceplibs modules
+module load sigio/2.3.2
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load bufr/11.4.0
+module load nemsio/2.5.2
+module load miniconda/3.7-base
+module load w3emc/2.7.3
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+
+
+export myFC=mpiifort
+export myFCFLAGS="-march=ivybridge -O3 -convert big_endian -traceback -g -fp-model source -qopenmp"
+export myCPP=/lib/cpp
+export myCPPFLAGS="-P"
+export NETCDF_LIB=$NETCDF_LIBRARIES
+export NETCDF_INC=$NETCDF_INCLUDES
+export HDF5_LIB=$HDF5_LIBRARIES
+export HDF5_INC=$HDF5_INCLUDES
diff --git a/modulefiles/gfs_fbwndgfs.s4 b/modulefiles/gfs_fbwndgfs.s4
@@ -0,0 +1,14 @@
+#%Module#####################################################
+## Module file for fbwndgfs 
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load bacio/2.4.1
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+
+export FFLAGS="-march=ivybridge"
diff --git a/modulefiles/module_base.s4 b/modulefiles/module_base.s4
@@ -0,0 +1,23 @@
+#%Module######################################################################
+##
+##      FV3GFS prerequisites
+##
+
+module load license_intel/S4
+module load intel/18.0.3
+module load gempak/7.5.1
+
+#Load from official NCEPLIBS
+module load emc-hpc-stack/2020-q3
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load wgrib2/2.0.8
+module load g2tmpl/1.9.1
+module load grib_util/1.1.1
+module load crtm/2.3.0
+module load prod_util/1.1.0
+
+module load  esmf/8_1_0_beta_snapshot_27
+
+# python
+module load miniconda/3.7-base
diff --git a/modulefiles/modulefile.fv3nc2nemsio.s4 b/modulefiles/modulefile.fv3nc2nemsio.s4
@@ -0,0 +1,15 @@
+#%Module#####################################################
+## Module file for fv3nc2nemsio
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load w3nco/2.4.1
+
+export FCMP="ifort"
+export FFLAGS="-march=ivybridge -g -O2 -traceback"
diff --git a/modulefiles/modulefile.grib_util.s4 b/modulefiles/modulefile.grib_util.s4
@@ -0,0 +1,19 @@
+#%Module######################################################################
+#  Module file for grib_util
+##############################################################################
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load bacio/2.4.1
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+
+module load jasper/2.0.15
+module load png/1.6.35
+module load zlib/1.2.11
+
+export FCMP=ifort
+export CCMP=icc
+
diff --git a/modulefiles/modulefile.regrid_nemsio.s4 b/modulefiles/modulefile.regrid_nemsio.s4
@@ -0,0 +1,19 @@
+#%Module#####################################################
+## Module file for regrid_nemsio
+#############################################################
+
+module load license_intel/S4
+module load intel/18.0.3
+module load emc-hpc-stack/2020-q3
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load w3nco/2.4.1
+module load sp/2.3.3
+
+export FCMP="mpiifort"
+export OPTIMIZATION="-O3 -march=ivybridge"
+export  NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf  -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz"
+export  NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf  -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz"
+export NETCDF_INCLUDE=-I$NETCDF/include
diff --git a/modulefiles/modulefile.storm_reloc_v6.0.0.s4 b/modulefiles/modulefile.storm_reloc_v6.0.0.s4
@@ -0,0 +1,19 @@
+#%Module#####################################################
+## Module file for storm_reloc_v6.0.0 - S4
+#############################################################
+
+module load emc-hpc-stack/2020-q3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load bacio/2.4.1
+module load sp/2.3.3
+module load nemsio/2.5.2
+module load nemsiogfs/2.5.3
+module load sigio/2.3.2
+module load jasper/2.0.15
+module load png/1.6.35
+module load zlib/1.2.11
+module load g2/3.4.1
+
+export FC="ifort"
+export MyFFLAGS="-march=ivybridge"
diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh
@@ -17,6 +17,7 @@ if [ ! -d "../exec" ]; then
 fi
 
 if [ $target = hera ]; then target=hera.intel ; fi
+if [ $target = s4 ]; then target=s4.intel ; fi
 
 cd fv3gfs.fd/
 FV3=$( pwd -P )/FV3

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
@@ -24,9 +24,9 @@ echo $topdir
 echo fv3gfs checkout ...
 if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
-    git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
+    git clone https://github.com/DavidHuber-NOAA/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    git checkout GFS.v16.0.13
+    git checkout GFS.v16.0.14_port2s4
     git submodule update --init --recursive
     cd ${topdir}
 else
@@ -36,9 +36,9 @@ fi
 echo gsi checkout ...
 if [[ ! -d gsi.fd ]] ; then
     rm -f ${topdir}/checkout-gsi.log
-    git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
+    git clone --recursive https://github.com/DavidHuber-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
     cd gsi.fd
-    git checkout gfsda.v16.0.0
+    git checkout gfsda.v16.0.0_port2s4
     git submodule update
     cd ${topdir}
 else
@@ -48,9 +48,9 @@ fi
 echo gldas checkout ...
 if [[ ! -d gldas.fd ]] ; then
     rm -f ${topdir}/checkout-gldas.log
-    git clone https://github.com/NOAA-EMC/GLDAS  gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1
+    git clone https://github.com/DavidHuber-NOAA/GLDAS  gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1
     cd gldas.fd
-    git checkout gldas_gfsv16_release.v1.11.0
+    git checkout gldas_gfsv16_release.v1.11.0_port2s4
     cd ${topdir}
 else
     echo 'Skip.  Directory gldas.fd already exists.'
@@ -59,9 +59,9 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone https://github.com/DavidHuber-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    git checkout ops-gfsv16.0.0
+    git checkout ops-gfsv16.0.0_port2s4
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'
@@ -70,9 +70,9 @@ fi
 echo EMC_post checkout ...
 if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
-    git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
+    git clone https://github.com/DavidHuber-NOAA/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout upp_gfsv16_release.v1.1.0
+    git checkout upp_gfsv16_release.v1.1.0_port2s4
     ################################################################################
     # checkout_gtg
     ## yes: The gtg code at NCAR private repository is available for ops. GFS only.
@@ -93,9 +93,9 @@ fi
 echo EMC_gfs_wafs checkout ...
 if [[ ! -d gfs_wafs.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_wafs.log
-    git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1
+    git clone --recursive https://github.com/DavidHuber-NOAA/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1
     cd gfs_wafs.fd
-    git checkout gfs_wafs.v6.0.10
+    git checkout gfs_wafs.v6.0.10_port2s4
     cd ${topdir}
 else
     echo 'Skip.  Directory gfs_wafs.fd already exists.'

diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
@@ -175,6 +175,13 @@ export NCEPLIBS=/mnt/lfs3/projects/hfv3gfs/gwv/NCEPLIBS.15X
 export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src
 export myFC=mpiifort
 
+elif [[ -d /data/prod ]] ; then
+    # We are on SSEC's S4
+    target=s4
+    module purge
+    module load license_intel/S4
+    module load intel/18.0.3
+
 else
     echo WARNING: UNKNOWN PLATFORM 1>&2
 fi

diff --git a/sorc/supvit.fd/makefile b/sorc/supvit.fd/makefile
@@ -7,7 +7,7 @@ LDFLAGS=
 ##ccs FFLAGS= -O -qflttrap=ov:zero:inv:enable -qcheck -qextchk -qwarn64 -qintsize=$(ISIZE) -qrealsize=$(RSIZE)
 # FFLAGS= -O2 -check bounds -check format -xHost -fpe0
 # DEBUG= -check bounds -check format
-FFLAGS= -O2 -g -i$(ISIZE) -r$(RSIZE)
+FFLAGS= $(MyFFLAGS) -O2 -g -i$(ISIZE) -r$(RSIZE)
 
 supvit:     supvit_main.f supvit_modules.o
 	@echo " "

diff --git a/sorc/syndat_getjtbul.fd/makefile b/sorc/syndat_getjtbul.fd/makefile
@@ -4,7 +4,7 @@ SHELL=		/bin/sh
 FC=		ifort   
 #DEBUG = 	-ftrapuv  -check all  -fp-stack-check  -fstack-protector
 ##DEBUG = 	-ftrapuv  -fp-stack-check  -fstack-protector
-FFLAGS=		-O3 -g -traceback -assume noold_ldout_format $(DEBUG)
+FFLAGS=		$(MyFFLAGS) -O3 -g -traceback -assume noold_ldout_format $(DEBUG)
 LDFLAGS=	
 SRCS=		getjtbul.f
 OBJS=		getjtbul.o

diff --git a/sorc/syndat_maksynrc.fd/makefile b/sorc/syndat_maksynrc.fd/makefile
@@ -3,7 +3,7 @@ SHELL=		/bin/sh
 ##LIBS_SYN_MAK=		-L/contrib/nceplibs/nwprod/lib -lw3nco_v2.0.5_4 -lbacio_v2.0.1_4 
 FC=		ifort   
 #DEBUG =          -ftrapuv -check all -check nooutput_conversion -fp-stack-check -fstack-protector
-FFLAGS=		-O3 -g -traceback -assume noold_ldout_format $(DEBUG)
+FFLAGS=		$(MyFFLAGS) -O3 -g -traceback -assume noold_ldout_format $(DEBUG)
 LDFLAGS=	
 SRCS=		maksynrc.f
 OBJS=		maksynrc.o

diff --git a/sorc/syndat_qctropcy.fd/makefile b/sorc/syndat_qctropcy.fd/makefile
@@ -5,7 +5,7 @@ FC=		ifort
 #DEBUG =		-ftrapuv -check all -check noarg_temp_created -fp-stack-check -fstack-protector
 ## if '-check all' enabled, include '-check noarg_temp_created' to avoid warning msgs indicating 
 ##   slight performance hit due to chosen method of passing array arguments to w3difdat  
-FFLAGS=		-O3 -g -traceback -r8 -i8 -assume byterecl -assume noold_ldout_format $(DEBUG)
+FFLAGS=		$(MyFFLAGS) -O3 -g -traceback -r8 -i8 -assume byterecl -assume noold_ldout_format $(DEBUG)
 LDFLAGS=	
 SRCS=		qctropcy.f
 OBJS=		qctropcy.o

diff --git a/sorc/tave.fd/makefile b/sorc/tave.fd/makefile
@@ -6,7 +6,7 @@ COMP=   ifort
 ##LIBS=    -L/contrib/nceplibs/nwprod/lib -lw3emc_d -lw3nco_d -lg2_d -lbacio_4 -ljasper -lpng -lz
 LDFLAGS= 
 # DEBUG= -check all -debug all -traceback
-FFLAGS= -O2 -g -traceback -I $(INC)  -i$(ISIZE) -r$(RSIZE)
+FFLAGS= $(MyFFLAGS) -O2 -g -traceback -I $(INC)  -i$(ISIZE) -r$(RSIZE)
 # FFLAGS= -O3 -I $(INC)  -i$(ISIZE) -r$(RSIZE) 
 
 tave:      tave.f