Github Issue #16

1. add options to workflow scripts to use parallel netcdf for I/O. The application varies with model resolution and computing platform. See config.fcst and exglobal_fcst_nemsfv3gfs.sh for the settings. 2. Update UPP to upp_gfsv16_release.v1.0.1 modified: modulefiles/module_base.hera modified: modulefiles/module_base.wcoss_dell_p3 modified: parm/config/config.fcst modified: scripts/exglobal_fcst_nemsfv3gfs.sh modified: sorc/checkout.sh deleted: parm/config/config.resources.C96
NOAA-EMC · Feb 2, 2020 · 9d5e0a0 · 9d5e0a0
1 parent 3a66788
commit 9d5e0a0
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 228 deletions.
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
@@ -7,22 +7,27 @@ module load intel/18.0.5.274
 module load impi/2018.0.4
 module load wgrib2/2.0.8
 module load hpss/hpss
-module load pnetcdf/1.10.0
-module load netcdf/4.7.0
 module load nco/4.7.0
-module load hdf5/1.10.5
 module load gempak/7.4.2
 
+#module load pnetcdf/1.10.0
+#module load netcdf/4.7.0
+#module load hdf5/1.10.5
+module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
+module load hdf5_parallel/1.10.6
+module load netcdf_parallel/4.7.4
+module load esmf/8.0.0_ParallelNetCDF
+
+#module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
+#module load esmf/8.0.0bs48
+
 #Load from official NCEPLIBS
 module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
 module load g2tmpl/1.6.0
 module load grib_util/1.1.1
 module load crtm/2.2.6
 module load prod_util/1.1.0
 
-module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0bs48
-
 # python
 module use -a /contrib/modulefiles
 module load anaconda/2.3.0
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
@@ -17,19 +17,22 @@ module unload grib_util/1.0.6
 module load grib_util/1.1.0
 
 module load NCO/4.7.0   
-module load HDF5-serial/1.10.1
-module load NetCDF/4.5.0
 module load CFP/2.0.1
 setenv USE_CFP YES
-
-# modules used by vrfy
 module load pm5
 
+#module load HDF5-serial/1.10.1
+#module load NetCDF/4.5.0
+module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles
+module load hdf5_parallel/1.10.6
+module load netcdf_parallel/4.7.4  
+module load esmf/8.0.0_ParallelNetCDF
+
 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
 module load gempak/7.3.1
 
-module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0bs48
+#module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles
+#module load esmf/8.0.0bs48
 
 # python
 module load python/3.6.3

diff --git a/parm/config/config.fcst b/parm/config/config.fcst
@@ -138,6 +138,25 @@ export ishuffle=0
 export shuffle=1
 export deflate_level=1
 
+if [ "$OUTPUT_FILE" = "netcdf" ]; then
+    export OUTPUT_FILETYPES="netcdf"
+    if [ $machine = "WCOSS_DELL_P3" ]; then
+        if [ $(echo $CASE |cut -c 2-) -ge 768 ]; then
+            export OUTPUT_FILETYPES=" 'netcdf_parallel' 'netcdf_parallel' "
+            export ichunk3d=$((4*RESTILE)) 
+            export jchunk3d=$((2*RESTILE))
+            export kchunk3d=1
+        else
+            export OUTPUT_FILETYPES=" 'netcdf_parallel' 'netcdf' "
+        fi
+    fi
+    if [ $machine = "HERA" ]; then
+        export OUTPUT_FILETYPES=" 'netcdf_parallel' 'netcdf_parallel' "
+    fi
+fi
+
+
+#---------------------------------------------------------------------
 # Disable the use of coupler.res; get model start time from model_configure
 export USE_COUPLER_RES="NO"
 

diff --git a/parm/config/config.resources.C96 b/parm/config/config.resources.C96
diff --git a/scripts/exglobal_fcst_nemsfv3gfs.sh b/scripts/exglobal_fcst_nemsfv3gfs.sh
@@ -651,7 +651,12 @@ write_dopost:            ${WRITE_DOPOST:-".false."}
 num_files:               ${NUM_FILES:-2}
 filename_base:           'atm' 'sfc'
 output_grid:             $OUTPUT_GRID
-output_file:             $OUTPUT_FILE
+output_file:             $OUTPUT_FILETYPES
+ichunk2d:                ${ichunk2d:-0}
+jchunk2d:                ${jchunk2d:-0}
+ichunk3d:                ${ichunk3d:-0}
+jchunk3d:                ${jchunk3d:-0}
+kchunk3d:                ${kchunk3d:-0}
 ideflate:                ${ideflate:-1}
 nbits:                   ${nbits:-14}
 write_nemsioflip:        $WRITE_NEMSIOFLIP

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
@@ -10,14 +10,8 @@ if [[ ! -d fv3gfs.fd ]] ; then
     #git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     git clone https://github.com/junwang-noaa/ufs-weather-model  fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    git checkout gfsv16_updates
+    git checkout netcdf_parallel
     git submodule update --init --recursive
-
-    #git clone gerrit:NEMSfv3gfs fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
-    #cd fv3gfs.fd
-    ##git checkout gfs.v16_PhysicsUpdate
-    #git submodule update --init --recursive
-
     cd ${topdir}
 else
     echo 'Skip.  Directory fv3gfs.fd already exists.'
@@ -60,7 +54,7 @@ if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
     git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout upp_gfsv16_release.v1.0.0
+    git checkout upp_gfsv16_release.v1.0.1
     cd ${topdir}
 else
     echo 'Skip.  Directory gfs_post.fd already exists.'