Merge pull request NOAA-EMC#10 from DavidHuber-NOAA/intel2022_dh

Update Cheyenne and Gaea module files, merge develop, fix ncdiag bug, create runtime directories on one PE
RussTreadon-NOAA · Jun 22, 2023 · 06772e5 · 06772e5
2 parents 9434673 + e5d89c3
commit 06772e5
Show file tree

Hide file tree

Showing 15 changed files with 65 additions and 109 deletions.
diff --git a/fix b/fix
diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua
@@ -9,7 +9,7 @@ load("mpt/2.22")
 load("ncarcompilers/0.5.0")
 unload("netcdf")
 
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/gnu10.1.0/modulefiles/stack")
 
 load("hpc/1.2.0")
 load("hpc-gnu/10.1.0")

diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua
@@ -17,8 +17,8 @@ load("mkl/2022.1")
 
 load("gsi_common")
 
-local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
+pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix")
 
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")

diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua
@@ -7,7 +7,6 @@ local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
 local bufr_ver=os.getenv("bufr_ver") or "11.7.0"
 local bacio_ver=os.getenv("bacio_ver") or "2.4.1"
 local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
-local w3nco_ver=os.getenv("w3nco_ver") or "2.4.1"
 local sp_ver=os.getenv("sp_ver") or "2.3.3"
 local ip_ver=os.getenv("ip_ver") or "3.3.3"
 local sigio_ver=os.getenv("sigio_ver") or "2.3.2"
@@ -32,4 +31,7 @@ load(pathJoin("nemsio", nemsio_ver))
 load(pathJoin("wrf_io", wrf_io_ver))
 load(pathJoin("ncio", ncio_ver))
 load(pathJoin("crtm", crtm_ver))
-load(pathJoin("ncdiag",ncdiag_ver))
+--load(pathJoin("ncdiag",ncdiag_ver))
+
+setenv("ncdiag_ROOT", "/scratch1/NESDIS/nesdis-rdo2/David.Huber/GSI/ncdiag/1.1.1")
+setenv("ncdiag_VERSION", "1.1.1")
diff --git a/modulefiles/gsi_gaea b/modulefiles/gsi_gaea
diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua
@@ -0,0 +1,29 @@
+help([[
+]])
+
+load("cmake/3.20.1")
+
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+
+load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2"))
+load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20"))
+load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))
+
+load("gsi_common")
+
+local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
+load(pathJoin("prod_util", prod_util_ver))
+
+local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
+prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
+pushenv("MKLROOT", MKLROOT)
+
+pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix")
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+pushenv("CRAYPE_LINK_TYPE","dynamic")
+
+whatis("Description: GSI environment on Gaea with Intel Compilers")
diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua
@@ -20,6 +20,6 @@ load(pathJoin("prod_util", prod_util_ver))
 
 pushenv("MKLROOT", "/apps/oneapi/mkl/2022.0.2")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Hera with GNU Compilers")
diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua
@@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Hera with Intel Compilers")
diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua
@@ -26,6 +26,6 @@ pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2")
 pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2")
 
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Jet with Intel Compilers")
diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua
@@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Orion with Intel Compilers")
diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua
@@ -23,6 +23,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-march=ivybridge")
 pushenv("FFLAGS", "-march=ivybridge")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on S4 with Intel Compilers")
diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua
@@ -20,6 +20,6 @@ load(pathJoin("prod_util", prod_util_ver))
 
 load("gsi_common")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230601")
 
 whatis("Description: GSI environment on WCOSS2")
diff --git a/src/gsi/gsimod.F90 b/src/gsi/gsimod.F90
@@ -2213,7 +2213,7 @@ subroutine gsimain_initialize
   endif
 
 ! Set up directories (or pe specific filenames)
-  call init_directories(mype)
+  call init_directories(mype,npe)
 
 ! Initialize space for qc
   call create_qcvars

diff --git a/src/gsi/obsmod.F90 b/src/gsi/obsmod.F90
@@ -981,7 +981,7 @@ subroutine init_obsmod_dflts
     return
   end subroutine init_obsmod_dflts
 
-  subroutine init_directories(in_pe)
+  subroutine init_directories(in_pe,num_pe)
 !$$$  subprogram documentation block
 !                .      .    .                                       .
 ! subprogram:    create sub-directories
@@ -1012,24 +1012,34 @@ subroutine init_directories(in_pe)
     implicit none
 
     integer(i_kind),intent(in   ) :: in_pe
+    integer(i_kind),intent(in   ) :: num_pe
     logical :: l_mkdir_stat
 
     character(len=144):: command
-    character(len=8):: pe_name
+    character(len=8):: pe_name, loc_pe_name
+    character(len=128):: loc_dirname
+    integer(i_kind) :: i
 
     if (lrun_subdirs) then
        write(pe_name,'(i4.4)') in_pe
        dirname = 'dir.'//trim(pe_name)//'/'
-       command = 'mkdir -p -m 755 ' // trim(dirname)
+! Only create directories on one PE
+       if(in_pe == 0) then
+          do i = 0, num_pe
+             write(loc_pe_name,'(i4.4)') i
+             loc_dirname = 'dir.'//trim(loc_pe_name)
 #ifdef __INTEL_COMPILER
-       l_mkdir_stat = MAKEDIRQQ(trim(dirname))
-       if(.not. l_mkdir_stat) then
-          write(6, *) "Failed to create directory ", trim(dirname), " for PE ", pe_name
-          call stop2(678)
-       endif
+             l_mkdir_stat = MAKEDIRQQ(trim(loc_dirname))
+             if(.not. l_mkdir_stat) then
+                write(6, *) "Failed to create directory ", trim(loc_dirname), " for PE ", loc_pe_name
+                call stop2(678)
+             endif
 #else
-       call system(command)
+             command = 'mkdir -p -m 755 ' // trim(loc_dirname)
+             call system(command)
 #endif
+          enddo
+       endif
     else
        write(pe_name,100) in_pe
 100 format('pe',i4.4,'.')

diff --git a/ush/module-setup.sh b/ush/module-setup.sh
@@ -57,33 +57,10 @@ elif [[ $MACHINE_ID = gaea* ]] ; then
         # /etc/profile here.
         source /etc/profile
         __ms_source_etc_profile=yes
-    else
-        __ms_source_etc_profile=no
-    fi
-    module purge
-    # clean up after purge
-    unset _LMFILES_
-    unset _LMFILES_000
-    unset _LMFILES_001
-    unset LOADEDMODULES
-    module load modules
-    if [[ -d /opt/cray/ari/modulefiles ]] ; then
-        module use -a /opt/cray/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then
-        module use -a /opt/cray/pe/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then
-        module use -a /opt/cray/pe/craype/default/modulefiles
-    fi
-    if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then
-        source /etc/opt/cray/pe/admin-pe/site-config
-    fi
-    if [[ "$__ms_source_etc_profile" == yes ]] ; then
-        source /etc/profile
-        unset __ms_source_etc_profile
     fi
 
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
+
 elif [[ $MACHINE_ID = expanse* ]]; then
     # We are on SDSC Expanse
     if ( ! eval module help > /dev/null 2>&1 ) ; then