Merge pull request NOAA-EMC#29 from climbfuji/update_gsd_develop_from…

…_develop_20200616 Update gsd/develop from develop 2020/06/16
zhanglikate · Jun 19, 2020 · 150fc3f · 150fc3f
2 parents df183da + 663d513
commit 150fc3f
Show file tree

Hide file tree

Showing 123 changed files with 27,550 additions and 74,474 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -17,3 +17,4 @@
 [submodule "stochastic_physics"]
 	path = stochastic_physics
 	url = https://github.com/noaa-psd/stochastic_physics
+	branch = master
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,10 @@ add_definitions(-DUSE_COND)
 add_definitions(-DNEW_TAUCTMAX)
 add_definitions(-DINTERNAL_FILE_NML)
 
+if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)        
+  message(FATAL_ERROR "GNU Compiler >= 9 is required")
+endif()
+
 if(CCPP)
 
     find_package(Python 3 QUIET COMPONENTS Interpreter)
@@ -97,89 +101,92 @@ add_library(
     FMS/amip_interp/amip_interp.F90
     FMS/astronomy/astronomy.F90
     FMS/axis_utils/axis_utils.F90
+    FMS/axis_utils/axis_utils2.F90
     FMS/block_control/block_control.F90
     FMS/column_diagnostics/column_diagnostics.F90
     FMS/constants/constants.F90
     FMS/coupler/atmos_ocean_fluxes.F90
     FMS/coupler/coupler_types.F90
     FMS/coupler/ensemble_manager.F90
     FMS/data_override/data_override.F90
+    FMS/diag_integral/diag_integral.F90
     FMS/diag_manager/diag_axis.F90
     FMS/diag_manager/diag_data.F90
     FMS/diag_manager/diag_grid.F90
     FMS/diag_manager/diag_manager.F90
-    FMS/diag_manager/diag_manifest.F90
     FMS/diag_manager/diag_output.F90
     FMS/diag_manager/diag_table.F90
     FMS/diag_manager/diag_util.F90
     FMS/drifters/cloud_interpolator.F90
+    FMS/drifters/drifters.F90
     FMS/drifters/drifters_comm.F90
     FMS/drifters/drifters_core.F90
-    FMS/drifters/drifters.F90
     FMS/drifters/drifters_input.F90
     FMS/drifters/drifters_io.F90
     FMS/drifters/quicksort.F90
     FMS/exchange/stock_constants.F90
     FMS/exchange/xgrid.F90
-    FMS/fft/fft99.F90
-    FMS/fft/fft.F90
     FMS/field_manager/field_manager.F90
     FMS/field_manager/fm_util.F90
     FMS/fms/fms.F90
     FMS/fms/fms_io.F90
+    FMS/fms2_io/blackboxio.F90
+    FMS/fms2_io/fms2_io.F90
+    FMS/fms2_io/fms_io_utils.F90
+    FMS/fms2_io/fms_netcdf_domain_io.F90
+    FMS/fms2_io/fms_netcdf_unstructured_domain_io.F90
+    FMS/fms2_io/netcdf_io.F90
+    FMS/horiz_interp/horiz_interp.F90
     FMS/horiz_interp/horiz_interp_bicubic.F90
     FMS/horiz_interp/horiz_interp_bilinear.F90
     FMS/horiz_interp/horiz_interp_conserve.F90
-    FMS/horiz_interp/horiz_interp.F90
     FMS/horiz_interp/horiz_interp_spherical.F90
     FMS/horiz_interp/horiz_interp_type.F90
     FMS/interpolator/interpolator.F90
     FMS/memutils/memutils.F90
+    FMS/monin_obukhov/monin_obukhov.F90
+    FMS/monin_obukhov/monin_obukhov_kernel.F90
     FMS/mosaic/gradient.F90
     FMS/mosaic/grid.F90
     FMS/mosaic/mosaic.F90
+    FMS/mosaic/mosaic2.F90
+    FMS/mpp/mpp.F90
     FMS/mpp/mpp_data.F90
     FMS/mpp/mpp_domains.F90
     FMS/mpp/mpp_efp.F90
-    FMS/mpp/mpp.F90
     FMS/mpp/mpp_io.F90
     FMS/mpp/mpp_memutils.F90
     FMS/mpp/mpp_parameter.F90
     FMS/mpp/mpp_pset.F90
     FMS/mpp/mpp_utilities.F90
-    FMS/oda_tools/oda_core_ecda.F90
-    FMS/oda_tools/oda_core.F90
-    FMS/oda_tools/oda_types.F90
-    FMS/oda_tools/write_ocean_data.F90
-    FMS/oda_tools/xbt_drop_rate_adjust.f90
     FMS/platform/platform.F90
     FMS/random_numbers/MersenneTwister.F90
     FMS/random_numbers/random_numbers.F90
     FMS/sat_vapor_pres/sat_vapor_pres.F90
     FMS/sat_vapor_pres/sat_vapor_pres_k.F90
-    FMS/station_data/station_data.F90
-    FMS/time_interp/time_interp_external.F90
     FMS/time_interp/time_interp.F90
+    FMS/time_interp/time_interp_external.F90
+    FMS/time_interp/time_interp_external2.F90
     FMS/time_manager/get_cal_time.F90
     FMS/time_manager/time_manager.F90
     FMS/topography/gaussian_topog.F90
     FMS/topography/topography.F90
     FMS/tracer_manager/tracer_manager.F90
     FMS/tridiagonal/tridiagonal.F90
-    FMS/memutils/memuse.c
+
     FMS/mosaic/create_xgrid.c
     FMS/mosaic/gradient_c2l.c
     FMS/mosaic/interp.c
     FMS/mosaic/mosaic_util.c
     FMS/mosaic/read_mosaic.c
-    FMS/affinity/affinity.c
+    FMS/mpp/mpp_memuse.c
     FMS/mpp/nsclock.c
     FMS/mpp/threadloc.c
 )
 # stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
 set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
 
-target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/mpp/include)
+target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/fms2_io/include FMS/mpp/include)
 target_include_directories(fms PRIVATE ${NETCDF_INC_DIR})
 
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)

diff --git a/FMS b/FMS
diff --git a/FV3 b/FV3
diff --git a/NEMS b/NEMS
diff --git a/cmake/GNU.cmake b/cmake/GNU.cmake
@@ -61,3 +61,10 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+    add_definitions(-DENABLE_QUAD_PRECISION)
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake
@@ -50,6 +50,10 @@ if(OPENMP)
     add_definitions(-DOPENMP)
 endif()
 
+if(QUAD_PRECISION)
+    add_definitions(-DENABLE_QUAD_PRECISION)
+endif()
+
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__IFC -sox -fp-model source")
 
@@ -90,3 +94,8 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
diff --git a/cmake/PGI.cmake b/cmake/PGI.cmake
@@ -60,3 +60,10 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+    add_definitions(-DENABLE_QUAD_PRECISION)
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
diff --git a/cmake/configure_cheyenne.gnu.cmake b/cmake/configure_cheyenne.gnu.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_cheyenne.intel-impi.cmake b/cmake/configure_cheyenne.intel-impi.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_cheyenne.intel.cmake b/cmake/configure_cheyenne.intel.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_gaea.cmake b/cmake/configure_gaea.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_hera.gnu.cmake b/cmake/configure_hera.gnu.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_hera.intel.cmake b/cmake/configure_hera.intel.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_jet.cmake → cmake/configure_jet.intel.cmake b/cmake/configure_jet.cmake → cmake/configure_jet.intel.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_linux.gnu.cmake b/cmake/configure_linux.gnu.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_linux.intel.cmake b/cmake/configure_linux.intel.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_macosx.gnu.cmake b/cmake/configure_macosx.gnu.cmake
@@ -21,6 +21,7 @@ else()
 option(OPENMP  "Enable OpenMP threading" ON)
 endif()
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_orion.intel.cmake b/cmake/configure_orion.intel.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_stampede.intel.cmake b/cmake/configure_stampede.intel.cmake
@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")
diff --git a/cmake/configure_wcoss_cray.cmake b/cmake/configure_wcoss_cray.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_wcoss_dell_p3.cmake b/cmake/configure_wcoss_dell_p3.cmake
@@ -16,6 +16,7 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/conf/before_components.mk b/conf/before_components.mk
@@ -18,14 +18,6 @@ CHOSEN_MODULE=$(BUILD_TARGET)/fv3
 
 CONFIGURE_NEMS_FILE=configure.fv3.$(BUILD_TARGET)
 
-# ----------------------------------------------------------------------
-# Exit for systems that are currently not supported
-ifeq ($(BUILD_TARGET),theia.pgi)
-  $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET))
-else ifeq ($(BUILD_TARGET),cheyenne.pgi)
-  $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET))
-endif
-
 # ----------------------------------------------------------------------
 # Copy the executable and modules.nems files into the tests/ directory
 # if a TEST_BUILD_NAME is specified.

diff --git a/conf/configure.fv3.cheyenne.gnu b/conf/configure.fv3.cheyenne.gnu
@@ -56,6 +56,7 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -118,6 +119,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -fno-range-check
 FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check
 FFLAGS_DEBUG = -g -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall

diff --git a/conf/configure.fv3.cheyenne.intel b/conf/configure.fv3.cheyenne.intel
@@ -56,6 +56,7 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -127,6 +128,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
 FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv

diff --git a/conf/configure.fv3.cheyenne.intel-impi b/conf/configure.fv3.cheyenne.intel-impi
@@ -52,6 +52,7 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -123,6 +124,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
 FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv