Skip to content

Commit

Permalink
GFSv16 netcdf post ficein cpl (NCAR#36)
Browse files Browse the repository at this point in the history 
* Updated fv3atm (see fv3atm PR NCAR#48 NOAA-EMC/fv3atm#48)
* Updated WW3
  • Loading branch information
@DusanJovic-NOAA
DusanJovic-NOAA authored Jan 25, 2020
1 parent b41e023 commit 52795b8
Show file tree
Hide file tree
Showing 55 changed files with 12,112 additions and 125,625 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
coupled*.mk
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 18 files
+2 −0 .gitmodules
+1 −1 atmos_model.F90
+1 −1 ccpp/framework
+1 −1 ccpp/physics
+1 −1 gfsphysics/physics/gfdl_cloud_microphys.F90
+0 −0 gfsphysics/physics/module_sf_noahmp_glacier.f90
+0 −0 gfsphysics/physics/module_sf_noahmplsm.f90
+0 −0 gfsphysics/physics/module_wrf_utl.f90
+5 −1 gfsphysics/physics/moninedmf_hafs.f
+0 −0 gfsphysics/physics/noahmp_tables.f90
+14 −14 gfsphysics/physics/samfdeepcnv.f
+7 −4 gfsphysics/physics/satmedmfvdifq.f
+0 −0 gfsphysics/physics/sfc_noahmp_drv.f
+1 −1 io/FV3GFS_io.F90
+1 −1 io/module_write_nemsio.F90
+61 −54 io/module_write_netcdf.F90
+42 −22 io/module_wrt_grid_comp.F90
+11 −3 io/post_gfs.F90
2 changes: 1 addition & 1 deletion WW3
Submodule WW3 updated 167 files
2 changes: 2 additions & 0 deletions cmake/configure_gaea.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF)
option(OPENMP "Enable OpenMP threading" ON)
option(AVX2 "Enable AVX2 instruction set" ON)

option(INLINE_POST "Enable inline post" OFF)

include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )

set(NEMSIO_INC $ENV{NEMSIO_INC})
Expand Down
4 changes: 2 additions & 2 deletions compsets/all.input
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ load 'fv3.input'
run fv3_control @ fv3, standard, baseline, ccpptrans
run fv3_decomp @ fv3, standard
run fv3_2threads @ fv3, standard
run fv3_ccpp_control @ fv3, standard, ccpptrans, plat==hera.intel
#run fv3_ccpp_control @ fv3, standard, ccpptrans, plat==hera.intel
run fv3_restart @ fv3, standard, baseline
run fv3_read_inc @ fv3, standard, baseline
run fv3_gfdlmp @ fv3, standard, baseline
Expand All @@ -22,7 +22,7 @@ run fv3_wrtGauss_netcdf_esmf @ fv3, standard, baseline
run fv3_wrtGauss_netcdf @ fv3, standard, baseline
run fv3_wrtGauss_nemsio @ fv3, standard, baseline

run fv3_wrtGauss_nemsio_c768 @ fv3, baseline, ww3, plat==hera.intel, plat==wcoss.cray
run fv3_wrtGauss_nemsio_c768 @ fv3, baseline, ww3, plat==hera.intel
run fv3_wrtGauss_nemsio_c192 @ fv3, standard, baseline
run fv3_stochy @ fv3, standard, baseline
run fv3_iau @ fv3, standard, baseline
Expand Down
18 changes: 11 additions & 7 deletions compsets/fv3.input
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1764,10 +1764,11 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
TEST_DESCR="Compare c768 FV3-WW3 Gaussian grid nemsio output results with previous trunk version"
CNTL_NAME="fv3_wrtGauss_nemsio_c768"

TASKS='1752'
TASKS='1470'
PPN="@[plat%C768_PPN]"
THRD="@[plat%C768_THRD]"
INPES='16'
JNPES='16'
JNPES='12'
NPX='769'
NPY='769'
NPZ='64'
Expand All @@ -1779,7 +1780,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
FHMAX='06'
WLCLK='30'
WRITE_GROUP='3'
WRTTASK_PER_GROUP='64'
WRTTASK_PER_GROUP='36'
FDIAG='0,1,2,3,4,5,6,7,8,9,10,11,12'
OUTPUT_GRID="'gaussian_grid'"
OUTPUT_FILE="'nemsio'"
Expand All @@ -1805,9 +1806,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {

nems_configure="blocked_atm_wav"
atm_model='fv3'
atm_petlist_bounds="0 1727"
atm_petlist_bounds="0 1259"
wav_model='ww3'
wav_petlist_bounds="1728 1751"
wav_petlist_bounds="1260 1469"
coupling_interval_sec=1800.0 # 30min coupling, fv3 time step

# The build variable is the fv3_ww3.exe, which
Expand All @@ -1834,6 +1835,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
'nems.configure' <=atparse= "@[PARMnems]/nems.configure.@[nems_configure].IN"
'mod_def.*' <=copyfrom= "@[WW3_input_data]"
'ww3_multi.inp' <=copyfrom= "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
'rmp_src*' <=copyfrom= "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
}

# Specify output files:
Expand Down Expand Up @@ -1879,7 +1881,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
'RESTART/sfc_data.tile4.nc' .bitcmp. "@[CNTL]/RESTART/"
'RESTART/sfc_data.tile5.nc' .bitcmp. "@[CNTL]/RESTART/"
'RESTART/sfc_data.tile6.nc' .bitcmp. "@[CNTL]/RESTART/"
'out_grd.glo_30m' .bitcmp. "@[CNTL]"
'out_grd.glo_10m' .bitcmp. "@[CNTL]"
'out_grd.ant_9km' .bitcmp. "@[CNTL]"
'out_grd.aoc_9km' .bitcmp. "@[CNTL]"
# Executable validation. This makes an MD5 sum of the fv3.exe
# for comparison against the MD5 sum made in the build job.
# This is to ensure the executable did not change during the
Expand All @@ -1888,7 +1892,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
}

spawn execute {
{"@[build%target]", ranks="@[TASKS]", ppn="@[PPN]" }
{"@[build%target]", ranks="@[TASKS]", threads="@[THRD]", ppn="@[PPN]" }
}
}

Expand Down
7 changes: 4 additions & 3 deletions compsets/hera.input
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ platform hera.intel {

cores_per_node=40 # Number of cores per node on compute nodes
cpus_per_core=2
C768_PPN=30
C768_PPN=10
C768_THRD=1

# hera testing: using longer run time
DEFAULT_TEST_WALLTIME=3000
Expand All @@ -30,8 +31,8 @@ platform hera.intel {
# INPUTS is the input directory, which should contain fix and parm
# files, plus any restarts or other inputs.

BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
INPUTS="@[BASELINE]"

default_resources={
Expand Down
1 change: 1 addition & 0 deletions compsets/platforms.input
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ platform.defaults={
walltime="@[DEFAULT_TEST_WALLTIME]"

C768_PPN="@[cores_per_node]"
C768_THRD=1
execution_time_modules=""
}

Expand Down
32 changes: 21 additions & 11 deletions compsets/wcoss.input
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ platform wcoss.phase2 {
pex='1'
cores_per_node=24
cpus_per_core=2
C768_PPN=6
C768_THRD=1

# WCOSS Phase 2 has been slow of late.
DEFAULT_TEST_WALLTIME=3000
Expand All @@ -26,8 +28,8 @@ platform wcoss.phase2 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'
MACHINE_ID='wcoss.phase2'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"

default_resources={
TASKS=156
Expand Down Expand Up @@ -120,9 +122,12 @@ platform wcoss.phase1 {
pex='1'
cores_per_node=16
cpus_per_core=2
C768_PPN=6
C768_THRD=1

MACHINE_ID='wcoss.phase1'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"

# WCOSS Phase 2 has been slow of late.
DEFAULT_TEST_WALLTIME=2700
Expand Down Expand Up @@ -197,25 +202,27 @@ platform wcoss_dell_p3 {
pex='3'
cores_per_node=28
cpus_per_core=2
C768_PPN=7
C768_THRD=1

DEFAULT_TEST_WALLTIME=1800
DEFAULT_TEST_WALLTIME=3000

MPI='LSF_IMPI'
MACHINE_ID='wcoss_dell_p3'
SHORT_TEST_QUEUE='&SHORTQ;'
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'

BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"

default_resources={
TASKS=156
WRTTASK_PER_GROUP=12
TPN=8
TPN=6
INPES=3
JNPES=8
THRD=1
THRD=4
atm_petlist_bounds="0 155"
wav_petlist_bounds="156 263"
CPLTASKS=264
Expand Down Expand Up @@ -295,6 +302,9 @@ platform wcoss.cray {
pex='cray'
cores_per_node=24
cpus_per_core=2
C768_PPN=6
C768_THRD=1

MPI='lsf_cray_intel'
SHORT_TEST_QUEUE='&SHORTQ;'
LONG_TEST_QUEUE='&LONGQ;'
Expand All @@ -304,8 +314,8 @@ platform wcoss.cray {
BUILD_WALLTIME="3600"
DEFAULT_TEST_WALLTIME=1800

BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"

execution_time_modules=[[[
module load alps
Expand Down
4 changes: 4 additions & 0 deletions conf/configure.fv3.gaea.intel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(NAM_phys),Y)
CPPDEFS += -DNAM_phys
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
Expand Down
2 changes: 1 addition & 1 deletion conf/configure.fv3.macosx.gnu
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
## NEMS configuration file
##
## Platform: Darwin Mac OS X
## Compiler: GNU (clang/gfortran) with MPICH
## Compiler: GNU (gcc/clang+gfortran) with MPICH

SHELL=/bin/sh

Expand Down
Loading

0 comments on commit 52795b8

Please sign in to comment.