From 9d9a43cd383c51e8442559d536fda94f0d563c51 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 6 Dec 2022 18:28:31 -0500 Subject: [PATCH 1/3] wip --- env/CONTAINER.env | 40 ++++++++++++++++++++++++++++ jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN | 4 +-- parm/config/config.base.emc.dyn | 8 +++--- parm/config/config.ocnanal | 12 ++++----- workflow/hosts.py | 5 +++- workflow/hosts/container.yaml | 23 ++++++++++++++++ 6 files changed, 79 insertions(+), 13 deletions(-) create mode 100755 env/CONTAINER.env create mode 100644 workflow/hosts/container.yaml diff --git a/env/CONTAINER.env b/env/CONTAINER.env new file mode 100755 index 0000000000..b16bb7ab58 --- /dev/null +++ b/env/CONTAINER.env @@ -0,0 +1,40 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + echo "argument can be any one of the following:" + echo "atmanalrun atmensanalrun" + echo "aeroanlrun" + echo "anal sfcanl fcst post vrfy metp" + echo "eobs eupd ecen efcs epos" + echo "postsnd awips gempak" + exit 1 + +fi + +step=$1 + +export npe_node_max=40 +export launcher="mpirun" +export mpmd_opt="--multi-prog" + +# Configure MPI environment +export MPI_BUFS_PER_PROC=2048 +export MPI_BUFS_PER_HOST=2048 +export MPI_GROUP_MAX=256 +export MPI_MEMMAP_OFF=1 +export MP_STDOUTMODE="ORDERED" +export KMP_AFFINITY=scatter +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 +#export LD_BIND_NOW=1 + +ulimit -s unlimited +ulimit -a + + +if [ ${step} = "ocnanalrun" ]; then + export NTHREADS_OCNANAL=1 + export APRUN_OCNANAL="${launcher} -n 2" +fi diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN index 374d889ee6..a1af6d2049 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN @@ -84,7 +84,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" || exit 1 -[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" +#cd "${DATAROOT}" || exit 1 +#[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 62d7ff59ef..c716575654 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -22,7 +22,7 @@ export PARTITION_SERVICE="@PARTITION_SERVICE@" HPSS_PROJECT=emc-global # Directories relative to installation areas: -export HOMEgfs=@HOMEgfs@ +export HOMEgfs=/home/gvernier/sandboxes/ctest-sprint/global-workflow/ #@HOMEgfs@ export PARMgfs=$HOMEgfs/parm export FIXgfs=$HOMEgfs/fix export USHgfs=$HOMEgfs/ush @@ -44,7 +44,7 @@ export DMPDIR="@DMPDIR@" # USER specific paths export HOMEDIR="@HOMEDIR@" -export STMP="@STMP@" +export STMP=/home/gvernier/sandboxes/ctest-sprint/global-workflow/sorc/gdas.cd/build/test/soca/gw/testrun/RUNDIR export PTMP="@PTMP@" export NOSCRUB="@NOSCRUB@" @@ -109,8 +109,8 @@ export DUMP_SUFFIX="" if [[ "$CDATE" -ge "2019092100" && "$CDATE" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi -export DATAROOT="${STMP}/RUNDIRS/${PSLOT}" # TODO: set via prod_envir in Ops -export RUNDIR="${DATAROOT}" # TODO: Should be removed; use DATAROOT instead +export RUNDIR="$STMP/RUNDIRS/$PSLOT" +export DATAROOT="$RUNDIR/$CDATE/$CDUMP" export ARCDIR="$NOSCRUB/archive/$PSLOT" export ICSDIR="@ICSDIR@" export ATARDIR="@ATARDIR@" diff --git a/parm/config/config.ocnanal b/parm/config/config.ocnanal index 385db2dc86..6a56190584 100644 --- a/parm/config/config.ocnanal +++ b/parm/config/config.ocnanal @@ -9,17 +9,17 @@ export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config export OBS_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_LIST=${OBS_YAML_DIR}/obs_list.yaml export FV3JEDI_STAGE_YAML=${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml -export SOCA_INPUT_FIX_DIR=/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca # TODO: Should be moved somewhere else +export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ export SOCA_VARS=tocn,socn,ssh -export SOCA_NINNER=50 -export CASE_ANL="C48" +export SOCA_NINNER=@SOCA_NINNER@ +export CASE_ANL=@CASE_ANL@ export DOMAIN_STACK_SIZE=2000000 export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin # TODO: Move the R2D2 configuration to a common space export R2D2_OBS_DB=shared -export R2D2_OBS_DUMP=s2s_v1 -export R2D2_OBS_SRC=gdas_marine +export R2D2_OBS_DUMP=@R2D2_OBS_DUMP@ +export R2D2_OBS_SRC=@R2D2_OBS_SRC@ export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed -export COMIN_OBS=/scratch2/NCEPDEV/marineda/r2d2 +export COMIN_OBS=@COMIN_OBS@ echo "END: config.ocnanal" diff --git a/workflow/hosts.py b/workflow/hosts.py index 91925da71a..ee9510df9e 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -15,7 +15,7 @@ class Host: """ SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', - 'WCOSS2', 'S4'] + 'WCOSS2', 'S4', 'CONTAINER'] def __init__(self, host=None): @@ -32,6 +32,7 @@ def __init__(self, host=None): def detect(cls): machine = 'NOTFOUND' + container = os.getenv('SINGULARITY_NAME') if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' @@ -43,6 +44,8 @@ def detect(cls): machine = 'WCOSS2' elif os.path.exists('/data/prod'): machine = 'S4' + elif container != None: + machine = 'CONTAINER' if machine not in Host.SUPPORTED_HOSTS: raise NotImplementedError(f'This machine is not a supported host.\n' + diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml new file mode 100644 index 0000000000..3aab3df751 --- /dev/null +++ b/workflow/hosts/container.yaml @@ -0,0 +1,23 @@ +BASE_GIT: '' +DMPDIR: '/home/${USER}' +PACKAGEROOT: '' +COMROOT: '' +COMINsyn: '' +HOMEDIR: '/home/${USER}' +STMP: '/home/${USER}' +PTMP: '/home/${USER}' +NOSCRUB: $HOMEDIR +SCHEDULER: none +ACCOUNT: '' +QUEUE: '' +QUEUE_SERVICE: '' +PARTITION_BATCH: '' +PARTITION_SERVICE: '' +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C96', 'C48'] From 8ea33b5cd05011f73f1cd889e2df7fac1420d96b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 7 Dec 2022 12:13:00 -0500 Subject: [PATCH 2/3] container support & defaults for ocnanal --- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN | 4 ++-- parm/config/config.base.emc.dyn | 8 ++++---- parm/config/yaml/defaults.yaml | 8 ++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN index a1af6d2049..374d889ee6 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN @@ -84,7 +84,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -#cd "${DATAROOT}" || exit 1 -#[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index c716575654..62d7ff59ef 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -22,7 +22,7 @@ export PARTITION_SERVICE="@PARTITION_SERVICE@" HPSS_PROJECT=emc-global # Directories relative to installation areas: -export HOMEgfs=/home/gvernier/sandboxes/ctest-sprint/global-workflow/ #@HOMEgfs@ +export HOMEgfs=@HOMEgfs@ export PARMgfs=$HOMEgfs/parm export FIXgfs=$HOMEgfs/fix export USHgfs=$HOMEgfs/ush @@ -44,7 +44,7 @@ export DMPDIR="@DMPDIR@" # USER specific paths export HOMEDIR="@HOMEDIR@" -export STMP=/home/gvernier/sandboxes/ctest-sprint/global-workflow/sorc/gdas.cd/build/test/soca/gw/testrun/RUNDIR +export STMP="@STMP@" export PTMP="@PTMP@" export NOSCRUB="@NOSCRUB@" @@ -109,8 +109,8 @@ export DUMP_SUFFIX="" if [[ "$CDATE" -ge "2019092100" && "$CDATE" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi -export RUNDIR="$STMP/RUNDIRS/$PSLOT" -export DATAROOT="$RUNDIR/$CDATE/$CDUMP" +export DATAROOT="${STMP}/RUNDIRS/${PSLOT}" # TODO: set via prod_envir in Ops +export RUNDIR="${DATAROOT}" # TODO: Should be removed; use DATAROOT instead export ARCDIR="$NOSCRUB/archive/$PSLOT" export ICSDIR="@ICSDIR@" export ATARDIR="@ATARDIR@" diff --git a/parm/config/yaml/defaults.yaml b/parm/config/yaml/defaults.yaml index 7537454957..b119aee6a8 100644 --- a/parm/config/yaml/defaults.yaml +++ b/parm/config/yaml/defaults.yaml @@ -1,3 +1,11 @@ aeroanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 + +ocnanal: + SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25' + CASE_ANL: 'C48' + COMIN_OBS: '/scratch2/NCEPDEV/marineda/r2d2' + SOCA_NINNER: 50 + R2D2_OBS_SRC: 'gdas_marine' + R2D2_OBS_DUMP: 's2s_v1' From f142516c997e8899ff2256f182c6021463927e9a Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Sat, 10 Dec 2022 12:37:49 -0500 Subject: [PATCH 3/3] removed changes unrelated to pr --- env/CONTAINER.env | 3 +-- parm/config/config.ocnanal | 12 ++++++------ parm/config/yaml/defaults.yaml | 8 -------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/env/CONTAINER.env b/env/CONTAINER.env index b16bb7ab58..48014ab313 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -28,13 +28,12 @@ export MP_STDOUTMODE="ORDERED" export KMP_AFFINITY=scatter export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 -#export LD_BIND_NOW=1 ulimit -s unlimited ulimit -a -if [ ${step} = "ocnanalrun" ]; then +if [ "${step}" = "ocnanalrun" ]; then export NTHREADS_OCNANAL=1 export APRUN_OCNANAL="${launcher} -n 2" fi diff --git a/parm/config/config.ocnanal b/parm/config/config.ocnanal index 6a56190584..385db2dc86 100644 --- a/parm/config/config.ocnanal +++ b/parm/config/config.ocnanal @@ -9,17 +9,17 @@ export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config export OBS_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_LIST=${OBS_YAML_DIR}/obs_list.yaml export FV3JEDI_STAGE_YAML=${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml -export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ +export SOCA_INPUT_FIX_DIR=/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca # TODO: Should be moved somewhere else export SOCA_VARS=tocn,socn,ssh -export SOCA_NINNER=@SOCA_NINNER@ -export CASE_ANL=@CASE_ANL@ +export SOCA_NINNER=50 +export CASE_ANL="C48" export DOMAIN_STACK_SIZE=2000000 export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin # TODO: Move the R2D2 configuration to a common space export R2D2_OBS_DB=shared -export R2D2_OBS_DUMP=@R2D2_OBS_DUMP@ -export R2D2_OBS_SRC=@R2D2_OBS_SRC@ +export R2D2_OBS_DUMP=s2s_v1 +export R2D2_OBS_SRC=gdas_marine export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed -export COMIN_OBS=@COMIN_OBS@ +export COMIN_OBS=/scratch2/NCEPDEV/marineda/r2d2 echo "END: config.ocnanal" diff --git a/parm/config/yaml/defaults.yaml b/parm/config/yaml/defaults.yaml index b119aee6a8..7537454957 100644 --- a/parm/config/yaml/defaults.yaml +++ b/parm/config/yaml/defaults.yaml @@ -1,11 +1,3 @@ aeroanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 - -ocnanal: - SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25' - CASE_ANL: 'C48' - COMIN_OBS: '/scratch2/NCEPDEV/marineda/r2d2' - SOCA_NINNER: 50 - R2D2_OBS_SRC: 'gdas_marine' - R2D2_OBS_DUMP: 's2s_v1'