From 08a535492952d863f860977f0ab0da4bc79bc931 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 25 Mar 2020 14:58:14 +0000 Subject: [PATCH] Issue #21 - add phase 3.5 support - add partition option to setup scripts - remove machine if-blocks from config.base and add variable population to setup_expt*py scripts - add phase 3.5 ppn value to WCOSS_DELL_P3 env and config.resources files --- env/WCOSS_DELL_P3.env | 5 +- parm/config/config.base.emc.dyn | 87 ++++++------------------------- parm/config/config.resources | 3 ++ ush/rocoto/setup_expt.py | 54 +++++++++++++++++++ ush/rocoto/setup_expt_fcstonly.py | 54 +++++++++++++++++++ 5 files changed, 131 insertions(+), 72 deletions(-) diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env index dabfbdb351..b7000b0f89 100755 --- a/env/WCOSS_DELL_P3.env +++ b/env/WCOSS_DELL_P3.env @@ -14,8 +14,11 @@ fi step=$1 # WCOSS_DELL_P3 information -export npe_node_max=28 export launcher="mpirun -n" +export npe_node_max=28 +if [ "$QUEUE" = "dev2" -o "$QUEUE" = "devonprod2" ]; then # WCOSS Dell 3.5 + export npe_node_max=40 +fi # Due to ESMF issue, fv3gfs model must run with npe_node_max=24 if [ $step = "fcst" -o $step = "efcs" ]; then diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 6fd5c6a0f4..69965154bf 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -12,19 +12,9 @@ export machine="@MACHINE@" export RUN_ENVIR="emc" # Account, queue, etc. -if [ $machine = "HERA" ]; then - - export ACCOUNT="fv3-cpu" - export QUEUE="batch" - export QUEUE_ARCH="service" - -elif [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" ]; then - - export ACCOUNT="GFS-DEV" - export QUEUE="dev" - export QUEUE_ARCH="dev_transfer" - -fi +export ACCOUNT="@ACCOUNT@" +export QUEUE="@QUEUE@" +export QUEUE_ARCH="@QUEUE_ARCH@" # Project to use in mass store: HPSS_PROJECT=emc-global @@ -41,64 +31,19 @@ export SCRgfs=$HOMEgfs/scripts ######################################################################## # GLOBAL static environment parameters -if [ $machine = "HERA" ]; then - - #export NWPROD="/scratch1/NCEPDEV/global/glopara/nwpara" - export DMPDIR="/scratch1/NCEPDEV/global/glopara/dump" - export RTMFIX=$CRTM_FIX - -elif [ $machine = "WCOSS_C" ]; then - - export NWPROD="/gpfs/hps/nco/ops/nwprod" - export DMPDIR="/gpfs/dell3/emc/global/dump" - export RTMFIX=$CRTM_FIX - -elif [ $machine = "WCOSS_DELL_P3" ]; then - - export NWPROD="/gpfs/dell1/nco/ops/nwprod" - export DMPDIR="/gpfs/dell3/emc/global/dump" - export RTMFIX=$CRTM_FIX -fi - - -# Machine specific paths used everywhere -if [ $machine = "HERA" ]; then - - # USER specific paths - export HOMEDIR="/scratch1/NCEPDEV/global/$USER" - export STMP="/scratch1/NCEPDEV/stmp2/$USER" - export PTMP="/scratch1/NCEPDEV/stmp4/$USER" - export NOSCRUB="$HOMEDIR" - - # Base directories for various builds - export BASE_GIT="/scratch1/NCEPDEV/global/glopara/git" - export BASE_SVN="/scratch1/NCEPDEV/global/glopara/svn" - -elif [ $machine = "WCOSS_C" ]; then - - # USER specific paths - export HOMEDIR="/gpfs/hps3/emc/global/noscrub/$USER" - export STMP="/gpfs/hps2/stmp/$USER" - export PTMP="/gpfs/hps2/ptmp/$USER" - export NOSCRUB="/gpfs/hps3/emc/global/noscrub/$USER" - - # Base directories for various builds - export BASE_GIT="/gpfs/hps3/emc/global/noscrub/emc.glopara/git" - export BASE_SVN="/gpfs/hps3/emc/global/noscrub/emc.glopara/svn" - -elif [ $machine = "WCOSS_DELL_P3" ]; then - - # USER specific paths - export HOMEDIR="/gpfs/dell2/emc/modeling/noscrub/$USER" - export STMP="/gpfs/dell3/stmp/$USER" - export PTMP="/gpfs/dell3/ptmp/$USER" - export NOSCRUB="/gpfs/dell2/emc/modeling/noscrub/$USER" - - # Base directories for various builds - export BASE_GIT="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git" - export BASE_SVN="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git" - -fi +export NWPROD="@NWPROD@" +export DMPDIR="@DMPDIR@" +export RTMFIX=$CRTM_FIX + +# USER specific paths +export HOMEDIR="@HOMEDIR@" +export STMP="@STMP@" +export PTMP="@PTMP@" +export NOSCRUB="@NOSCRUB@" + +# Base directories for various builds +export BASE_GIT="@BASE_GIT@" +export BASE_SVN="@BASE_SVN@" # Toggle to turn on/off GFS downstream processing. export DO_BUFRSND="NO" diff --git a/parm/config/config.resources b/parm/config/config.resources index 9923b94d66..88d677ad21 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -21,6 +21,9 @@ echo "BEGIN: config.resources" if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_node_max=28 + if [ "$QUEUE" = "dev2" -o "$QUEUE" = "devonprod2" ]; then # WCOSS Dell 3.5 + export npe_node_max=40 + fi elif [[ "$machine" = "WCOSS_C" ]]; then export npe_node_max=24 elif [[ "$machine" = "JET" ]]; then diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index a4630f74d2..5e9c0d03a9 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -91,6 +91,17 @@ def edit_baseconfig(): .replace('@CASECTL@', 'C%d' % resdet) \ .replace('@NMEM_ENKF@', '%d' % nens) \ .replace('@HOMEgfs@', top) \ + .replace('@BASE_GIT@', base_git) \ + .replace('@BASE_SVN@', base_svn) \ + .replace('@DMPDIR@', dmpdir) \ + .replace('@NWPROD@', nwprod) \ + .replace('@HOMEDIR@', homedir) \ + .replace('@STMP@', stmp) \ + .replace('@PTMP@', ptmp) \ + .replace('@NOSCRUB@', noscrub) \ + .replace('@ACCOUNT@', account) \ + .replace('@QUEUE@', queue) \ + .replace('@QUEUE_ARCH@', queue_arch) \ .replace('@gfs_cyc@', '%d' % gfs_cyc) if expdir is not None: line = line.replace('@EXPDIR@', os.path.dirname(expdir)) @@ -129,6 +140,7 @@ def edit_baseconfig(): parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20) parser.add_argument('--cdump', help='CDUMP to start the experiment', type=str, required=False, default='gdas') parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None) args = parser.parse_args() @@ -149,6 +161,48 @@ def edit_baseconfig(): nens = args.nens cdump = args.cdump gfs_cyc = args.gfs_cyc + partition = args.partition + + # Set machine defaults + if machine is 'WCOSS_DELL_P3': + base_git = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' + base_svn = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' + dmpdir = '/gpfs/dell3/emc/global/dump' + nwprod = '/gpfs/dell1/nco/ops/nwprod' + homedir = '/gpfs/dell2/emc/modeling/noscrub/$USER' + stmp = '/gpfs/dell3/stmp/$USER' + ptmp = '/gpfs/dell3/ptmp/$USER' + noscrub = '/gpfs/dell2/emc/modeling/noscrub/$USER' + account = 'GFS-DEV' + queue = 'dev' + queue_arch = 'dev_transfer' + if partition is not None and partition in ['3p5']: + queue = 'dev2' + queue_arch = 'dev_transfer2' + elif machine is 'WCOSS_C': + base_git = '/gpfs/hps3/emc/global/noscrub/emc.glopara/git' + base_svn = '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn' + dmpdir = '/gpfs/dell3/emc/global/dump' + nwprod = '/gpfs/hps/nco/ops/nwprod' + homedir = '/gpfs/hps3/emc/global/noscrub/$USER' + stmp = '/gpfs/hps2/stmp/$USER' + ptmp = '/gpfs/hps2/ptmp/$USER' + noscrub = '/gpfs/hps3/emc/global/noscrub/$USER' + account = 'GFS-DEV' + queue = 'dev' + queue_arch = 'dev_transfer' + elif machine is 'HERA': + base_git = '/scratch1/NCEPDEV/global/glopara/git' + base_svn = '/scratch1/NCEPDEV/global/glopara/svn' + dmpdir = '/scratch1/NCEPDEV/global/glopara/dump' + nwprod = '/scratch1/NCEPDEV/global/glopara/nwpara' + homedir = '/scratch1/NCEPDEV/global/$USER' + stmp = '/scratch1/NCEPDEV/stmp2/$USER' + ptmp = '/scratch1/NCEPDEV/stmp4/$USER' + noscrub = '$HOMEDIR' + account = 'fv3-cpu' + queue = 'batch' + queue_arch = 'service' if args.icsdir is not None and not os.path.exists(icsdir): msg = 'Initial conditions do not exist in %s' % icsdir diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py index 51666a3626..12b85216ac 100755 --- a/ush/rocoto/setup_expt_fcstonly.py +++ b/ush/rocoto/setup_expt_fcstonly.py @@ -65,6 +65,17 @@ def edit_baseconfig(): .replace('@EDATE@', edate.strftime('%Y%m%d%H')) \ .replace('@CASECTL@', 'C%d' % res) \ .replace('@HOMEgfs@', top) \ + .replace('@BASE_GIT@', base_git) \ + .replace('@BASE_SVN@', base_svn) \ + .replace('@DMPDIR@', dmpdir) \ + .replace('@NWPROD@', nwprod) \ + .replace('@HOMEDIR@', homedir) \ + .replace('@STMP@', stmp) \ + .replace('@PTMP@', ptmp) \ + .replace('@NOSCRUB@', noscrub) \ + .replace('@ACCOUNT@', account) \ + .replace('@QUEUE@', queue) \ + .replace('@QUEUE_ARCH@', queue_arch) \ .replace('@gfs_cyc@', '%d' % gfs_cyc) if expdir is not None: line = line.replace('@EXPDIR@', os.path.dirname(expdir)) @@ -99,6 +110,7 @@ def edit_baseconfig(): parser.add_argument('--edate', help='end date experiment', type=str, required=True) parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None) parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None) args = parser.parse_args() @@ -115,6 +127,48 @@ def edit_baseconfig(): comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot) expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot) gfs_cyc = args.gfs_cyc + partition = args.partition + + # Set machine defaults + if machine is 'WCOSS_DELL_P3': + base_git = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' + base_svn = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' + dmpdir = '/gpfs/dell3/emc/global/dump' + nwprod = '/gpfs/dell1/nco/ops/nwprod' + homedir = '/gpfs/dell2/emc/modeling/noscrub/$USER' + stmp = '/gpfs/dell3/stmp/$USER' + ptmp = '/gpfs/dell3/ptmp/$USER' + noscrub = '/gpfs/dell2/emc/modeling/noscrub/$USER' + account = 'GFS-DEV' + queue = 'dev' + queue_arch = 'dev_transfer' + if partition is not None and partition in ['3p5']: + queue = 'dev2' + queue_arch = 'dev_transfer2' + elif machine is 'WCOSS_C': + base_git = '/gpfs/hps3/emc/global/noscrub/emc.glopara/git' + base_svn = '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn' + dmpdir = '/gpfs/dell3/emc/global/dump' + nwprod = '/gpfs/hps/nco/ops/nwprod' + homedir = '/gpfs/hps3/emc/global/noscrub/$USER' + stmp = '/gpfs/hps2/stmp/$USER' + ptmp = '/gpfs/hps2/ptmp/$USER' + noscrub = '/gpfs/hps3/emc/global/noscrub/$USER' + account = 'GFS-DEV' + queue = 'dev' + queue_arch = 'dev_transfer' + elif machine is 'HERA': + base_git = '/scratch1/NCEPDEV/global/glopara/git' + base_svn = '/scratch1/NCEPDEV/global/glopara/svn' + dmpdir = '/scratch1/NCEPDEV/global/glopara/dump' + nwprod = '/scratch1/NCEPDEV/global/glopara/nwpara' + homedir = '/scratch1/NCEPDEV/global/$USER' + stmp = '/scratch1/NCEPDEV/stmp2/$USER' + ptmp = '/scratch1/NCEPDEV/stmp4/$USER' + noscrub = '$HOMEDIR' + account = 'fv3-cpu' + queue = 'batch' + queue_arch = 'service' # COMROT directory create_comrot = True