diff --git a/AGCM.rc.tmpl b/AGCM.rc.tmpl index 4019cd9e..920a29e6 100644 --- a/AGCM.rc.tmpl +++ b/AGCM.rc.tmpl @@ -4,9 +4,6 @@ NX: @NX NY: @NY -IOSERVER_NODES: @NUM_OSERVER_NODES -NUM_BACKEND_PES: @NUM_BACKEND_PES - AGCM_IM: @AGCM_IM AGCM_JM: @AGCM_JM AGCM_LM: @AGCM_LM diff --git a/gcm_run.j b/gcm_run.j index b7b9f152..c91e0262 100755 --- a/gcm_run.j +++ b/gcm_run.j @@ -91,8 +91,6 @@ set OGCM_JM = `grep '^\s*OGCM\.JM_WORLD:' $HOMDIR/AGCM.rc | cut -d: -f2` # -------------------------------------------- set USE_IOSERVER = @USE_IOSERVER -set NUM_OSERVER_NODES = `grep '^\s*IOSERVER_NODES:' $HOMDIR/AGCM.rc | cut -d: -f2` -set NUM_BACKEND_PES = `grep '^\s*NUM_BACKEND_PES:' $HOMDIR/AGCM.rc | cut -d: -f2` # Check for Over-Specification of CPU Resources # --------------------------------------------- @@ -106,30 +104,49 @@ endif @ MODEL_NPES = $NX * $NY -set NCPUS_PER_NODE = @NCPUS_PER_NODE +@RUN_SPECIFIC set NCPUS_PER_NODE = @NCPUS_PER_NODE +@RUN_ANYWHERE set NCPUS_PER_NODE=`echo "scale=6;($NCPUS / $SLURM_NNODES)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` set NUM_MODEL_NODES=`echo "scale=6;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` if ( $NCPUS != NULL ) then if ( $USE_IOSERVER == 1 ) then - @ TOTAL_NODES = $NUM_MODEL_NODES + $NUM_OSERVER_NODES - @ TOTAL_PES = $TOTAL_NODES * $NCPUS_PER_NODE - - if( $TOTAL_PES > $NCPUS ) then - echo "CPU Resources are Over-Specified" - echo "--------------------------------" - echo "Allotted NCPUs: $NCPUS" - echo "Requested NCPUs: $TOTAL_PES" - echo "" - echo "Specified NX: $NX" - echo "Specified NY: $NY" - echo "" - echo "Specified model nodes: $NUM_MODEL_NODES" - echo "Specified oserver nodes: $NUM_OSERVER_NODES" - echo "Specified cores per node: $NCPUS_PER_NODE" - exit - endif + # Next the number of frontend PEs is 10% of the model PEs + set NUM_FRONTEND_PES=`echo "scale=6;($MODEL_NPES * 0.1)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + + # We roughly figure out the number of collections in the HISTORY.rc (this is not perfect, but is close to right) + set NUM_HIST_COLLECTIONS=`cat HISTORY.rc | sed -n '/^COLLECTIONS:/,/^ *::$/{p;/^ *::$/q}' | grep -v '^ *#' | wc -l` + + # And the total number of oserver PEs is frontend PEs plus number of history collections + @ NUM_OSERVER_PES=$NUM_FRONTEND_PES + $NUM_HIST_COLLECTIONS + + # Now calculate the number of oserver nodes + set NUM_OSERVER_NODES=`echo "scale=6;($NUM_OSERVER_PES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + + # And then the number of backend PEs is the number of history collections divided by the number of oserver nodes + set NUM_BACKEND_PES=`echo "scale=6;($NUM_HIST_COLLECTIONS / $NUM_OSERVER_NODES)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + + # multigroup requires at least two backend pes + if ($NUM_BACKEND_PES < 2) set NUM_BACKEND_PES = 2 + +@RUN_ANYWHERE @ TOTAL_PES = $NCPUS +@RUN_SPECIFIC @ TOTAL_NODES = $NUM_MODEL_NODES + $NUM_OSERVER_NODES +@RUN_SPECIFIC @ TOTAL_PES = $TOTAL_NODES * $NCPUS_PER_NODE +@RUN_SPECIFIC if( $TOTAL_PES > $NCPUS ) then +@RUN_SPECIFIC echo "CPU Resources are Over-Specified" +@RUN_SPECIFIC echo "--------------------------------" +@RUN_SPECIFIC echo "Allotted NCPUs: $NCPUS" +@RUN_SPECIFIC echo "Requested NCPUs: $TOTAL_PES" +@RUN_SPECIFIC echo "" +@RUN_SPECIFIC echo "Specified NX: $NX" +@RUN_SPECIFIC echo "Specified NY: $NY" +@RUN_SPECIFIC echo "" +@RUN_SPECIFIC echo "Specified model nodes: $NUM_MODEL_NODES" +@RUN_SPECIFIC echo "Specified oserver nodes: $NUM_OSERVER_NODES" +@RUN_SPECIFIC echo "Specified cores per node: $NCPUS_PER_NODE" +@RUN_SPECIFIC exit +@RUN_SPECIFIC endif else @@ -144,8 +161,8 @@ if ( $NCPUS != NULL ) then echo "Specified NX: $NX" echo "Specified NY: $NY" echo "" - echo "Specified model nodes: $NUM_MODEL_NODES" - echo "Specified cores per node: $NCPUS_PER_NODE" + @RUN_ANYWHERE echo "Specified model nodes: $NUM_MODEL_NODES" + @RUN_ANYWHERE echo "Specified cores per node: $NCPUS_PER_NODE" exit endif @@ -325,7 +342,7 @@ setenv EMISSIONS @EMISSIONS @MOM5setenv BCTAG `basename $ABCSDIR` @MOM6setenv BCTAG `basename $ABCSDIR` #this is hard-wired for NAS for now - should make it more general -@MITsetenv GRIDDIR /nobackupp18/afahad/GEOSMITgcmFiles/GRIDDIR/a${AGCM_IM}x${AGCM_JM}_o${OGCM_IM}x${OGCM_JM} +@MITsetenv GRIDDIR /nobackupp18/afahad/GEOSMITgcmFiles/GRIDDIR/a${AGCM_IM}x${AGCM_JM}_o${OGCM_IM}x${OGCM_JM} @MITsetenv BCTAG `basename $GRIDDIR` @DATAOCEANsetenv BCTAG `basename $BCSDIR` @@ -544,7 +561,7 @@ set tile_rsts = (catch catchcn route lake landice openwater saltwater seaicether # ---------------------------------- set rst_by_face = NO if( $GCMEMIP == TRUE ) then - if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_rst & -e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then + if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_rst & -e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then echo "grid-based internal_rst and internal_face_x_rst should not co-exist" echo "please remove all *internal_rst except these tile-based restarts :" foreach rst ( $tile_rsts ) @@ -552,7 +569,7 @@ if( $GCMEMIP == TRUE ) then end exit endif - if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then + if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then set rst_by_face = YES endif else @@ -564,7 +581,7 @@ else end exit endif - if(-e $EXPDIR/fvcore_internal_face_1_rst) then + if(-e $EXPDIR/fvcore_internal_face_1_rst) then set rst_by_face = YES endif endif @@ -609,7 +626,7 @@ foreach rst ( $dummy ) if ( $rst =~ *$tile_rst* ) then set is_tile_rst = TRUE break - endif + endif end endif if ($is_tile_rst == FALSE & $rst_by_face == YES) then @@ -618,7 +635,7 @@ foreach rst ( $dummy ) set rst = ${part1}_face_${n}_rst set rst_file_names = `echo $rst_file_names $rst` end - else + else set rst_file_names = `echo $rst_file_names $rst` endif end @@ -1038,53 +1055,53 @@ endif @MIT # --------------------------------------------------- @MIT # For MITgcm restarts - before running GEOSgcm.x @MIT # --------------------------------------------------- -@MIT +@MIT @MIT # set time interval for segment in seconds -@MIT +@MIT @MIT set yearc = `echo $nymdc | cut -c1-4` @MIT set monthc = `echo $nymdc | cut -c5-6` @MIT set dayc = `echo $nymdc | cut -c7-8` @MIT set hourc = `echo $nhmsc | cut -c1-2` @MIT set minutec = `echo $nhmsc | cut -c3-4` @MIT set secondc = `echo $nhmsc | cut -c5-6` -@MIT +@MIT @MIT set yearf = `echo $nymdf | cut -c1-4` @MIT set monthf = `echo $nymdf | cut -c5-6` @MIT set dayf = `echo $nymdf | cut -c7-8` @MIT set hourf = `echo $nhmsf | cut -c1-2` @MIT set minutef = `echo $nhmsf | cut -c3-4` @MIT set secondf = `echo $nhmsf | cut -c5-6` -@MIT +@MIT @MIT set yearf = `echo $nymdf | cut -c1-4` -@MIT +@MIT @MIT set time1 = `date -u -d "${yearc}-${monthc}-${dayc}T${hourc}:${minutec}:${secondc}" "+%s"` @MIT set time2 = `date -u -d "${yearf}-${monthf}-${dayf}T${hourf}:${minutef}:${secondf}" "+%s"` -@MIT +@MIT @MIT @ mitdt = $time2 - $time1 @MIT echo "Segment time: $mitdt" -@MIT -@MIT +@MIT +@MIT @MIT # Set-up MITgcm run directory @MIT if (! -e mitocean_run) mkdir -p mitocean_run @MIT cd mitocean_run -@MIT +@MIT @MIT # link mit configuration and initialization files @MIT ln -sf $EXPDIR/mit_input/* . @MIT # link mitgcm restarts if exist @MIT /bin/ln -sf $EXPDIR/restarts/pic* . @MIT # make an archive folder for mitgcm run @MIT mkdir $EXPDIR/mit_output -@MIT +@MIT @MIT # Calculate segment time steps @MIT set mit_nTimeSteps = `cat ${SCRDIR}/AGCM.rc | grep OGCM_RUN_DT: | cut -d: -f2 | tr -s " " | cut -d" " -f2` @MIT @ mit_nTimeSteps = ${mitdt} / $mit_nTimeSteps -@MIT +@MIT @MIT #change namelist variables in data - nTimeSteps, chkptFreq and monitorFreq @MIT sed -i "s/nTimeSteps.*/nTimeSteps = ${mit_nTimeSteps},/" data @MIT sed -i "s/chkptFreq.*/chkptFreq = ${mitdt}.0,/" data @MIT sed -i "s/pChkptFreq.*/pChkptFreq = ${mitdt}.0,/" data @MIT # get nIter0 -@MIT +@MIT @MIT if (! -e ${EXPDIR}/restarts/MITgcm_restart_dates.txt ) then @MIT set nIter0 = `grep nIter0 data | tr -s " " | cut -d"=" -f2 | cut -d"," -f1 | awk '{$1=$1;print}'` @MIT else @@ -1099,7 +1116,7 @@ endif @MIT sed -i "s/nIter0.*/ nIter0 = ${nIter0},/" data @MIT endif @MIT endif -@MIT +@MIT @MIT cd .. @MIT # --------------------------------------------------- @MIT # End MITgcm restarts - before running GEOSgcm.x @@ -1114,7 +1131,7 @@ setenv OMP_NUM_THREADS 1 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null if( $USE_IOSERVER == 1 ) then - set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $NUM_OSERVER_NODES" + set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES " set IOSERVER_EXTRA = "--oserver_type multigroup --npes_backend_pernode $NUM_BACKEND_PES" else set IOSERVER_OPTIONS = "" @@ -1136,29 +1153,29 @@ echo GEOSgcm Run Status: $rc @MIT # --------------------------------------------------- @MIT # For MITgcm restarts - after running GEOSgcm.x @MIT # --------------------------------------------------- -@MIT +@MIT @MIT set STEADY_STATE_OCEAN=`grep STEADY_STATE_OCEAN AGCM.rc | cut -d':' -f2 | tr -d " "` -@MIT +@MIT @MIT # update ocean only if activated. Otherwize use the same pickups (passive ocean). @MIT if ( ${STEADY_STATE_OCEAN} != 0 ) then -@MIT +@MIT @MIT if ( ${rc} == 0 ) then -@MIT +@MIT @MIT # Update nIter0 for next segment @MIT set znIter00 = `echo $nIter0 | awk '{printf("%010d",$1)}'` @MIT @ nIter0 = $nIter0 + $mit_nTimeSteps @MIT set znIter0 = `echo $nIter0 | awk '{printf("%010d",$1)}'` -@MIT +@MIT @MIT # to update MITgcm restart list file @MIT sed -i "/${nIter0}/d" ${EXPDIR}/restarts/MITgcm_restart_dates.txt @MIT echo "Date_GEOS5 $nymdf $nhmsf NITER0_MITgcm ${nIter0}" >> ${EXPDIR}/restarts/MITgcm_restart_dates.txt -@MIT +@MIT @MIT /bin/mv $SCRDIR/mitocean_run/STDOUT.0000 $EXPDIR/mit_output/STDOUT.${znIter00} -@MIT +@MIT @MIT endif -@MIT +@MIT @MIT cd $SCRDIR/mitocean_run -@MIT +@MIT @MIT # Check existance of roling pickups @MIT set nonomatch rp = ( pickup*ckptA* ) @MIT echo $rp @@ -1172,7 +1189,7 @@ echo GEOSgcm Run Status: $rc @MIT /bin/mv ${fname} $EXPDIR/restarts/${bname}.${timeStepNumber}.${aname} @MIT end @MIT endif -@MIT +@MIT @MIT # Check existance of permanent pickups @MIT set nonomatch pp = ( pickup* ) @MIT echo $pp @@ -1182,7 +1199,7 @@ echo GEOSgcm Run Status: $rc @MIT if ( ! -e $EXPDIR/restarts/${fname} ) /bin/mv ${fname} $EXPDIR/restarts/${fname} @MIT end @MIT endif -@MIT +@MIT @MIT /bin/mv T.* $EXPDIR/mit_output/ @MIT /bin/mv S.* $EXPDIR/mit_output/ @MIT /bin/mv U.* $EXPDIR/mit_output/ @@ -1190,31 +1207,31 @@ echo GEOSgcm Run Status: $rc @MIT /bin/mv W.* $EXPDIR/mit_output/ @MIT /bin/mv PH* $EXPDIR/mit_output/ @MIT /bin/mv Eta.* $EXPDIR/mit_output/ -@MIT +@MIT @MIT /bin/mv AREA.* $EXPDIR/mit_output/ @MIT /bin/mv HEFF.* $EXPDIR/mit_output/ @MIT /bin/mv HSNOW.* $EXPDIR/mit_output/ @MIT /bin/mv UICE.* $EXPDIR/mit_output/ @MIT /bin/mv VICE.* $EXPDIR/mit_output/ -@MIT +@MIT @MIT #copy mit output to mit_output @MIT foreach i (`grep -i filename data.diagnostics | grep "^ " | cut -d"=" -f2 | cut -d"'" -f2 | awk '{$1=$1;print}'`) @MIT /bin/mv ${i}* $EXPDIR/mit_output/ @MIT end -@MIT +@MIT @MIT foreach i (`grep -i stat_fName data.diagnostics | grep "^ " | cut -d"=" -f2 | cut -d"'" -f2 | awk '{$1=$1;print}'`) @MIT /bin/mv ${i}* $EXPDIR/mit_output/ @MIT end -@MIT +@MIT @MIT cd $SCRDIR -@MIT +@MIT @MIT endif -@MIT +@MIT @MIT # --------------------------------------------------- @MIT # End MITgcm restarts - after running GEOSgcm.x @MIT # --------------------------------------------------- - + ####################################################################### # Rename Final Checkpoints => Restarts for Next Segment and Archive # Note: cap_restart contains the current NYMD and NHMS diff --git a/gcm_setup b/gcm_setup index 14ac48f9..8969efa8 100755 --- a/gcm_setup +++ b/gcm_setup @@ -390,19 +390,24 @@ endif ASKPROC: +set RUN_ANYWHERE = "#DELETE" +set RUN_SPECIFIC = "" + if ( $SITE == 'NCCS' ) then echo "Enter the ${C1}Processor Type${CN} you wish to run on:" - echo " ${C2}sky (Skylake)${CN} (default)" + echo " ${C2}sky (Skylake)${CN}" echo " ${C2}cas (Cascade Lake)${CN}" + echo " ${C2}any (Any Node Type)${CN} (default)" echo " " set MODEL = `echo $<` set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` if ( .$MODEL == .) then - set MODEL = 'sky' + set MODEL = 'any' endif - if( $MODEL != 'sky' & \ - $MODEL != 'cas' ) goto ASKPROC + if( $MODEL != 'sky' & \ + $MODEL != 'cas' & \ + $MODEL != 'any' ) goto ASKPROC if ($MODEL == 'sky') then set NCPUS_PER_NODE = 40 @@ -414,6 +419,12 @@ if ( $SITE == 'NCCS' ) then # that doesn't waste too much #set NCPUS_PER_NODE = 48 set NCPUS_PER_NODE = 45 + else if ($MODEL == 'any') then + set NCPUS_PER_NODE = ANY + # We need to know the MAX_CPUS_PER_NODE for calculations + set MAX_CPUS_PER_NODE = 45 + set RUN_ANYWHERE = "" + set RUN_SPECIFIC = "#DELETE" endif else if ( $SITE == 'NAS' ) then @@ -1183,7 +1194,6 @@ if( $AGCM_IM == "c2880" ) then set CNV_NX = 8 set POST_NDS = 32 set USE_SHMEM = 1 - set DEF_IOS_NDS = 4 endif if( $AGCM_IM == "c5760" ) then set DT = 30 @@ -1206,7 +1216,6 @@ if( $AGCM_IM == "c5760" ) then set CNV_NX = 8 set POST_NDS = 32 set USE_SHMEM = 1 - set DEF_IOS_NDS = 4 endif # CONUS Stretched Grids set CONUS = '#' @@ -1665,38 +1674,56 @@ echo "Using ${C2}${HISTORY_TEMPLATE}${CN}" # Calculate OSERVER nodes based on recommended algorithm if ( $DO_IOS == TRUE ) then - # In the calculations below, the weird bc-awk command is to round up the floating point calcs + # In the calculations below, the weird bc-awk ceil command is to round up the floating point calcs + # + # function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}' + # + # For rounding down, we use a floor function + # + # function floor(x, y){y=int(x); return(xy?y+1:y)} {print ceil($1)}'` + + if ( "$NCPUS_PER_NODE" != "ANY" ) then - # First we calculate the number of model nodes - set NUM_MODEL_NODES=`echo "scale=6;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + # We calculate the number of nodes the total number of PEs will fit on + set MODEL_NODES=`echo "scale=6;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + set NODES=`echo "scale=6;($TOTAL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` - # Next the number of frontend PEs is 10% of the model PEs - set NUM_FRONTEND_PES=`echo "scale=6;($MODEL_NPES * 0.1)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + # Now if NODES equals MODEL_NODES, then we need to add one to NODES as we need at least one node for the IO server + if ( $NODES == $MODEL_NODES ) @ NODES = $NODES + 1 - # Now we roughly figure out the number of collections in the HISTORY.rc (this is not perfect, but is close to right) - set NUM_HIST_COLLECTIONS=`cat $TMPHIST | sed -n '/^COLLECTIONS:/,/^ *::$/{p;/^ *::$/q}' | grep -v '^ *#' | wc -l` + else - # And the total number of oserver PEs is frontend PEs plus number of history collections - @ NUM_OSERVER_PES=$NUM_FRONTEND_PES + $NUM_HIST_COLLECTIONS + # Okay so we have the total NPEs but we need to guarantee we have at least one extra node for the IO server + # So we use MAX_CPUS_PER_NODE to calculate the worst case scenario. For example, if we have MODEL_NPES=96, then + # the above calculation will give us 105.6, rounded up to 106. But at 48 cores per node, both 96 and 106 result + # in 3 total nodes. So we need to take our total PEs, divide by MAX_CPUS_PER_NODE, round up, then add a node and then + # multiply by MAX_CPUS_PER_NODE to get our new TOTAL_NPES - # Now calculate the number of oserver nodes - set NUM_OSERVER_NODES=`echo "scale=6;($NUM_OSERVER_PES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + set TOTAL_MAX_NODES=`echo "scale=6;($TOTAL_NPES / $MAX_CPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` - # And then the number of backend PEs is the number of history collections divided by the number of oserver nodes - set NUM_BACKEND_PES=`echo "scale=6;($NUM_HIST_COLLECTIONS / $NUM_OSERVER_NODES)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + # Now add one to TOTAL_MAX_NODES + @ TOTAL_MAX_NODES = $TOTAL_MAX_NODES + 1 - # multigroup requires at least two backend pes - if ($NUM_BACKEND_PES < 2) set NUM_BACKEND_PES = 2 + # Now multiply by MAX_CPUS_PER_NODE to get new TOTAL_NPES + @ TOTAL_NPES = $TOTAL_MAX_NODES * $MAX_CPUS_PER_NODE - # Calculate the total number of nodes to request from batch - @ NODES=$NUM_MODEL_NODES + $NUM_OSERVER_NODES + endif else - # Calculate the number of model nodes - set NODES=`echo "scale=6;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` - set NUM_OSERVER_NODES = 0 - set NUM_BACKEND_PES = 0 + set TOTAL_NPES=$MODEL_NPES + + if ( "$NCPUS_PER_NODE" != "ANY" ) then + + # Calculate the number of model nodes + set NODES=`echo "scale=6;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + + endif + endif @ CNV_NPES = $CNV_NX * $CNV_NY @@ -1783,16 +1810,25 @@ else if( $SITE == 'NCCS' ) then setenv PLOT_T "12:00:00" # Wallclock Time for gcm_plot.j setenv ARCHIVE_T "2:00:00" # Wallclock Time for gcm_archive.j - @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` - - setenv RUN_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_run.j - setenv RUN_P "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_run.j - setenv RUN_FP "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_forecast.j - setenv POST_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_post.j - setenv PLOT_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_plot.j + if ("$NCPUS_PER_NODE" == "ANY") then + setenv RUN_Q "SBATCH --constraint=[sky|cas]" # batch queue name for gcm_run.j + setenv RUN_P "SBATCH --ntasks=${TOTAL_NPES}" # PE Configuration for gcm_run.j + setenv RUN_FP "SBATCH --ntasks=${TOTAL_NPES}" # PE Configuration for gcm_forecast.j + setenv POST_P "SBATCH --ntasks=${POST_NPES}" # PE Configuration for gcm_post.j + setenv POST_Q "SBATCH --constraint=[sky|cas]" # batch queue name for gcm_post.j + setenv PLOT_Q "SBATCH --constraint=[sky|cas]" # batch queue name for gcm_plot.j + else + @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + + setenv RUN_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_run.j + setenv RUN_P "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_run.j + setenv RUN_FP "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_forecast.j + setenv POST_P "SBATCH --nodes=${NPCUS} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_post.j + setenv POST_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_post.j + setenv PLOT_Q "SBATCH --constraint=${MODEL}" # batch queue name for gcm_plot.j + endif setenv MOVE_Q "SBATCH --partition=datamove" # batch queue name for gcm_moveplot.j setenv ARCHIVE_Q "SBATCH --partition=datamove" # batch queue name for gcm_archive.j - setenv POST_P "SBATCH --nodes=${NPCUS} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for gcm_post.j setenv PLOT_P "SBATCH --nodes=4 --ntasks=4" # PE Configuration for gcm_plot.j setenv ARCHIVE_P "SBATCH --ntasks=1" # PE Configuration for gcm_archive.j setenv CONVERT_P "SBATCH --ntasks=${CNV_NPES}" # PE Configuration for gcm_convert.j @@ -1896,8 +1932,6 @@ else set CNV_NY = ${NY} # By default on desktop, just ignore IOSERVER for now set USE_IOSERVER = 0 - set NUM_OSERVER_NODES = 0 - set NUM_BACKEND_PES = 0 endif ####################################################################### @@ -2425,8 +2459,6 @@ s/@NX/$NX/g s/@NY/$NY/g s/@USE_SHMEM/$USE_SHMEM/g s/@USE_IOSERVER/$USE_IOSERVER/g -s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g -s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g s/@NUM_READERS/$NUM_READERS/g @@ -2475,6 +2507,8 @@ s?@BACM_1M_?$BACM_1M_?g s?@GFDL_1M_?$GFDL_1M_?g s?@MGB2_2M_?$MGB2_2M_?g +s/@RUN_ANYWHERE/$RUN_ANYWHERE/g +s/@RUN_SPECIFIC/$RUN_SPECIFIC/g EOF # Added FV3/GFDL-MP Specific Parameters @@ -2779,7 +2813,7 @@ if( $OGCM == TRUE ) then sed -i -e "s/DT = [0-9]*\.[0-9]*/DT = $OCEAN_DT/g" \ -e "s/DT_THERM = [0-9]*\.[0-9]*/DT_THERM = $OCEAN_DT/g" $HOMDIR/MOM_override else if ("$OCNMODEL" == "MOM5" ) then - # With MOM5 we need to change dt lines in input.nml to + # With MOM5 we need to change dt lines in input.nml to # use $OCEAN_DT instead. NOTE: This sed assumes integer followed by comma sed -i -e "s/dt_cpld = [0-9]*,/dt_cpld = $OCEAN_DT,/g" \