config_batch.xml

<?xml version="1.0"?>
<config_batch version="2.1">
  <!--
     File:    config_batch.xml
     Purpose: abstract out the parts of run scripts that are different, and use this configuration to
     create cesm run scripts from a single template.

     batch_system:     the batch system type and version
     batch_query:      the batch query command for each batch system.
     batch_redirect:   Whether a redirect character is needed to submit jobs.
     batch_directive:  The string that prepends a batch directive for the batch system.
     jobid_pattern:    A perl regular expression used to filter out the returned job id from a
                       queue submission.

 ===============================================================
 batch_system
 ===============================================================
 The batch_system and associated tags are meant for configuring batch systems and
 queues across machines.  The batch_system tag denotes the name for a particular
 batch system, these can either be shared between one or more machines, or can be
 defined for a specific machine if need be.

 Machine specific entries take precidence over generic entries, directives are appended

 queues:
 one or more queues can be defined per batch_system. if the attribute default="true"
 is used, then that queue will be used by default. Alternatively, multiple queues can
 be used.  The following variables can be used to choose a queue :
 walltimemin: Giving the minimum amount of walltime for the queue.
 walltimemax: The maximum amount of walltime for a queue.
 nodemin:      The minimum node count required to use this queue.
 nodemax:      The maximum node count required to use this queue.
 jobmin:      The minimum task count required to use this queue. This should only rarely be used to select queues that only use a fraction of a node. This cannot be used in conjuction with nodemin.
 jobmax:      The maximum task count required to use this queue. This should only rarely be used to select queues that only use a fraction of a node. This cannot be used in conjuction with nodemax.
    -->
  <batch_system type="template" >
    <batch_query args=""></batch_query>
    <batch_submit></batch_submit>
    <batch_redirect></batch_redirect>
    <batch_directive></batch_directive>
    <directives>
      <directive></directive>
    </directives>
  </batch_system>

  <batch_system type="none" >
    <batch_query args=""></batch_query>
    <batch_submit></batch_submit>
    <batch_redirect></batch_redirect>
    <batch_directive></batch_directive>
    <directives>
      <directive></directive>
    </directives>
  </batch_system>

  <batch_system MACH="aws-hpc6a" type="slurm">
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
    </submit_args>
    <queues>
      <queue walltimemax="144:00:00" nodemin="1" nodemax="96">regular</queue>
    </queues>
  </batch_system>

  <batch_system type="cobalt" >
    <batch_query>qstat</batch_query>
    <batch_submit>qsub</batch_submit>
    <batch_cancel>qdel</batch_cancel>
    <batch_env>-v</batch_env>
    <batch_directive></batch_directive>
    <jobid_pattern>(\d+)</jobid_pattern>
    <depend_string> --dependencies</depend_string>
    <walltime_format>%H:%M:%s</walltime_format>
    <batch_mail_flag>-M</batch_mail_flag>
    <batch_mail_type_flag></batch_mail_type_flag>
    <batch_mail_type></batch_mail_type>
    <submit_args>
      <arg flag="--cwd" name="CASEROOT"/>
      <arg flag="-A" name="PROJECT"/>
      <arg flag="-t" name="JOB_WALLCLOCK_TIME"/>
      <!-- space required at beginning of name -->
      <arg flag="-n" name=" $TOTALPES / $MAX_MPITASKS_PER_NODE"/>
      <arg flag="-q" name="JOB_QUEUE"/>
      <arg flag="--mode script"/>
    </submit_args>
  </batch_system>

  <batch_system type="cobalt_theta" >
    <batch_query>qstat</batch_query>
    <batch_submit>qsub</batch_submit>
    <batch_cancel>qdel</batch_cancel>
    <batch_env>--env</batch_env>
    <batch_directive>#COBALT</batch_directive>
    <jobid_pattern>(\d+)</jobid_pattern>
    <depend_string> --dependencies</depend_string>
    <batch_mail_flag>-M</batch_mail_flag>
    <batch_mail_type_flag></batch_mail_type_flag>
    <batch_mail_type></batch_mail_type>
    <submit_args>
      <arg flag="-A" name="PROJECT"/>
      <arg flag="-t" name="JOB_WALLCLOCK_TIME"/>
      <arg flag="-n" name=" $TOTALPES/$MAX_MPITASKS_PER_NODE"/>
      <arg flag="-q" name="JOB_QUEUE"/>
      <arg flag="--mode script"/>
    </submit_args>
  </batch_system>

  <batch_system type="lsf">
    <batch_query args=" -w" >bjobs</batch_query>
    <batch_submit>bsub</batch_submit>
    <batch_cancel>bkill</batch_cancel>
    <batch_redirect>&lt;</batch_redirect>
    <batch_env> </batch_env>
    <batch_directive>#BSUB</batch_directive>
    <jobid_pattern>&lt;(\d+)&gt;</jobid_pattern>
    <depend_string> -w 'done(jobid)'</depend_string>
    <depend_allow_string> -w 'ended(jobid)'</depend_allow_string>
    <depend_separator>&amp;&amp;</depend_separator>
    <walltime_format>%H:%M</walltime_format>
    <batch_mail_flag>-u</batch_mail_flag>
    <batch_mail_type_flag></batch_mail_type_flag>
    <batch_mail_type></batch_mail_type>
    <directives>
      <directive                       > -J {{ job_id }} </directive>
      <directive                       > -n {{ total_tasks }} </directive>
      <directive                       > -W $JOB_WALLCLOCK_TIME </directive>
      <directive default="cesm.stdout" > -o {{ job_id }}.%J  </directive>
      <directive default="cesm.stderr" > -e {{ job_id }}.%J  </directive>
    </directives>
  </batch_system>

  <batch_system type="pbs" >
    <batch_query args="-f" >qstat</batch_query>
    <batch_submit>qsub </batch_submit>
    <batch_cancel>qdel</batch_cancel>
    <batch_env>-v</batch_env>
    <batch_directive>#PBS</batch_directive>
    <jobid_pattern>^(\S+)$</jobid_pattern>
    <depend_string> -W depend=afterok:jobid</depend_string>
    <depend_allow_string> -W depend=afterany:jobid</depend_allow_string>
    <depend_separator>:</depend_separator>
    <walltime_format>%H:%M:%S</walltime_format>
    <batch_mail_flag>-M</batch_mail_flag>
    <batch_mail_type_flag>-m</batch_mail_type_flag>
    <batch_mail_type>, bea, b, e, a</batch_mail_type>
    <submit_args>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="-l walltime=" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-A" name="$PROJECT"/>
    </submit_args>
    <directives>
      <directive>-N {{ job_id }}</directive>
      <directive default="n"> -r {{ rerunnable }} </directive>
      <!-- <directive> -j oe {{ job_id }} </directive> -->
      <directive> -j oe </directive>
      <directive> -V </directive>
    </directives>
  </batch_system>

  <batch_system type="sge" >
    <batch_query args="-j">qstat</batch_query>
    <batch_submit>qsub </batch_submit>
    <batch_cancel>qdel</batch_cancel>
    <batch_env>-v</batch_env>
    <batch_directive>#$ </batch_directive>
    <jobid_pattern>(\d+)</jobid_pattern>
    <depend_string> -hold_jid jobid</depend_string>
    <depend_separator> , </depend_separator>
    <walltime_format>%H:%M:%S</walltime_format>
    <batch_mail_flag>-M</batch_mail_flag>
    <batch_mail_type_flag>-m</batch_mail_type_flag>
    <batch_mail_type>, bea, b, e, a, n, bes</batch_mail_type>
    <submit_args>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="-P" name="$PROJECT"/>
      <arg flag="-l h_rt=" name="$JOB_WALLCLOCK_TIME"/>
    </submit_args>
    <directives>
      <directive> -N {{ job_id }}</directive>
      <directive> -V </directive>
      <directive> -l nodes={{ num_nodes }},ppn={{ tasks_per_node }},tpp={{ thread_count }}</directive>
    </directives>
  </batch_system>

  <batch_system type="slurm" >
    <batch_query per_job_arg="-j">squeue</batch_query>
    <batch_cancel>scancel</batch_cancel>
    <batch_directive>#SBATCH</batch_directive>
    <jobid_pattern>(\d+)$</jobid_pattern>
    <depend_string> --dependency=afterok:jobid</depend_string>
    <depend_allow_string> --dependency=afterany:jobid</depend_allow_string>
    <depend_separator>,</depend_separator>
    <walltime_format>%H:%M:%S</walltime_format>
    <batch_mail_flag>--mail-user</batch_mail_flag>
    <batch_mail_type_flag>--mail-type</batch_mail_type_flag>
    <batch_mail_type>none, all, begin, end, fail</batch_mail_type>
    <directives>
      <directive> --job-name={{ job_id }}</directive>
      <directive> --nodes={{ num_nodes }}</directive>
      <directive> --ntasks-per-node={{ tasks_per_node }}</directive>
      <directive> --output={{ job_id }}   </directive>
    </directives>
  </batch_system>

  <batch_system MACH="aleph" type="pbs" >
    <directives>
      <directive>-l nodes={{ num_nodes }}</directive>
      <directive>-q iccp</directive>
    </directives>
    <queues>
      <queue walltimemax="24:00:00" default="true" >iccp</queue>
    </queues>
  </batch_system>

  <batch_system MACH="arc4" type="sge">
    <queues>
      <queue walltimemax="48:00:00" nodemin="1" nodemax="149" default="true" >40core-192G.q</queue>
      <queue walltimemax="48:00:00" nodemax="1">planet.q</queue>
    </queues>
  </batch_system>

  <batch_system MACH="archer2" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
      <arg flag="--export=ALL"/>
    </submit_args>

    <directives queue="standard">
      <directive>--partition=standard</directive>
      <directive>--qos=standard</directive>
      <directive>--cpus-per-task={{ thread_count }}</directive>
    </directives>

    <directives queue="short">
      <directive>--partition=standard</directive>
      <directive>--qos=short</directive>
      <directive>--cpus-per-task={{ thread_count }}</directive>
    </directives>

    <directives queue="serial">
      <directive>--partition=serial</directive>
      <directive>--qos=serial</directive>
    </directives>

    <!-- following will need updating as system develops -->
    <queues>
      <queue walltimemax="24:00:00" nodemin="1" nodemax="1024" default="true">standard</queue>
      <!-- <queue walltimemin="24:00:00" walltimemax="48:00:00" nodemin="1" nodemax="64" >long</queue> -->
      <queue walltimemax="00:20:00" nodemin="1" nodemax="32">short</queue>
      <queue walltimemax="24:00:00" nodemin="1" nodemax="1">serial</queue>
    </queues>
  </batch_system>

  <!-- athena is lsf -->
  <batch_system MACH="athena" type="lsf">
    <batch_env>none</batch_env>
    <submit_args>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="-W" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-P" name="$PROJECT"/>
    </submit_args>
    <directives>
      <directive                       > -R "span[ptile={{ tasks_per_node }}]"</directive>
      <directive                       > -N  </directive>
      <directive default="poe"         > -a {{ poe }} </directive>
    </directives>
    <queues>
      <queue walltimemin="00:00" walltimemax="02:00">poe_short</queue>
      <queue walltimemin="02:00" walltimemax="04:00" default="true">poe_medium</queue>
      <queue walltimemin="04:00" walltimemax="08:00">poe_long</queue>
    </queues>
  </batch_system>

  <!-- bluewaters is PBS -->
  <batch_system MACH="bluewaters" type="pbs" >
    <jobid_pattern>(\d+.bw)$</jobid_pattern>
    <directives>
      <directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}:xe</directive>
      <directive default="/bin/bash" > -S {{ shell }} </directive>
    </directives>
    <queues>
      <queue walltimemax="24:00:00">normal</queue>
      <queue walltimemax="00:30:00" nodemin="1" nodemax="16" default="true">debug</queue>
    </queues>
  </batch_system>

  <batch_system MACH="cheyenne" type="pbs">
    <directives queue="regular">
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>

    <directives queue="premium">
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>

    <directives queue="economy">
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>

    <directives queue="share">
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select=1:mpiprocs={{ total_tasks }}:ompthreads={{ thread_count }}</directive>
    </directives>

    <!-- Unknown queues use the batch directives for the regular queue -->
    <unknown_queue_directives>regular</unknown_queue_directives>

    <queues>
      <queue walltimemax="12:00:00" nodemin="1" nodemax="4032">regular</queue>
      <queue walltimemax="12:00:00" nodemin="1" nodemax="4032">premium</queue>
      <queue default="true" walltimemax="06:00:00" jobmin="1" jobmax="18">share</queue>
      <queue walltimemax="12:00:00" nodemin="1" nodemax="4032">economy</queue>
    </queues>
  </batch_system>

  <batch_system type="slurm" MACH="comet">
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
    </submit_args>
    <queues>
      <queue walltimemax="48:00:00" nodemin="1" nodemax="72">compute</queue>
      <queue default="true" walltimemax="48:00:00" jobmin="1" jobmax="1">shared</queue>
      <queue walltimemax="00:30:00" nodemin="1" nodemax="2">debug</queue>
    </queues>
  </batch_system>

  <batch_system type="slurm" MACH="constance">
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
  </batch_system>

  <batch_system MACH="daint" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue default="true">default</queue>
    </queues>
  </batch_system>

  <batch_system MACH="derecho" type="pbs" >
    <batch_submit>qsub</batch_submit>
    <submit_args>
      <arg flag="-l job_priority=" name="$JOB_PRIORITY"/>
    </submit_args>
    <directives>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>
    <queues>
      <queue walltimemax="12:00:00" nodemin="1" nodemax="2488" default="true">main</queue>
      <queue walltimemax="1:00:00" nodemin="1" nodemax="2">develop</queue>
    </queues>
  </batch_system>


  <batch_system MACH="eastwind" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue nodemin="1" nodemax="833" default="true">batch</queue>
    </queues>
  </batch_system>

  <batch_system MACH="edison" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemax="36:00:00" nodemin="1" nodemax="2712" >regular</queue>
      <queue walltimemax="00:30:00" nodemin="1" nodemax="256" default="true">debug</queue>
    </queues>
  </batch_system>

  <!-- euler2 is LSF -->
  <batch_system MACH="euler2" type="lsf" >
    <directives>
      <directive> -R "select[model==XeonE5_2680v3]" </directive>
    </directives>
    <queues>
      <queue walltimemax="23:59:00" default="true">normal.24h</queue>
      <queue walltimemax="03:59:00" >normal.4h</queue>
    </queues>
  </batch_system>

  <!-- euler3 is LSF -->
  <batch_system MACH="euler3" type="lsf" >
    <directives>
      <directive> -R "span[ptile=4] select[model==XeonE3_1585Lv5]" </directive>
    </directives>
    <queues>
      <queue walltimemax="23:59:00" default="true">normal.24h</queue>
      <queue walltimemax="03:59:00" >normal.4h</queue>
    </queues>
  </batch_system>

  <!-- euler4 is LSF -->
  <batch_system MACH="euler4" type="lsf" >
    <directives>
      <directive> -R "select[model==XeonGold_6150]" </directive>
    </directives>
    <queues>
      <queue walltimemax="23:59:00" default="true">normal.24h</queue>
      <queue walltimemax="03:59:00" >normal.4h</queue>
    </queues>
  </batch_system>
  <!-- gaea is PBS -->
  <batch_system MACH="gaea" type="pbs" >
    <directives>
      <directive>-A cpo</directive>
      <directive>-l {{ partition }}</directive>
      <directive>-l size={{ mppsize }}</directive>
      <directive>-E </directive>
      <directive>-d $RUNDIR</directive>
      <directive>-o $RUNDIR/$CASE.out </directive>
      <directive>-S /bin/bash  </directive>
    </directives>
    <queues>
      <queue walltimemax="01:00:00" nodemin="1" nodemax="35">debug</queue>
      <queue walltimemax="24:00:00" nodemin="861" nodemax="4166" default="true">batch</queue>
    </queues>
  </batch_system>

  <batch_system MACH="grace" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
    </submit_args>
    <directives>
      <directive> --mem=360GB </directive>
    </directives>
    <queues>
      <queue walltimemax="02:00:00" nodemin="1" nodemax="32" default="true">short</queue>
      <queue walltimemax="24:00:00" nodemin="1" nodemax="128">medium</queue>
      <queue walltimemax="168:00:00" nodemin="1" nodemax="64" >long</queue>
      <queue walltimemax="504:00:00" nodemin="1" nodemax="32" >xlong</queue>
    </queues>
  </batch_system>

  <batch_system MACH="gust" type="pbs" >
    <batch_submit>qsub</batch_submit>
    <directives>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>
    <queues>
      <queue walltimemax="4:00:00" nodemin="1" nodemax="8" default="true">main</queue>
      <queue walltimemax="1:00:00" nodemin="9" nodemax="16" >bigcpu</queue>
    </queues>
  </batch_system>


  <!-- hobart is PBS -->
  <batch_system type="pbs" MACH="hobart" >
    <directives>
      <directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemax="02:00:00"   strict="true" nodemin="1"  nodemax="8">short</queue>
      <queue walltimemax="08:00:00"   strict="true" nodemin="1"  nodemax="6" default="true">medium</queue>
      <queue walltimemax="40:00:00"   strict="true" nodemin="1"  nodemax="8">long</queue>
      <queue walltimemax="80:00:00"   strict="true" nodemin="1"  nodemax="8">verylong</queue>
      <queue walltimemax="32:00:00"   strict="true" nodemax="16" nodemin="1">overnight</queue>
      <queue walltimemax="3000:00:00" strict="true" nodemax="32" nodemin="1">monster</queue>
    </queues>
  </batch_system>

  <batch_system type="pbs" MACH="izumi" >
    <batch_submit>ssh izumi cd $CASEROOT ; qsub</batch_submit>
    <jobid_pattern>(\d+.izumi.cgd.ucar.edu)$</jobid_pattern>
    <directives>
      <directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemax="02:00:00"   strict="true" nodemin="1"  nodemax="8">short</queue>
      <queue walltimemax="08:00:00"   strict="true" nodemin="1"  nodemax="6" default="true">medium</queue>
      <queue walltimemax="40:00:00"   strict="true" nodemin="1"  nodemax="8">long</queue>
      <queue walltimemax="120:00:00"   strict="true" nodemin="1"  nodemax="8">verylong</queue>
      <queue walltimemax="32:00:00"   strict="true" nodemax="14" nodemin="1">overnight</queue>
      <queue walltimemax="3000:00:00" strict="true" nodemax="14" nodemin="1">monster</queue>
    </queues>
  </batch_system>

  <batch_system MACH="laramie" type="pbs">
    <directives>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
    </directives>
    <queues>
      <queue default="true" walltimemax="12:00" nodemin="1" nodemax="72">regular</queue>
    </queues>
  </batch_system>

  <batch_system MACH="lawrencium-lr3" type="slurm">
    <batch_submit>sbatch</batch_submit>
    <directives>
      <directive>--qos=lr_normal</directive>
      <directive>--partition=lr3</directive>
      <directive>--account={{ project }}</directive>
      <directive>--ntasks-per-node={{ tasks_per_node }}</directive>
    </directives>
    <queues>
      <queue walltimemin="00:00:00" walltimemax="72:00:00" nodemin="1" nodemax="64" default="true">lr3</queue>
    </queues>
  </batch_system>

  <batch_system MACH="lawrencium-lr2" type="slurm">
    <batch_submit>sbatch</batch_submit>
    <directives>
       <directive>--qos=lr_normal</directive>
       <directive>--partition=lr2</directive>
       <directive>--account={{ project }}</directive>
       <directive>--ntasks-per-node={{ tasks_per_node }}</directive>
    </directives>
    <queues>
      <queue walltimemin="00:00:00" walltimemax="72:00:00" nodemin="1" nodemax="64" default="true">lr2</queue>
    </queues>
  </batch_system>

  <batch_system MACH="lonestar5" type="slurm" >
    <batch_submit>ssh login1.ls5.tacc.utexas.edu cd $CASEROOT ; sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemax="48:00:00" nodemin="1" nodemax="171" default="true">normal</queue>
      <queue walltimemax="24:00:00" nodemin="160" nodemax="342">large</queue>
      <queue walltimemax="02:00:00" nodemin="1" nodemax="11" >development</queue>
    </queues>
  </batch_system>

  <batch_system MACH="mira" type="cobalt">
    <queues>
      <queue walltimemax="06:00:00" nodemin="1" nodemax="12288" default="true">default</queue>
    </queues>
  </batch_system>

  <!-- modex is PBS -->
  <batch_system MACH="modex" type="pbs">
    <directives>
      <directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemax="36:00:00" default="true">batch</queue>
    </queues>
  </batch_system>

  <batch_system MACH="olympus" type="slurm">
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemin="0" walltimemax="00:59:00" nodemin="0" nodemax="312" default="true">queue</queue>
    </queues>
  </batch_system>

  <batch_system MACH="perlmutter" type="slurm" >
    <batch_submit>sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME" />
      <arg flag="-q" name="$JOB_QUEUE" />
      <arg flag="--account" name="$PROJECT" />
    </submit_args>
    <directives>
      <directive> --constraint=cpu</directive>
    </directives>
    <queues>
      <queue walltimemax="24:00:00" nodemin="1" nodemax="3072">regular</queue>
      <queue walltimemax="00:30:00" nodemin="1" nodemax="8" default="true">debug</queue>
    </queues>
  </batch_system>



  <!-- NAS pleiades machines -->
  <batch_system type="pbs" MACH="pleiades-bro" >
    <directives>
      <directive>-W group_list=$PROJECT</directive>
      <directive>-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=bro</directive>
      <directive>-l place=scatter:excl</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemin="" walltimemax="08:00:00" nodemin="0" nodemax="357" default="true">normal</queue>
    </queues>
  </batch_system>

  <batch_system type="pbs" MACH="pleiades-has" >
    <directives>
      <directive>-W group_list=$PROJECT</directive>
      <directive>-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=has</directive>
      <directive>-l place=scatter:excl</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemin="" walltimemax="08:00:00" nodemin="0" nodemax="357" default="true">normal</queue>
    </queues>
  </batch_system>

  <batch_system type="pbs" MACH="pleiades-ivy" >
    <directives>
      <directive>-W group_list=$PROJECT </directive>
      <directive>-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=ivy</directive>
      <directive>-l place=scatter:excl</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemin="" walltimemax="08:00:00" nodemin="0" nodemax="500" default="true">normal</queue>
    </queues>
  </batch_system>

  <batch_system type="pbs" MACH="pleiades-san" >
    <directives>
      <directive>-W group_list=$PROJECT </directive>
      <directive>-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=san</directive>
      <directive>-l place=scatter:excl</directive>
      <directive default="/bin/bash" > -S {{ shell }}  </directive>
    </directives>
    <queues>
      <queue walltimemin="" walltimemax="08:00:00" nodemin="0" nodemax="624" default="true">normal</queue>
    </queues>
  </batch_system>

  <batch_system MACH="stampede3-spr" type="slurm" >
    <batch_submit>ssh login1 cd $CASEROOT ; sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemax="24:00:00" nodemin="1" nodemax="16" default="true">spr</queue>
    </queues>
  </batch_system>

  <batch_system MACH="stampede2-skx" type="slurm" >
    <batch_submit>ssh login1 cd $CASEROOT ; sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemax="48:00:00" nodemin="1" nodemax="256" default="true">skx</queue>
      <queue walltimemax="02:00:00" nodemin="1" nodemax="4" >skx-dev</queue>
    </queues>
  </batch_system>

  <batch_system MACH="stampede2-knl" type="slurm" >
    <batch_submit>ssh login2 cd $CASEROOT ; sbatch</batch_submit>
    <submit_args>
      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-p" name="$JOB_QUEUE"/>
      <arg flag="--account" name="$PROJECT"/>
    </submit_args>
    <queues>
      <queue walltimemax="48:00:00" nodemin="1" nodemax="256" >icx</queue>
    </queues>
  </batch_system>

  <batch_system type="pbs" MACH="theia" >
    <submit_args>
      <arg flag="-l procs=" name=" $TOTALPES"/>
    </submit_args>
    <queues>
      <queue walltimemax="01:00:00" nodemin="1" nodemax="35">batch</queue>
    </queues>
  </batch_system>

  <batch_system MACH="theta" type="cobalt_theta">
    <queues>
      <queue walltimemax="00:60:00" nodemin="1" nodemax="50" default="true">default</queue>
    </queues>
  </batch_system>

  <batch_system MACH="zeus" type="lsf">
    <batch_env>-env</batch_env>
    <submit_args>
      <arg flag="-q" name="$JOB_QUEUE"/>
      <arg flag="-W" name="$JOB_WALLCLOCK_TIME"/>
      <arg flag="-P" name="$PROJECT"/>
    </submit_args>
    <directives>
      <directive> -R "span[ptile={{ tasks_per_node }}]"</directive>
    </directives>
    <queues>
      <queue walltimemin="00:00" walltimemax="02:00" default="true">p_short</queue>
      <queue walltimemin="02:00" walltimemax="04:00">p_medium</queue>
      <queue walltimemin="04:00" walltimemax="08:00">p_long</queue>
    </queues>
  </batch_system>
</config_batch>