Allow changing job array max task count

Add ability to change a job array's maximum running task count:
    "scontrol update jobid=# arraytaskthrottle=#"
bug 1863

NEWS

@@ -72,6 +72,8 @@ documents those changes that are of interest to users and administrators.
  -- Fix issue with wrong protocol version when using the srun --no-allocate
     option.
  -- Fix TRES counts on GRES on a clean start of the slurmctld.
+ -- Add ability to change a job array's maximum running task count:
+    "scontrol update jobid=# arraytaskthrottle=#"
 
 * Changes in Slurm 15.08.0
 ==========================

doc/man/man1/scontrol.1

@@ -1,4 +1,4 @@
-.TH scontrol "1" "Slurm Commands" "June 2015" "Slurm Commands"
+.TH scontrol "1" "Slurm Commands" "September 2015" "Slurm Commands"
 
 .SH "NAME"
 scontrol \- Used view and modify Slurm configuration and state.
@@ -479,6 +479,15 @@ Repeat the last command executed.
 Account name to be changed for this job's resource use.
 Value may be cleared with blank data value, "Account=".
 .TP
+\fIArrayTaskThrottle\fP=<count>
+Speciify the maximum number of tasks in a job array that can execute at
+the same time.
+Set the count to zero in order to eliminate any limit.
+The task throttle count for a job array is reported as part of its ArrayTaskId
+field, preceeded with a percent sign.
+For example "ArrayTaskId=1\-10%2" indicates the maximum number of running tasks
+is limited to 2.
+.TP
 \fIBurstBuffer\fP=<spec>
 Burst buffer specification to be changed for this job's resource use.
 Value may be cleared with blank data value, "BurstBuffer=".

src/scontrol/update_job.c

@@ -730,6 +730,18 @@ scontrol_update_job (int argc, char *argv[])
 		if (strncasecmp(tag, "JobId", MAX(taglen, 3)) == 0) {
 			job_msg.job_id_str = val;
 		}
+		else if (strncasecmp(tag, "ArrayTaskThrottle",
+				     MAX(taglen, 10)) == 0) {
+			int throttle;
+			throttle = strtoll(val, (char **) NULL, 10);
+			if (throttle < 0) {
+				error("Invalid ArrayTaskThrottle value");
+				exit_code = 1;
+				return 0;
+			}
+			job_msg.array_inx = val;
+			update_cnt++;
+		}
 		else if (strncasecmp(tag, "Comment", MAX(taglen, 3)) == 0) {
 			job_msg.comment = val;
 			update_cnt++;

src/slurmctld/job_mgr.c

@@ -10992,6 +10992,21 @@ static int _update_job(struct job_record *job_ptr, job_desc_msg_t * job_specs,
 		}
 	}
 
+	if (job_specs->array_inx && job_ptr->array_recs) {
+		int throttle;
+		throttle = strtoll(job_specs->array_inx, (char **) NULL, 10);
+		if (throttle >= 0) {
+			info("update_job: set max_run_tasks to %d for "
+			     "job array %u", throttle, job_ptr->job_id);
+			job_ptr->array_recs->max_run_tasks = throttle;
+		} else {
+			info("update_job: invalid max_run_tasks of %d for "
+			     "job array %u, ignored",
+			     throttle, job_ptr->job_id);
+			error_code = ESLURM_BAD_TASK_COUNT;
+		}
+	}
+
 	if (job_specs->ntasks_per_node != (uint16_t) NO_VAL) {
 		if ((!IS_JOB_PENDING(job_ptr)) || (detail_ptr == NULL))
 			error_code = ESLURM_JOB_NOT_PENDING;