Add option to destroy job on allocation failed

SpiNNakerManchester · Feb 7, 2024 · 332ad00 · 332ad00
1 parent 22d7f46
commit 332ad00
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 7 deletions.
diff --git a/SpiNNaker-allocserv/src/main/java/uk/ac/manchester/spinnaker/alloc/SpallocProperties.java b/SpiNNaker-allocserv/src/main/java/uk/ac/manchester/spinnaker/alloc/SpallocProperties.java
@@ -590,6 +590,9 @@ public static class AllocatorProperties {
 		/** Name of user that system-generated reports are done by. */
 		private String systemReportUser;
 
+		/** Whether an allocation that fails should immediately fail a job. */
+		private boolean failJobOnFailAllocation;
+
 		/**
 		 * @param period
 		 *            Time between runs of the main allocation algorithm.
@@ -603,17 +606,22 @@ public static class AllocatorProperties {
 		 *            taken out of service.
 		 * @param systemReportUser
 		 *            Name of user that system-generated reports are done by.
+		 * @param failJobOnFailAllocation
+		 *            Fail jobs immediately if the allocation fails, rather than
+		 *            queuing the job.  Avoids system locking up.
 		 */
 		public AllocatorProperties(@DefaultValue("5s") Duration period,
 				@DefaultValue("10000") int importanceSpan,
 				@DefaultValue PriorityScale priorityScale,
 				@DefaultValue("2") int reportActionThreshold,
-				@DefaultValue("") String systemReportUser) {
+				@DefaultValue("") String systemReportUser,
+				@DefaultValue("true") boolean failJobOnFailAllocation) {
 			this.period = period;
 			this.importanceSpan = importanceSpan;
 			this.priorityScale = priorityScale;
 			this.reportActionThreshold = reportActionThreshold;
 			this.systemReportUser = systemReportUser;
+			this.failJobOnFailAllocation = failJobOnFailAllocation;
 		}
 
 		/**
@@ -683,6 +691,15 @@ public String getSystemReportUser() {
 		void setSystemReportUser(String systemReportUser) {
 			this.systemReportUser = systemReportUser;
 		}
+
+		/** @return true if jobs should be failed if allocation fails. */
+		public boolean isFailJobOnFailAllocation() {
+			return failJobOnFailAllocation;
+		}
+
+		void setFailJobOnFailAllocation(boolean failJobOnFailAllocation) {
+			this.failJobOnFailAllocation = failJobOnFailAllocation;
+		}
 	}
 
 	/**

diff --git a/...ker-allocserv/src/main/java/uk/ac/manchester/spinnaker/alloc/allocator/AllocatorTask.java b/...ker-allocserv/src/main/java/uk/ac/manchester/spinnaker/alloc/allocator/AllocatorTask.java
@@ -651,13 +651,23 @@ private Allocations allocate(Connection conn) {
 					continue;
 				}
 				var handled = task.allocate(sql);
-				// If we handled it, delete the request
-				if (handled.size() > 0) {
-					sql.delete.call(task.id);
+
+				if (handled.size() == 0
+						&& allocProps.isFailJobOnFailAllocation()) {
+					destroyJob(conn, task.jobId,
+							"Unable to allocate - please try again");
+					log.debug("allocate for {} (job {}) failed",
+							task.id, task.jobId);
+				} else {
+
+					// If we handled it, delete the request
+					if (handled.size() > 0) {
+						sql.delete.call(task.id);
+					}
+					allocations.addAll(task.jobId, handled);
+					log.debug("allocate for {} (job {}): {}", task.id,
+							task.jobId, handled);
 				}
-				allocations.addAll(task.jobId, handled);
-				log.debug("allocate for {} (job {}): {}", task.id,
-						task.jobId, handled);
 			}
 			/*
 			 * Those tasks which weren't allocated get their importance bumped