Add properties to disable new job submission

genie-docs/src/docs/asciidoc/_properties.adoc

@@ -256,6 +256,16 @@ to work.
 |1.2
 |no
 
+|genie.jobs.submission.enabled
+|Whether new job submission is enabled (`true`) or disabled (`false`)
+|true
+|yes
+
+|genie.jobs.submission.disabledMessage
+|A message to return to the users when new job submission is disabled
+|Job submission is currently disabled. Please try again later.
+|yes
+
 |genie.leader.enabled
 |Whether this node should be the leader of the cluster or not. Should only be used if leadership is not being
 determined by Zookeeper or other mechanism via Spring

genie-web/src/main/java/com/netflix/genie/web/apis/rest/v3/controllers/JobRestController.java

@@ -19,6 +19,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.ByteStreams;
 import com.netflix.genie.common.dto.JobMetadata;
 import com.netflix.genie.common.dto.JobRequest;
@@ -28,6 +29,7 @@
 import com.netflix.genie.common.exceptions.GenieException;
 import com.netflix.genie.common.exceptions.GenieNotFoundException;
 import com.netflix.genie.common.exceptions.GenieServerException;
+import com.netflix.genie.common.exceptions.GenieServerUnavailableException;
 import com.netflix.genie.common.internal.exceptions.unchecked.GenieJobNotFoundException;
 import com.netflix.genie.common.internal.jobs.JobConstants;
 import com.netflix.genie.common.internal.util.GenieHostInfo;
@@ -61,6 +63,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.core.env.Environment;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.web.PageableDefault;
@@ -124,6 +127,13 @@
 @RequestMapping(value = "/api/v3/jobs")
 @Slf4j
 public class JobRestController {
+    @VisibleForTesting
+    static final String JOB_SUBMISSION_ENABLED_PROPERTY_KEY = "genie.jobs.submission.enabled";
+    @VisibleForTesting
+    static final String JOB_SUBMISSION_DISABLED_MESSAGE_KEY = "genie.jobs.submission.disabledMessage";
+    @VisibleForTesting
+    static final String JOB_SUBMISSION_DISABLED_DEFAULT_MESSAGE
+        = "Job submission is currently disabled. Please try again later.";
     private static final String TRANSFER_ENCODING_HEADER = "Transfer-Encoding";
     private static final String FORWARDED_FOR_HEADER = "X-Forwarded-For";
     private static final String NAME_HEADER_COOKIE = "cookie";
@@ -147,6 +157,7 @@ public class JobRestController {
     private final JobsProperties jobsProperties;
     private final AgentRoutingService agentRoutingService;
     private final JobPersistenceService jobPersistenceService;
+    private final Environment environment;
 
     // Metrics
     private final Counter submitJobWithoutAttachmentsRate;
@@ -166,6 +177,7 @@ public class JobRestController {
      * @param registry                  The metrics registry to use
      * @param jobPersistenceService     Job persistence service
      * @param agentRoutingService       Agent routing service
+     * @param environment               The application environment to pull dynamic properties from
      */
     @Autowired
     @SuppressWarnings("checkstyle:parameternumber")
@@ -180,7 +192,8 @@ public JobRestController(
         final JobsProperties jobsProperties,
         final MeterRegistry registry,
         final JobPersistenceService jobPersistenceService,
-        final AgentRoutingService agentRoutingService
+        final AgentRoutingService agentRoutingService,
+        final Environment environment
     ) {
         this.jobCoordinatorService = jobCoordinatorService;
         this.jobSearchService = jobSearchService;
@@ -199,6 +212,7 @@ public JobRestController(
         this.jobsProperties = jobsProperties;
         this.agentRoutingService = agentRoutingService;
         this.jobPersistenceService = jobPersistenceService;
+        this.environment = environment;
 
         // Set up the metrics
         this.submitJobWithoutAttachmentsRate = registry.counter("genie.api.v3.jobs.submitJobWithoutAttachments.rate");
@@ -264,6 +278,19 @@ private ResponseEntity<Void> handleSubmitJob(
         @Nullable final String userAgent,
         final HttpServletRequest httpServletRequest
     ) throws GenieException {
+        // TODO: This is quick and dirty and we may want to handle it better overall for the system going forward
+        //       e.g. should it be in an filter that we can do for more APIs?
+        //            should value be cached rather than checking every time?
+        if (!this.environment.getProperty(JOB_SUBMISSION_ENABLED_PROPERTY_KEY, Boolean.class, true)) {
+            // Job Submission is disabled
+            throw new GenieServerUnavailableException(
+                this.environment.getProperty(
+                    JOB_SUBMISSION_DISABLED_MESSAGE_KEY,
+                    JOB_SUBMISSION_DISABLED_DEFAULT_MESSAGE
+                )
+            );
+        }
+
         // get client's host from the context
         final String localClientHost;
         if (StringUtils.isNotBlank(clientHost)) {

genie-web/src/test/java/com/netflix/genie/web/apis/rest/v3/controllers/JobRestControllerTest.java

@@ -18,9 +18,11 @@
 package com.netflix.genie.web.apis.rest.v3.controllers;
 
 import com.google.common.collect.Sets;
+import com.netflix.genie.common.dto.JobRequest;
 import com.netflix.genie.common.dto.JobStatus;
 import com.netflix.genie.common.exceptions.GenieException;
 import com.netflix.genie.common.exceptions.GenieNotFoundException;
+import com.netflix.genie.common.exceptions.GenieServerUnavailableException;
 import com.netflix.genie.common.internal.exceptions.unchecked.GenieJobNotFoundException;
 import com.netflix.genie.common.internal.util.GenieHostInfo;
 import com.netflix.genie.web.agent.services.AgentRoutingService;
@@ -47,11 +49,13 @@
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
+import org.assertj.core.api.Assertions;
 import org.hamcrest.Matchers;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
+import org.springframework.core.env.Environment;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
@@ -93,6 +97,7 @@ public class JobRestControllerTest {
     private RestTemplate restTemplate;
     private JobDirectoryServerService jobDirectoryServerService;
     private JobsProperties jobsProperties;
+    private Environment environment;
 
     private JobRestController controller;
 
@@ -108,6 +113,7 @@ public void setup() {
         this.restTemplate = Mockito.mock(RestTemplate.class);
         this.jobDirectoryServerService = Mockito.mock(JobDirectoryServerService.class);
         this.jobsProperties = JobsProperties.getJobsPropertiesDefaults();
+        this.environment = Mockito.mock(Environment.class);
 
         final MeterRegistry registry = Mockito.mock(MeterRegistry.class);
         final Counter counter = Mockito.mock(Counter.class);
@@ -125,7 +131,8 @@ public void setup() {
             this.jobsProperties,
             registry,
             this.jobPersistenceService,
-            this.agentRoutingService
+            this.agentRoutingService,
+            this.environment
         );
     }
 
@@ -838,7 +845,8 @@ public void canHandleForwardJobOutputRequestWithSuccess() throws IOException, Se
             this.jobsProperties,
             registry,
             this.jobPersistenceService,
-            this.agentRoutingService
+            this.agentRoutingService,
+            this.environment
         );
         jobController.getJobOutput(jobId, forwardedFrom, request, response);
 
@@ -857,6 +865,40 @@ public void canHandleForwardJobOutputRequestWithSuccess() throws IOException, Se
             );
     }
 
+    /**
+     * Make sure when job submission is disabled it won't run the job and will return the proper error message.
+     *
+     * @throws GenieException On unexpected exception
+     */
+    @Test
+    public void whenJobSubmissionIsDisabledItThrowsCorrectError() throws GenieException {
+        final String errorMessage = UUID.randomUUID().toString();
+        try {
+            Mockito.when(
+                this.environment.getProperty(
+                    JobRestController.JOB_SUBMISSION_ENABLED_PROPERTY_KEY,
+                    Boolean.class,
+                    true
+                )
+            ).thenReturn(false);
+            Mockito.when(
+                this.environment.getProperty(
+                    JobRestController.JOB_SUBMISSION_DISABLED_MESSAGE_KEY,
+                    JobRestController.JOB_SUBMISSION_DISABLED_DEFAULT_MESSAGE
+                )
+            ).thenReturn(errorMessage);
+            this.controller.submitJob(
+                Mockito.mock(JobRequest.class),
+                null,
+                null,
+                Mockito.mock(HttpServletRequest.class)
+            );
+            Assert.fail("Expected exception not thrown");
+        } catch (final GenieServerUnavailableException e) {
+            Assertions.assertThat(e.getMessage()).isEqualTo(errorMessage);
+        }
+    }
+
     private ResourceAssemblers createMockResourceAssembler() {
         return new ResourceAssemblers(
             Mockito.mock(ApplicationResourceAssembler.class),