Switching memory based health check to make go out of service when ca…

…n't fit one more max job in (#409)

genie-web/src/main/java/com/netflix/genie/web/health/GenieHealthIndicator.java

@@ -40,6 +40,7 @@ public class GenieHealthIndicator implements HealthIndicator {
     private static final String USED_MEMORY_KEY = "usedMemory";
     private static final String AVAILABLE_MEMORY = "availableMemory";
     private static final String AVAILABLE_DEFAULT_JOB_CAPACITY = "availableDefaultJobCapacity";
+    private static final String AVAILABLE_MAX_JOB_CAPACITY = "availableMaxJobCapacity";
 
     private final JobMetricsService jobMetricsService;
     private final JobsProperties jobsProperties;
@@ -67,12 +68,13 @@ public Health health() {
         final int usedMemory = this.jobMetricsService.getUsedMemory();
         final JobsMemoryProperties memoryProperties = this.jobsProperties.getMemory();
         final int availableMemory = memoryProperties.getMaxSystemMemory() - usedMemory;
+        final int maxJobMemory = memoryProperties.getMaxJobMemory();
         final int defaultJobMemory = memoryProperties.getDefaultJobMemory();
 
         final Health.Builder builder;
 
-        // If we can fit one more default job in we're still healthy
-        if (availableMemory >= defaultJobMemory) {
+        // If we can fit one more max job in we're still healthy
+        if (availableMemory >= maxJobMemory) {
             builder = Health.up();
         } else {
             builder = Health.outOfService();
@@ -83,6 +85,7 @@ public Health health() {
             .withDetail(USED_MEMORY_KEY, usedMemory)
             .withDetail(AVAILABLE_MEMORY, availableMemory)
             .withDetail(AVAILABLE_DEFAULT_JOB_CAPACITY, availableMemory % defaultJobMemory)
+            .withDetail(AVAILABLE_MAX_JOB_CAPACITY, availableMemory % maxJobMemory)
             .build();
     }
 }

genie-web/src/main/resources/application.yml

@@ -27,9 +27,6 @@ genie:
   file:
     cache:
       location: file:///tmp/genie/cache
-  health:
-    memory:
-      usedPhysicalMemoryPercent: 90
   jobs:
     forwarding:
       enabled: true
@@ -40,7 +37,6 @@ genie:
       attachments: file:///tmp/genie/attachments/
       jobs: file:///tmp/genie/jobs/
     max:
-      runningJobs: 2
       stdOutSize: 8589934592
       stdErrSize: 8589934592
     memory:
@@ -58,6 +54,9 @@ genie:
     #password:
   redis:
     enabled: false
+  retry:
+    s3:
+      noOfRetries: 5
   security:
     oauth2:
       enabled: false

genie-web/src/test/java/com/netflix/genie/web/health/GenieHealthIndicatorUnitTests.java

@@ -39,6 +39,7 @@ public class GenieHealthIndicatorUnitTests {
 
     private static final int MAX_SYSTEM_MEMORY = 10_240;
     private static final int DEFAULT_JOB_MEMORY = 1_024;
+    private static final int MAX_JOB_MEMORY = 5_120;
 
     private GenieHealthIndicator genieHealthIndicator;
     private JobMetricsService jobMetricsService;
@@ -52,6 +53,7 @@ public void setup() {
         final JobsProperties jobsProperties = new JobsProperties();
         jobsProperties.getMemory().setDefaultJobMemory(DEFAULT_JOB_MEMORY);
         jobsProperties.getMemory().setMaxSystemMemory(MAX_SYSTEM_MEMORY);
+        jobsProperties.getMemory().setMaxJobMemory(MAX_JOB_MEMORY);
 
         this.genieHealthIndicator = new GenieHealthIndicator(this.jobMetricsService, jobsProperties);
     }
@@ -63,7 +65,7 @@ public void setup() {
     public void canGetHealth() {
         Mockito.when(this.jobMetricsService.getNumActiveJobs()).thenReturn(1, 2, 3);
         Mockito.when(this.jobMetricsService.getUsedMemory())
-            .thenReturn(1024, 2048, MAX_SYSTEM_MEMORY - DEFAULT_JOB_MEMORY + 1);
+            .thenReturn(1024, 2048, MAX_SYSTEM_MEMORY - MAX_JOB_MEMORY + 1);
         Assert.assertThat(this.genieHealthIndicator.health().getStatus(), Matchers.is(Status.UP));
         Assert.assertThat(this.genieHealthIndicator.health().getStatus(), Matchers.is(Status.UP));
         Assert.assertThat(this.genieHealthIndicator.health().getStatus(), Matchers.is(Status.OUT_OF_SERVICE));