Merge pull request #313 from ajoymajumdar/develop

Added health indicator for physical memory used on the instance.

build.gradle

@@ -54,6 +54,7 @@ subprojects {
     apply plugin: 'nebula.optional-base'
     apply plugin: 'nebula.provided-base'
     apply plugin: 'com.gorylenko.gradle-git-properties'
+    apply plugin: 'groovy'
 
     group = "com.netflix.${githubProjectName}"
 
@@ -114,6 +115,9 @@ subprojects {
 
         testRuntime("javax.el:javax.el-api")
         testRuntime("org.glassfish:javax.el")
+
+        testCompile 'org.spockframework:spock-core'
+        testCompile 'org.spockframework:spock-spring'
     }
 
     task unitTests(type: Test, group: 'verification') {

genie-web/src/main/java/com/netflix/genie/web/configs/ServicesConfig.java

@@ -51,6 +51,7 @@
 import com.netflix.genie.core.services.impl.MailServiceImpl;
 import com.netflix.genie.core.services.impl.RandomizedClusterLoadBalancerImpl;
 import com.netflix.spectator.api.Registry;
+import com.sun.management.OperatingSystemMXBean;
 import org.apache.commons.exec.Executor;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
@@ -63,6 +64,7 @@
 import org.springframework.core.task.AsyncTaskExecutor;
 import org.springframework.mail.javamail.JavaMailSender;
 
+import java.lang.management.ManagementFactory;
 import java.util.List;
 
 /**
@@ -341,4 +343,14 @@ public JobCoordinatorService jobCoordinatorService(
             eventPublisher
         );
     }
+
+    /**
+     * Returns the operating system MX bean.
+     *
+     * @return The operating system MX bean.
+     */
+    @Bean
+    public OperatingSystemMXBean operatingSystemMXBean() {
+        return (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+    }
 }

genie-web/src/main/java/com/netflix/genie/web/health/MemoryHealthIndicator.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2016 Netflix, Inc.
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package com.netflix.genie.web.health;
+
+import com.sun.management.OperatingSystemMXBean;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+/**
+ * Health indicator for system memory usage. Also mark the service out-of-service if it crosses the threshold.
+ * @author amajumdar
+ * @since 3.0.0
+ */
+@Component
+@Slf4j
+public class MemoryHealthIndicator implements HealthIndicator {
+    private static final String FREE_PHYSICAL_MEMORY_SIZE = "freePhysicalMemorySize";
+    private static final String TOTAL_PHYSICAL_MEMORY_SIZE = "totalPhysicalMemorySize";
+
+    private final double maxUsedPhysicalMemoryPercentage;
+    private final OperatingSystemMXBean operatingSystemMXBean;
+
+    /**
+     * Constructor.
+     *
+     * @param maxUsedPhysicalMemoryPercentage The maximum physical memory threshold
+     * @param operatingSystemMXBean MX bean for operating system
+     */
+    @Autowired
+    public MemoryHealthIndicator(
+            @Value("${genie.health.memory.usedPhysicalMemoryPercent:90}")
+            final double maxUsedPhysicalMemoryPercentage,
+            final OperatingSystemMXBean operatingSystemMXBean
+    ) {
+        this.maxUsedPhysicalMemoryPercentage = maxUsedPhysicalMemoryPercentage;
+        this.operatingSystemMXBean = operatingSystemMXBean;
+    }
+
+    @Override
+    public Health health() {
+        final double freePhysicalMemorySize = (double) operatingSystemMXBean.getFreePhysicalMemorySize();
+        final double totalPhysicalMemorySize = (double) operatingSystemMXBean.getTotalPhysicalMemorySize();
+        final double usedPhysicalMemoryPercentage = ((totalPhysicalMemorySize - freePhysicalMemorySize)
+                / totalPhysicalMemorySize) * 100;
+
+        if (usedPhysicalMemoryPercentage > maxUsedPhysicalMemoryPercentage) {
+            log.warn("Physical memory {} crossed the threshold of {}", usedPhysicalMemoryPercentage,
+                    maxUsedPhysicalMemoryPercentage);
+            return Health
+                    .outOfService()
+                    .withDetail(FREE_PHYSICAL_MEMORY_SIZE, freePhysicalMemorySize)
+                    .withDetail(TOTAL_PHYSICAL_MEMORY_SIZE, totalPhysicalMemorySize)
+                    .build();
+        } else {
+            return Health
+                    .up()
+                    .withDetail(FREE_PHYSICAL_MEMORY_SIZE, freePhysicalMemorySize)
+                    .withDetail(TOTAL_PHYSICAL_MEMORY_SIZE, totalPhysicalMemorySize)
+                    .build();
+        }
+    }
+}

genie-web/src/main/resources/application.yml

@@ -24,6 +24,9 @@ eureka:
     enabled: false
 
 genie:
+  health:
+    memory:
+      usedPhysicalMemoryPercent: 90
   jobs:
     archive:
       location: base_archival_location_path

genie-web/src/test/groovy/com/netflix/genie/web/health/MemoryHealthIndicatorSpec.groovy

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2016 Netflix, Inc.
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package com.netflix.genie.web.health
+
+import com.sun.management.OperatingSystemMXBean
+import org.springframework.boot.actuate.health.Status
+import spock.lang.Specification
+import spock.lang.Unroll
+
+/**
+ * Unit tests for MemoryHealthIndicator.
+ *
+ * @author amajumdar
+ * @since 3.0.0
+ */
+@Unroll
+class MemoryHealthIndicatorSpec extends Specification{
+    OperatingSystemMXBean operatingSystemMXBean
+    MemoryHealthIndicator memoryHealthIndicator;
+
+    def setup(){
+        operatingSystemMXBean = Mock(OperatingSystemMXBean)
+        memoryHealthIndicator = new MemoryHealthIndicator(80, operatingSystemMXBean)
+    }
+
+    def 'Health should be #status when free memory is #freeMemory and totla is #totalMemory'(){
+        given:
+        operatingSystemMXBean.getFreePhysicalMemorySize() >> freeMemory
+        operatingSystemMXBean.getTotalPhysicalMemorySize() >> totalMemory
+        expect:
+        memoryHealthIndicator.health().getStatus() == status
+        where:
+        freeMemory | totalMemory | status
+        10.0       | 100.0       | Status.OUT_OF_SERVICE
+        20.0       | 100.0       | Status.UP
+        30.0       | 100.0       | Status.UP
+        30.0       | 50.0        | Status.UP
+    }
+}

genie-web/src/test/resources/application-ci.yml

@@ -17,6 +17,9 @@
 ##
 
 genie:
+  health:
+    memory:
+      usedPhysicalMemoryPercent: 100
   jobs:
     dir:
       location: "file:///tmp/"

genie-web/src/test/resources/application-integration.yml

@@ -17,6 +17,9 @@
 ##
 
 genie:
+  health:
+    memory:
+      usedPhysicalMemoryPercent: 100
   jobs:
     dir:
       location: "file:///tmp/"