making it possible to enable/disable instance from lb via jmx (#869)

* making it possible to enable/disable instance from lb via jmx

* Moved the load balancer state to a class

* Updated jmx interface to be similar to what we have in db-web-ui so as to reduce duplication in deployment script

tools/whois.init

@@ -24,7 +24,7 @@ GC_LOG="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintHeapAtGC -Xloggc:va
 JMX="-Dcom.sun.management.jmxremote -Dhazelcast.jmx=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=${JMXPORT}"
 
 ### Final assembled command line
-COMMAND="$JAVA -D${SERVICE} $JAVA_OPT $JMX $MEM -Dwhois.config=properties -Dhazelcast.config=hazelcast.xml -Dlog4j.configurationFile=file:log4j2.xml -jar whois.jar"
+COMMAND="$JAVA -D${SERVICE} $JAVA_OPT $JMX $MEM -Dwhois.config=properties -Duser.timezone=UTC -Dhazelcast.config=hazelcast.xml -Dlog4j.configurationFile=file:log4j2.xml -jar whois.jar"
 COMMAND_REGEXP="$JAVA -D${SERVICE}"'.*'" -jar whois.jar"
 
 # location of console.log
@@ -167,12 +167,20 @@ case "$1" in
                         exit 1
                 fi
                 ;;
+        disable)
+          PID=$(pgrep -f "$COMMAND_REGEXP")
+          echo -e "bean net.ripe.db.whois:name=ReadinessUpdater && run up" | java -jar $JMXTERMPATH --url $PID
+          ;;
+        enable)
+          PID=$(pgrep -f "$COMMAND_REGEXP")
+          echo -e "bean net.ripe.db.whois:name=ReadinessUpdater && run down" | java -jar $JMXTERMPATH --url $PID
+          ;;
 	restart)
 		$0 stop
 		$0 start
 		;;
 	*)
-		echo "Usage: $0 {start|stop|restart|jmx|threaddump}"
+		echo "Usage: $0 {start|stop|restart|jmx|threaddump|disable|enable}"
 		exit 1
 		;;
 esac

whois-api/src/main/java/net/ripe/db/whois/api/rest/HealthCheckService.java

@@ -1,5 +1,6 @@
 package net.ripe.db.whois.api.rest;
 
+import net.ripe.db.whois.common.ReadinessUpdater;
 import net.ripe.db.whois.common.iptree.IpTreeCacheManager;
 import net.ripe.db.whois.common.source.SourceContext;
 import org.apache.commons.io.FileUtils;
@@ -39,19 +40,23 @@ public class HealthCheckService {
     private final JdbcTemplate writeTemplate;
     private final IpTreeCacheManager ipTreeCacheManager;
     private final SourceContext sourceContext;
+    private final ReadinessUpdater readinessUpdater;
 
     private final File checkFile;
 
     @Autowired
     public HealthCheckService(@Qualifier("whoisSlaveDataSource") final DataSource readDataSource,
                               @Qualifier("sourceAwareDataSource") final DataSource writeDataSource,
+                              final ReadinessUpdater readinessUpdater,
                               final IpTreeCacheManager ipTreeCacheManager,
                               final SourceContext sourceContext,
                               @Value("${dir.var:}") final String filesystemRoot) {
+
         this.readTemplate = new JdbcTemplate(readDataSource);
         this.writeTemplate = new JdbcTemplate(writeDataSource);
         this.ipTreeCacheManager = ipTreeCacheManager;
         this.sourceContext = sourceContext;
+        this.readinessUpdater = readinessUpdater;
 
         if (StringUtils.isNotBlank(filesystemRoot)) {
             this.checkFile = new File(filesystemRoot, "lock");
@@ -64,9 +69,11 @@ public HealthCheckService(@Qualifier("whoisSlaveDataSource") final DataSource re
 
     @GET
     public Response check() {
-        return databaseHealthy.get() && filesystemHealthy.get() && ipTreeHealthy.get()?
-                Response.ok().build() :
-                Response.status(Status.SERVICE_UNAVAILABLE).build();
+        boolean isHealthy = readinessUpdater.isLoadBalancerEnabled() && databaseHealthy.get() && filesystemHealthy.get() && ipTreeHealthy.get();
+
+        return isHealthy ?
+                Response.ok().entity("OK").build() :
+                Response.status(Status.SERVICE_UNAVAILABLE).entity("DISABLED").build();
     }
 
     @Scheduled(fixedDelay = 60 * 1_000)

whois-api/src/test/java/net/ripe/db/whois/api/rest/HealthCheckServiceIntegrationTest.java

@@ -4,6 +4,7 @@
 import net.ripe.db.whois.api.RestTest;
 import net.ripe.db.whois.api.rest.client.RestClient;
 import net.ripe.db.whois.common.IntegrationTest;
+import net.ripe.db.whois.common.ReadinessUpdater;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -20,14 +21,29 @@ public class HealthCheckServiceIntegrationTest extends AbstractIntegrationTest {
     @Autowired
     RestClient restClient;
 
+    @Autowired
+    ReadinessUpdater readinessUpdater;
+
+    @Test
+    public void testHealthyLBEnabled() {
+        readinessUpdater.up();
+        Response response = RestTest.target(getPort(), "whois/healthcheck")
+                .request()
+                .get(Response.class);
+
+        assertThat(response.getStatus(), is(Status.OK.getStatusCode()));
+        assertThat(response.readEntity(String.class), is("OK"));
+    }
+
     @Test
-    public void testHealthy() {
-        assertThat(
-            RestTest.target(getPort(), "whois/healthcheck")
+    public void testHealthyLBDisabled() {
+        readinessUpdater.down();
+        Response response = RestTest.target(getPort(), "whois/healthcheck")
                 .request()
-                .get(Response.class).getStatus(),
-            is(Status.OK.getStatusCode())
-        );
+                .get(Response.class);
+
+        assertThat(response.getStatus(), is(Status.SERVICE_UNAVAILABLE.getStatusCode()));
+        assertThat(response.readEntity(String.class), is("DISABLED"));
     }
 
 }

whois-commons/src/main/java/net/ripe/db/whois/common/ReadinessUpdater.java

@@ -0,0 +1,45 @@
+package net.ripe.db.whois.common;
+
+import net.ripe.db.whois.common.jmx.JmxBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jmx.export.annotation.ManagedAttribute;
+import org.springframework.jmx.export.annotation.ManagedOperation;
+import org.springframework.jmx.export.annotation.ManagedResource;
+import org.springframework.stereotype.Component;
+
+/**
+ * in jmxterm, run with:
+ *      bean net.ripe.db.whois:name=ReadinessUpdater
+ *      run up - ready to receive traffic
+ *      run down - not ready to receive traffic
+ *
+ */
+@Component
+@ManagedResource(objectName = JmxBase.OBJECT_NAME_BASE + "ReadinessUpdater", description = "Loadbalancer switch")
+public class ReadinessUpdater extends JmxBase {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ReadinessUpdater.class);
+
+    private boolean loadBalancerEnabled;
+
+    public ReadinessUpdater() {
+        super(LOGGER);
+    }
+
+    @ManagedOperation
+    public void up(){
+        this.loadBalancerEnabled = true;
+        LOGGER.info("Marked service as ready to receive traffic");
+    }
+
+    @ManagedOperation
+    public void down() {
+        this.loadBalancerEnabled = false;
+        LOGGER.info("Marked service as not ready to receive traffic");
+    }
+
+    public boolean isLoadBalancerEnabled() {
+        return loadBalancerEnabled;
+    }
+
+}