Watch the keystore and truststore directories for changes (#1466)

src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java

@@ -33,6 +33,7 @@
 import javax.ws.rs.DefaultValue;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
 import io.dropwizard.Configuration;
 import io.dropwizard.client.HttpClientConfiguration;
 import org.apache.cassandra.repair.RepairParallelism;
@@ -735,6 +736,21 @@ public String getTruststore() {
       return truststore;
     }
 
+    @VisibleForTesting
+    public void setEnabled(Boolean enabled) {
+      this.enabled = enabled;
+    }
+
+    @VisibleForTesting
+    public void setKeystore(String keystore) {
+      this.keystore = keystore;
+    }
+
+    @VisibleForTesting
+    public void setTruststore(String truststore) {
+      this.truststore = truststore;
+    }
+
     public int getMgmtApiMetricsPort() {
       return mgmtApiMetricsPort == null ? DEFAULT_MGMT_API_METRICS_PORT : mgmtApiMetricsPort;
     }

src/server/src/main/java/io/cassandrareaper/management/http/HttpManagementConnectionFactory.java

@@ -29,10 +29,15 @@
 import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
+import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -45,6 +50,8 @@
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
@@ -88,6 +95,14 @@ public HttpManagementConnectionFactory(AppContext context, ScheduledExecutorServ
     this.config = context.config;
     registerConnectionsGauge();
     this.jobStatusPollerExecutor = jobStatusPollerExecutor;
+    if (context.config.getHttpManagement().getKeystore() != null && !context.config.getHttpManagement().getKeystore()
+        .isEmpty()) {
+      try {
+        createSslWatcher();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
   }
 
   @Override
@@ -152,7 +167,6 @@ private HttpCassandraManagementProxy connectImpl(Node node)
 
     LOG.trace("Wanting to create new connection to " + node.getHostname());
     return HTTP_CONNECTIONS.computeIfAbsent(node.getHostname(), new Function<String, HttpCassandraManagementProxy>() {
-      @Nullable
       @Override
       public HttpCassandraManagementProxy apply(@Nullable String hostName) {
         ReaperApplicationConfiguration.HttpManagement httpConfig = config.getHttpManagement();
@@ -237,7 +251,70 @@ private TrustManager[] getTrustManagers() throws ReaperException {
     } catch (IOException | NoSuchAlgorithmException | KeyStoreException | CertificateException e) {
       throw new ReaperException(e);
     }
+  }
 
+  @VisibleForTesting
+  void createSslWatcher() throws IOException {
+    WatchService watchService = FileSystems.getDefault().newWatchService();
+    Path trustStorePath = Paths.get(config.getHttpManagement().getTruststore());
+    Path keyStorePath = Paths.get(config.getHttpManagement().getKeystore());
+    Path keystoreParent = trustStorePath.getParent();
+    Path trustStoreParent = keyStorePath.getParent();
+
+    keystoreParent.register(
+        watchService,
+        StandardWatchEventKinds.ENTRY_CREATE,
+        StandardWatchEventKinds.ENTRY_DELETE,
+        StandardWatchEventKinds.ENTRY_MODIFY);
+
+    if (!keystoreParent.equals(trustStoreParent)) {
+      trustStoreParent.register(
+          watchService,
+          StandardWatchEventKinds.ENTRY_CREATE,
+          StandardWatchEventKinds.ENTRY_DELETE,
+          StandardWatchEventKinds.ENTRY_MODIFY);
+    }
+
+    ExecutorService executorService = Executors.newSingleThreadExecutor();
+    executorService.execute(
+        () -> {
+          while (true) {
+            try {
+              clearHttpConnections();
+              WatchKey key = watchService.take();
+              List<WatchEvent<?>> events = key.pollEvents();
+              boolean reloadNeeded = false;
+              for (WatchEvent<?> event : events) {
+                WatchEvent.Kind<?> kind = event.kind();
+
+                WatchEvent<java.nio.file.Path> ev = (WatchEvent<Path>) event;
+                Path eventFilename = ev.context();
+
+                if (keystoreParent.resolve(eventFilename).equals(keyStorePath)
+                    || trustStoreParent.resolve(eventFilename).equals(trustStorePath)) {
+                  // Something in the TLS has been modified.. recreate HTTP connections
+                  reloadNeeded = true;
+                }
+              }
+              if (!key.reset()) {
+                // The watched directories have disappeared..
+                break;
+              }
+              if (reloadNeeded) {
+                LOG.info("Detected change in the SSL/TLS certificates, reloading.");
+                clearHttpConnections();
+              }
+            } catch (InterruptedException e) {
+              LOG.error("Filesystem watcher received InterruptedException", e);
+            }
+          }
+        });
+  }
+
+  @VisibleForTesting
+  void clearHttpConnections() {
+    // Clearing this causes the connectImpl() to recreate new SSLContext
+    HTTP_CONNECTIONS.clear();
   }
 
   private Response getPid(Node node) {

src/server/src/test/java/io/cassandrareaper/management/http/HttpCassandraManagementProxyTest.java

@@ -17,6 +17,7 @@
 
 package io.cassandrareaper.management.http;
 
+import io.cassandrareaper.AppContext;
 import io.cassandrareaper.ReaperApplicationConfiguration;
 import io.cassandrareaper.ReaperException;
 import io.cassandrareaper.core.GenericMetric;
@@ -29,6 +30,9 @@
 
 import java.math.BigInteger;
 import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -74,6 +78,7 @@
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -798,4 +803,36 @@ public void testIsCoordinatorNode_NoTokens() {
     endpointState.put("OTHER", "value");
     assertTrue(HttpCassandraManagementProxy.isCoordinatorNode(endpointState));
   }
+
+  @Test
+  public void testSSLHotReload() throws Exception {
+    AppContext context = mock(AppContext.class);
+    ReaperApplicationConfiguration config = new ReaperApplicationConfiguration();
+    context.config = config;
+
+    Path tempDirectory = Files.createTempDirectory("reload-test");
+    Path ks = Paths.get("/home/runner/work/cassandra-reaper/cassandra-reaper/.github/files/keystore.jks");
+    Path ts = Paths.get("/home/runner/work/cassandra-reaper/cassandra-reaper/.github/files/truststore.jks");
+
+    Path tsCopy =
+        Files.copy(ts, tempDirectory.resolve(ts.getFileName()));
+    Path ksCopy =
+        Files.copy(ks, tempDirectory.resolve(ks.getFileName()));
+
+    config.getHttpManagement().setEnabled(true);
+    config.getHttpManagement().setKeystore(ksCopy.toAbsolutePath().toString());
+    config.getHttpManagement().setTruststore(tsCopy.toAbsolutePath().toString());
+    HttpManagementConnectionFactory connectionFactory = new HttpManagementConnectionFactory(context, null);
+    HttpManagementConnectionFactory spy = spy(connectionFactory);
+    spy.createSslWatcher();
+
+    verify(spy, Mockito.timeout(1000)).clearHttpConnections();
+
+    // Modify filepaths
+    Files.delete(ksCopy);
+
+    // We need 3 invocations, because we can't spy the original constructor call to the clearHttpConnections() and
+    // as such need to create more SslWatchers() for the same path
+    verify(spy, Mockito.timeout(30000).atLeast(2)).clearHttpConnections();
+  }
 }