Support reload certs in HTTPS server.

docs/en/changes/changes.md

@@ -81,7 +81,7 @@
 * Fix gRPC alarm cannot update settings from dynamic configuration source.
 * Add Python Websocket module component ID(7018).
 * [Optional] Optimize single trace query performance by customizing routing in ElasticSearch. SkyWalking trace segments and Zipkin spans are using trace ID for routing. This is OFF by default, controlled by `storage/elasticsearch/enableCustomRouting`.
-* Enhance OAP HTTP server to support HTTPS
+* Enhance OAP HTTP server to support HTTPS (Support reload certs).
 
 #### UI
 

oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.library.server.http;
 
 import com.google.common.collect.Sets;
+
 import com.linecorp.armeria.common.HttpMethod;
 import com.linecorp.armeria.common.HttpResponse;
 import com.linecorp.armeria.common.HttpStatus;
@@ -34,13 +35,16 @@
 import java.net.InetSocketAddress;
 
 import java.time.Duration;
+
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.skywalking.oap.server.library.server.Server;
 import org.apache.skywalking.oap.server.library.server.ssl.PrivateKeyUtil;
+import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor;
 
 import static java.util.Objects.requireNonNull;
 
@@ -51,6 +55,12 @@ public class HTTPServer implements Server {
     // Health check service, supports HEAD, GET method.
     private final Set<HttpMethod> allowedMethods = Sets.newHashSet(HttpMethod.HEAD);
 
+    private Set<Object> handlers;
+
+    private MultipleFilesChangeMonitor monitor;
+
+    private com.linecorp.armeria.server.Server httpServer;
+
     public HTTPServer(HTTPServerConfig config) {
         this.config = config;
     }
@@ -60,24 +70,14 @@ public void initialize() {
         // TODO replace prefix with real context path when Armeria supports it
         final String contextPath = StringUtils.stripEnd(config.getContextPath(), "/");
         sb = com.linecorp.armeria.server.Server
-            .builder()
-            .serviceUnder(contextPath + "/docs", DocService.builder().build())
-            .service("/internal/l7check", HealthCheckService.of())
-            .workerGroup(config.getMaxThreads())
-            .http1MaxHeaderSize(config.getMaxRequestHeaderSize())
-            .idleTimeout(Duration.ofMillis(config.getIdleTimeOut()))
-            .decorator(Route.ofCatchAll(), (delegate, ctx, req) -> {
-                if (!allowedMethods.contains(ctx.method())) {
-                    return HttpResponse.of(HttpStatus.METHOD_NOT_ALLOWED);
-                }
-                return delegate.serve(ctx, req);
-            })
-            .decorator(LoggingService.newDecorator());
-
+            .builder();
+        normalInitialize(sb);
         if (config.isEnableTLS()) {
+            handlers = new HashSet<>();
             sb.https(new InetSocketAddress(
                     config.getHost(),
                     config.getPort()));
+
             try (InputStream cert = new FileInputStream(config.getTlsCertChainPath());
                  InputStream key = PrivateKeyUtil.loadDecryptionKey(config.getTlsKeyPath())) {
                 sb.tls(cert, key);
@@ -90,11 +90,33 @@ public void initialize() {
                     config.getPort()
             ));
         }
+
+        monitor = new MultipleFilesChangeMonitor(
+                10,
+                readableContents -> updateCert(),
+                config.getTlsCertChainPath(),
+                config.getTlsKeyPath());
+
+        log.info("Server root context path: {}", contextPath);
+    }
+
+    private void normalInitialize(ServerBuilder sb) {
+        final String contextPath = StringUtils.stripEnd(config.getContextPath(), "/");
+        sb.serviceUnder(contextPath + "/docs", DocService.builder().build())
+        .service("/internal/l7check", HealthCheckService.of())
+        .workerGroup(config.getMaxThreads())
+        .http1MaxHeaderSize(config.getMaxRequestHeaderSize())
+        .idleTimeout(Duration.ofMillis(config.getIdleTimeOut()))
+        .decorator(Route.ofCatchAll(), (delegate, ctx, req) -> {
+            if (!allowedMethods.contains(ctx.method())) {
+                return HttpResponse.of(HttpStatus.METHOD_NOT_ALLOWED);
+            }
+            return delegate.serve(ctx, req);
+        })
+        .decorator(LoggingService.newDecorator());
         if (config.getAcceptQueueSize() > 0) {
             sb.maxNumConnections(config.getAcceptQueueSize());
         }
-
-        log.info("Server root context path: {}", contextPath);
     }
 
     /**
@@ -107,15 +129,37 @@ public void addHandler(Object handler, List<HttpMethod> httpMethods) {
             "Bind handler {} into http server {}:{}",
             handler.getClass().getSimpleName(), config.getHost(), config.getPort()
         );
-
+        if (config.isEnableTLS()) {
+            handlers.add(handler);
+        }
         sb.annotatedService()
           .pathPrefix(config.getContextPath())
           .build(handler);
         this.allowedMethods.addAll(httpMethods);
     }
 
+    public void updateCert() {
+        log.info("TLS cert chain file or TLS key file has been updated, reloading...");
+        httpServer.reconfigure(sb -> {
+            try (InputStream cert = new FileInputStream(config.getTlsCertChainPath());
+                 InputStream key = PrivateKeyUtil.loadDecryptionKey(config.getTlsKeyPath())) {
+                sb.tls(cert, key);
+            } catch (IOException e) {
+                throw new IllegalArgumentException(e);
+            }
+            normalInitialize(sb);
+            sb.annotatedService()
+              .pathPrefix(config.getContextPath())
+              .build(handlers.toArray());
+        });
+    }
+
     @Override
     public void start() {
-        sb.build().start().join();
+        httpServer = sb.build();
+        httpServer.start().join();
+        if (config.isEnableTLS()) {
+            monitor.start();
+        }
     }
 }