gracefully shutdown logback-classic logging in DittoService during co…

…ordinated shutdown * updated logstash-logback forwarder to 6.6 * configured "reconnectionDelay" to logstash to "1 second" * configured "logstash-logback-encoder" dependency to be included on "runtime" Maven scope * adjusted coordinated shutdown phase of SshClientProvider to be not in the last phase, but 1 earlier * increased coordinated shutdown "cluster-exiting" timeout to 20s Signed-off-by: Thomas Jaeckle <thomas.jaeckle@bosch.io>
eclipse-ditto · Jun 16, 2021 · e2e9690 · e2e9690
1 parent 58f5054
commit e2e9690
Show file tree

Hide file tree

Showing 16 changed files with 39 additions and 27 deletions.
diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/DittoService.java b/base/service/src/main/java/org/eclipse/ditto/base/service/DittoService.java
@@ -16,7 +16,6 @@
 import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull;
 
 import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
 import java.text.MessageFormat;
 import java.time.Duration;
 import java.util.Collection;
@@ -28,25 +27,23 @@
 import javax.annotation.concurrent.Immutable;
 import javax.annotation.concurrent.NotThreadSafe;
 
+import org.eclipse.ditto.base.model.signals.FeatureToggle;
 import org.eclipse.ditto.base.service.config.ServiceSpecificConfig;
-import org.eclipse.ditto.base.service.config.limits.LimitsConfig;
+import org.eclipse.ditto.base.service.devops.DevOpsCommandsActor;
+import org.eclipse.ditto.base.service.devops.LogbackLoggingFacade;
 import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
 import org.eclipse.ditto.internal.utils.config.DittoConfigError;
 import org.eclipse.ditto.internal.utils.config.InstanceIdentifierSupplier;
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
 import org.eclipse.ditto.internal.utils.config.raw.RawConfigSupplier;
-import org.eclipse.ditto.base.service.devops.DevOpsCommandsActor;
-import org.eclipse.ditto.base.service.devops.LogbackLoggingFacade;
 import org.eclipse.ditto.internal.utils.health.status.StatusSupplierActor;
-import org.eclipse.ditto.internal.utils.metrics.config.MetricsConfig;
 import org.eclipse.ditto.internal.utils.metrics.prometheus.PrometheusReporterRoute;
-import org.eclipse.ditto.internal.utils.persistence.mongo.config.MongoDbConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithMongoDbConfig;
-import org.eclipse.ditto.base.model.signals.FeatureToggle;
 import org.eclipse.ditto.messages.model.signals.commands.MessageCommandSizeValidator;
 import org.eclipse.ditto.policies.model.signals.commands.PolicyCommandSizeValidator;
 import org.eclipse.ditto.things.model.signals.commands.ThingCommandSizeValidator;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigException;
@@ -64,9 +61,9 @@
 import akka.cluster.pubsub.DistributedPubSub;
 import akka.http.javadsl.Http;
 import akka.http.javadsl.model.Uri;
-import akka.http.javadsl.server.Route;
 import akka.management.cluster.bootstrap.ClusterBootstrap;
 import akka.management.javadsl.AkkaManagement;
+import ch.qos.logback.classic.LoggerContext;
 import kamon.Kamon;
 import kamon.prometheus.PrometheusReporter;
 
@@ -498,6 +495,9 @@ private void setUpCoordinatedShutdown(final ActorSystem actorSystem) {
         coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate(),
                 "log_successful_graceful_shutdown", () -> {
             logger.info("Graceful shutdown completed.");
+            // close logback-classic logging correctly in order to flush/get the last logs:
+            final var loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+            loggerContext.stop();
             return CompletableFuture.completedFuture(Done.getInstance());
         });
     }

diff --git a/bom/pom.xml b/bom/pom.xml
@@ -60,7 +60,7 @@
 
         <slf4j.version>1.7.30</slf4j.version>
         <logback.version>1.2.3</logback.version>
-        <logstash-logback-encoder.version>6.4</logstash-logback-encoder.version>
+        <logstash-logback-encoder.version>6.6</logstash-logback-encoder.version>
         <janino.version>2.7.8</janino.version>
 
         <!-- ### Metrics and Tracing -->

diff --git a/concierge/service/pom.xml b/concierge/service/pom.xml
@@ -80,19 +80,16 @@
             <artifactId>akka-http_${scala.version}</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-        </dependency>
-
         <!-- logstash appender logging -->
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/concierge/service/src/main/resources/logback.xml b/concierge/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/connectivity/service/pom.xml b/connectivity/service/pom.xml
@@ -127,10 +127,12 @@ jmh-generator-annprocess). jmh-generator-annprocess overwrites the whole META-IN
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/.../main/java/org/eclipse/ditto/connectivity/service/messaging/tunnel/SshClientProvider.java b/.../main/java/org/eclipse/ditto/connectivity/service/messaging/tunnel/SshClientProvider.java
@@ -85,7 +85,7 @@ private SshClientProvider(final ActorSystem actorSystem) {
         }
 
         CoordinatedShutdown.get(actorSystem)
-                .addTask(CoordinatedShutdown.PhaseActorSystemTerminate(), "close ssh client",
+                .addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate(), "close_ssh_client",
                         () -> {
                             LOGGER.info("Closing ssh client before shutdown.");
                             final CompletableFuture<Done> done = new CompletableFuture<>();

diff --git a/connectivity/service/src/main/resources/logback.xml b/connectivity/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/gateway/service/pom.xml b/gateway/service/pom.xml
@@ -102,10 +102,12 @@
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/gateway/service/src/main/resources/logback.xml b/gateway/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/internal/utils/config/src/main/resources/ditto-akka-config.conf b/internal/utils/config/src/main/resources/ditto-akka-config.conf
@@ -245,6 +245,15 @@ akka {
       }
     }
   }
+
+  coordinated-shutdown {
+    phases {
+      cluster-exiting {
+        timeout = 20s # default: 10s
+        timeout = ${?AKKA_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT}
+      }
+    }
+  }
 }
 
 sharding-dispatcher {

diff --git a/policies/service/pom.xml b/policies/service/pom.xml
@@ -98,19 +98,16 @@
             <artifactId>akka-http_${scala.version}</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-        </dependency>
-
-        <!-- logstash appender logging -->
+        <!-- logging -->
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/policies/service/src/main/resources/logback.xml b/policies/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/things/service/pom.xml b/things/service/pom.xml
@@ -108,19 +108,16 @@
             <artifactId>akka-http_${scala.version}</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-        </dependency>
-
-        <!-- logstash appender logging -->
+        <!-- logging -->
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/things/service/src/main/resources/logback.xml b/things/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/thingsearch/service/pom.xml b/thingsearch/service/pom.xml
@@ -64,10 +64,12 @@
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/thingsearch/service/src/main/resources/logback.xml b/thingsearch/service/src/main/resources/logback.xml
@@ -34,6 +34,7 @@
         <then>
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
+                <reconnectionDelay>1 second</reconnectionDelay>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">