ISPN-11480 Call EmbeddedCacheManager#stop in shutdown hook

core/src/main/java/org/infinispan/factories/GlobalComponentRegistry.java

@@ -29,12 +29,12 @@
 import org.infinispan.factories.scopes.Scopes;
 import org.infinispan.globalstate.GlobalConfigurationManager;
 import org.infinispan.jmx.CacheManagerJmxRegistration;
-import org.infinispan.metrics.impl.CacheManagerMetricsRegistration;
 import org.infinispan.lifecycle.ComponentStatus;
 import org.infinispan.lifecycle.ModuleLifecycle;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.manager.ModuleRepository;
 import org.infinispan.marshall.core.EncoderRegistry;
+import org.infinispan.metrics.impl.CacheManagerMetricsRegistration;
 import org.infinispan.notifications.cachemanagerlistener.CacheManagerNotifier;
 import org.infinispan.notifications.cachemanagerlistener.CacheManagerNotifierImpl;
 import org.infinispan.persistence.factory.CacheStoreFactoryRegistry;
@@ -210,7 +210,7 @@ protected synchronized void addShutdownHook() {
          shutdownHook = new Thread(() -> {
             try {
                invokedFromShutdownHook = true;
-               GlobalComponentRegistry.this.stop();
+               cacheManager.stop();
             } finally {
                invokedFromShutdownHook = false;
             }

core/src/test/java/org/infinispan/statetransfer/StatefulSetRollingUpgradeTest.java

@@ -0,0 +1,96 @@
+package org.infinispan.statetransfer;
+
+import static org.infinispan.commons.test.CommonsTestingUtil.tmpDirectory;
+import static org.infinispan.test.fwk.TestCacheManagerFactory.createClusteredCacheManager;
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import org.infinispan.commons.util.Util;
+import org.infinispan.configuration.cache.CacheMode;
+import org.infinispan.configuration.cache.ConfigurationBuilder;
+import org.infinispan.configuration.global.GlobalConfigurationBuilder;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.partitionhandling.PartitionHandling;
+import org.infinispan.test.MultipleCacheManagersTest;
+import org.infinispan.test.fwk.CleanupAfterMethod;
+import org.infinispan.test.fwk.TransportFlags;
+import org.testng.annotations.Test;
+
+/**
+ * Start cluster (A,B) redeploy after upgrade. Rolling upgrades always occur in the order B,A and A does not restart
+ * until B has completed successfully.
+ *
+ * @author Ryan Emerson
+ * @since 11.0
+ */
+@CleanupAfterMethod
+@Test(groups = "functional", testName = "statetransfer.StatefulSetRollingUpgradeTest")
+public class StatefulSetRollingUpgradeTest extends MultipleCacheManagersTest {
+
+   private static final String CACHE_NAME = "testCache";
+   private static final int NUM_ROLLING_UPGRADES = 4;
+
+   private int numNodes;
+
+   @Override
+   public Object[] factory() {
+      return new Object[]{
+            new StatefulSetRollingUpgradeTest().setNumNodes(2),
+            new StatefulSetRollingUpgradeTest().setNumNodes(3),
+            new StatefulSetRollingUpgradeTest().setNumNodes(4),
+            new StatefulSetRollingUpgradeTest().setNumNodes(5)
+      };
+   }
+
+   private StatefulSetRollingUpgradeTest setNumNodes(int numNodes) {
+      this.numNodes = numNodes;
+      return this;
+   }
+
+   @Override
+   protected String[] parameterNames() {
+      return new String[]{"nodes"};
+   }
+
+   @Override
+   protected Object[] parameterValues() {
+      return new Object[]{numNodes};
+   }
+
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      Util.recursiveFileRemove(tmpDirectory(this.getClass().getSimpleName()));
+
+      for (int id = 0; id < numNodes; id++)
+         createStatefulCacheManager(id);
+
+      waitForClusterToForm(CACHE_NAME);
+   }
+
+   public void testStateTransferRestart() {
+      for (int i = 0; i < NUM_ROLLING_UPGRADES; i++) {
+         for (int j = numNodes - 1; j > -1; j--) {
+            manager(j).stop();
+            cacheManagers.remove(j);
+            waitForClusterToForm(CACHE_NAME);
+            createStatefulCacheManager(j);
+            waitForClusterToForm(CACHE_NAME);
+         }
+      }
+   }
+
+   private void createStatefulCacheManager(int id) {
+      String stateDirectory = tmpDirectory(this.getClass().getSimpleName() + File.separator + id);
+      GlobalConfigurationBuilder global = GlobalConfigurationBuilder.defaultClusteredBuilder();
+      global.globalState().enable().persistentLocation(stateDirectory);
+
+      ConfigurationBuilder config = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC);
+      config.clustering()
+            .partitionHandling().whenSplit(PartitionHandling.DENY_READ_WRITES)
+            .stateTransfer().timeout(5 * numNodes, TimeUnit.SECONDS);
+      EmbeddedCacheManager manager = createClusteredCacheManager(true, global, null, new TransportFlags());
+      manager.defineConfiguration(CACHE_NAME, config.build());
+      cacheManagers.add(id, manager);
+   }
+}

server/runtime/src/main/java/org/infinispan/server/Server.java

@@ -21,6 +21,7 @@
 
 import javax.xml.stream.XMLStreamException;
 
+import org.apache.logging.log4j.LogManager;
 import org.infinispan.commons.CacheConfigurationException;
 import org.infinispan.commons.configuration.ConfigurationFor;
 import org.infinispan.commons.configuration.ConfigurationInfo;
@@ -409,12 +410,16 @@ private void localShutdown(ExitStatus exitStatus) {
       }
       // Shutdown the protocol servers in parallel
       protocolServers.values().parallelStream().forEach(ProtocolServer::stop);
-      cacheManagers.values().forEach(cm -> SecurityActions.stopCacheManager(cm));
+      cacheManagers.values().forEach(SecurityActions::stopCacheManager);
       this.status = ComponentStatus.TERMINATED;
       // Don't wait for the scheduler to finish
       if (scheduler != null) {
          scheduler.shutdown();
       }
+      // Shutdown Log4jk context manually as we set shutdownHook="disable"
+      // Log4j's shutdownHook may run concurrently with our shutdownHook,
+      // disabling logging before the server has finished stopping.
+      LogManager.shutdown();
    }
 
    private void serverStopHandler(ExitStatus exitStatus) {

server/runtime/src/main/server/server/conf/log4j2.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration name="InfinispanServerConfig" monitorInterval="60">
+<Configuration name="InfinispanServerConfig" monitorInterval="60" shutdownHook="disable">
   <Properties>
     <Property name="path">${sys:infinispan.server.log.path}</Property>
     <Property name="accessLogPattern">%X{address} %X{user} [%d{dd/MMM/yyyy:HH:mm:ss Z}] &quot;%X{method} %m %X{protocol}&quot; %X{status} %X{requestSize} %X{responseSize} %X{duration}%n</Property>