GEODE-2644: Make LogWriterAppender optional and support log4j2.xml

LogWriterAppender is now configured in log4j2.xml and it supports
sessions that correspond with Cache lifecycle. This allows Geode to
pause and resume LogWriterAppender and GeodeConsoleAppender without
resorting to dynamically adding and removing appenders.

When the Cache exists, log events stop going to the Console and instead
go to the Geode log file. Whenever the Cache does not exist, log events
go only to the Console.

These changes remove as much of the Log4j2 Core manipulation as
possible.

Also fixed:
* GEODE-5789: Geode now updates log level of all Geode loggers.

List of changes:
* Change LogWriterAppender to be pausable and session-oriented
* Add GeodeConsoleAppender which is pausable
* Log4j2 Core dependency is now optional
* Internal Logging interfaces allow Logging service to be pluggable
* Log4j2 xml integration testing now uses JUnit Rule from Log4j2
* Reduce coupling between Logging and the rest of Geode
* Greatly increase test coverage for Logging

geode-assembly/src/acceptanceTest/java/org/apache/geode/management/internal/cli/commands/GfshStartLocatorLogAcceptanceTest.java

@@ -14,46 +14,46 @@
  */
 package org.apache.geode.management.internal.cli.commands;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.geode.internal.logging.Configuration.STARTUP_CONFIGURATION;
 
 import java.io.File;
-import java.nio.charset.StandardCharsets;
 
-import com.google.common.io.Files;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import org.apache.geode.internal.Banner;
+import org.apache.geode.test.assertj.LogFileAssert;
 import org.apache.geode.test.junit.categories.GfshTest;
 import org.apache.geode.test.junit.categories.LoggingTest;
 import org.apache.geode.test.junit.rules.gfsh.GfshExecution;
 import org.apache.geode.test.junit.rules.gfsh.GfshRule;
 import org.apache.geode.test.junit.rules.gfsh.GfshScript;
 
-@Category({GfshTest.class, LoggingTest.class})
+@Category({LoggingTest.class, GfshTest.class})
 public class GfshStartLocatorLogAcceptanceTest {
 
+  private File logFile;
+
   @Rule
   public GfshRule gfshRule = new GfshRule();
 
-  @Test
-  public void bannerOnlyLogsOnce() throws Exception {
-    final String banner = "Licensed to the Apache Software Foundation (ASF)";
-    String lines = getExecutionLogs();
-    assertThat(lines.indexOf(banner)).isEqualTo(lines.lastIndexOf(banner));
+  @Before
+  public void setUp() {
+    GfshExecution gfshExecution = GfshScript.of("start locator").execute(gfshRule);
+    File[] files = gfshExecution.getWorkingDir().listFiles();
+    String logName = files[0].getAbsolutePath() + "/" + files[0].getName() + ".log";
+    logFile = new File(logName);
   }
 
   @Test
-  public void startupConfigsOnlyLogsOnce() throws Exception {
-    final String startupConfigs = "### GemFire Properties using default values ###";
-    String lines = getExecutionLogs();
-    assertThat(lines.indexOf(startupConfigs)).isEqualTo(lines.lastIndexOf(startupConfigs));
+  public void bannerIsLoggedOnlyOnce() {
+    LogFileAssert.assertThat(logFile).containsOnlyOnce(Banner.BannerHeader.displayValues());
   }
 
-  private String getExecutionLogs() throws Exception {
-    GfshExecution gfshExecution = GfshScript.of("start locator").execute(gfshRule);
-    File[] files = gfshExecution.getWorkingDir().listFiles();
-    String logName = files[0].getAbsolutePath() + "/" + files[0].getName() + ".log";
-    return Files.readLines(new File(logName), StandardCharsets.UTF_8).toString();
+  @Test
+  public void startupConfigIsLoggedOnlyOnce() {
+    LogFileAssert.assertThat(logFile).containsOnlyOnce(STARTUP_CONFIGURATION);
   }
 }

geode-core/src/distributedTest/java/org/apache/geode/internal/logging/LoggingWithReconnectDistributedTest.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.logging;
+
+import static java.lang.System.lineSeparator;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.geode.distributed.ConfigurationProperties.DISABLE_AUTO_RECONNECT;
+import static org.apache.geode.distributed.ConfigurationProperties.ENABLE_CLUSTER_CONFIGURATION;
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.MAX_WAIT_TIME_RECONNECT;
+import static org.apache.geode.distributed.ConfigurationProperties.MEMBER_TIMEOUT;
+import static org.apache.geode.distributed.internal.membership.gms.MembershipManagerHelper.getMembershipManager;
+import static org.apache.geode.internal.Banner.BannerHeader.displayValues;
+import static org.apache.geode.internal.logging.Configuration.STARTUP_CONFIGURATION;
+import static org.apache.geode.test.awaitility.GeodeAwaitility.await;
+import static org.apache.geode.test.awaitility.GeodeAwaitility.getTimeout;
+import static org.apache.geode.test.dunit.IgnoredException.addIgnoredException;
+import static org.apache.geode.test.dunit.VM.getController;
+import static org.apache.geode.test.dunit.VM.getVM;
+import static org.apache.geode.test.dunit.VM.toArray;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Arrays;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.geode.ForcedDisconnectException;
+import org.apache.geode.distributed.LocatorLauncher;
+import org.apache.geode.distributed.ServerLauncher;
+import org.apache.geode.distributed.internal.InternalDistributedSystem;
+import org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager;
+import org.apache.geode.test.assertj.LogFileAssert;
+import org.apache.geode.test.dunit.VM;
+import org.apache.geode.test.dunit.rules.DistributedRule;
+import org.apache.geode.test.junit.categories.LoggingTest;
+import org.apache.geode.test.junit.rules.serializable.SerializableTemporaryFolder;
+import org.apache.geode.test.junit.rules.serializable.SerializableTestName;
+
+/**
+ * Distributed tests for logging during reconnect.
+ */
+@Category(LoggingTest.class)
+public class LoggingWithReconnectDistributedTest implements Serializable {
+
+  private static final long TIMEOUT = getTimeout().getValueInMS();
+
+  private static LocatorLauncher locatorLauncher;
+  private static ServerLauncher serverLauncher;
+
+  private static InternalDistributedSystem system;
+
+  private VM locatorVM;
+  private VM server1VM;
+  private VM server2VM;
+
+  private String locatorName;
+  private String server1Name;
+  private String server2Name;
+
+  private File locatorDir;
+  private File server1Dir;
+  private File server2Dir;
+
+  @Rule
+  public DistributedRule distributedRule = new DistributedRule();
+
+  @Rule
+  public SerializableTemporaryFolder temporaryFolder = new SerializableTemporaryFolder();
+
+  @Rule
+  public SerializableTestName testName = new SerializableTestName();
+
+  @Before
+  public void setUp() throws Exception {
+    locatorName = "locator-" + testName.getMethodName();
+    server1Name = "server-" + testName.getMethodName() + "-1";
+    server2Name = "server-" + testName.getMethodName() + "-2";
+
+    locatorVM = getVM(0);
+    server1VM = getVM(1);
+    server2VM = getController();
+
+    locatorDir = temporaryFolder.newFolder(locatorName);
+    server1Dir = temporaryFolder.newFolder(server1Name);
+    server2Dir = temporaryFolder.newFolder(server2Name);
+
+    int locatorPort = locatorVM.invoke(() -> createLocator());
+
+    server1VM.invoke(() -> createServer(server1Name, server1Dir, locatorPort));
+    server2VM.invoke(() -> createServer(server2Name, server2Dir, locatorPort));
+
+    addIgnoredException(ForcedDisconnectException.class);
+    addIgnoredException("Possible loss of quorum");
+  }
+
+  @After
+  public void tearDown() {
+    locatorVM.invoke(() -> {
+      locatorLauncher.stop();
+      locatorLauncher = null;
+      system = null;
+    });
+
+    for (VM vm : toArray(server1VM, server2VM)) {
+      vm.invoke(() -> {
+        serverLauncher.stop();
+        serverLauncher = null;
+        system = null;
+      });
+    }
+  }
+
+  @Test
+  public void logFileContainsBannerOnlyOnce() {
+    locatorVM.invoke(() -> {
+      assertThat(system.getDistributionManager().getDistributionManagerIds()).hasSize(3);
+    });
+
+    server2VM.invoke(() -> {
+      GMSMembershipManager membershipManager = (GMSMembershipManager) getMembershipManager(system);
+      membershipManager.forceDisconnect("Forcing disconnect in " + testName.getMethodName());
+
+      await().until(() -> system.isReconnecting());
+      system.waitUntilReconnected(TIMEOUT, MILLISECONDS);
+      assertThat(system.getReconnectedSystem()).isNotSameAs(system);
+    });
+
+    locatorVM.invoke(() -> {
+      assertThat(system.getDistributionManager().getDistributionManagerIds()).hasSize(3);
+    });
+
+    server2VM.invoke(() -> {
+      File[] files = server2Dir.listFiles((dir, name) -> name.endsWith(".log"));
+      assertThat(files).as(expectedOneLogFile(files)).hasSize(1);
+
+      File logFile = files[0];
+      assertThat(logFile).exists();
+
+      // Banner must be logged only once
+      LogFileAssert.assertThat(logFile).containsOnlyOnce(displayValues());
+
+      // Startup Config must be logged only once
+      String[] startupConfiguration = StringUtils
+          .split(STARTUP_CONFIGURATION + system.getConfig().toLoggerString(), lineSeparator());
+
+      LogFileAssert.assertThat(logFile).containsOnlyOnce(startupConfiguration);
+    });
+  }
+
+  private String expectedOneLogFile(File[] files) {
+    return "Expecting directory:" + lineSeparator() + " " + server2Dir.getAbsolutePath()
+        + lineSeparator() + "to contain only one log file:" + lineSeparator() + " " + server2Name
+        + ".log" + lineSeparator() + "but found multiple log files:" + lineSeparator() + " "
+        + Arrays.asList(files);
+  }
+
+  private int createLocator() {
+    LocatorLauncher.Builder builder = new LocatorLauncher.Builder();
+    builder.setMemberName(locatorName);
+    builder.setWorkingDirectory(locatorDir.getAbsolutePath());
+    builder.setPort(0);
+    builder.set(DISABLE_AUTO_RECONNECT, "false");
+    builder.set(ENABLE_CLUSTER_CONFIGURATION, "false");
+    builder.set(MAX_WAIT_TIME_RECONNECT, "1000");
+    builder.set(MEMBER_TIMEOUT, "2000");
+
+    locatorLauncher = builder.build();
+    locatorLauncher.start();
+
+    system = (InternalDistributedSystem) locatorLauncher.getCache().getDistributedSystem();
+
+    return locatorLauncher.getPort();
+  }
+
+  private void createServer(String serverName, File serverDir, int locatorPort) {
+    ServerLauncher.Builder builder = new ServerLauncher.Builder();
+    builder.setMemberName(serverName);
+    builder.setWorkingDirectory(serverDir.getAbsolutePath());
+    builder.setServerPort(0);
+    builder.set(LOCATORS, "localHost[" + locatorPort + "]");
+    builder.set(DISABLE_AUTO_RECONNECT, "false");
+    builder.set(ENABLE_CLUSTER_CONFIGURATION, "false");
+    builder.set(MAX_WAIT_TIME_RECONNECT, "1000");
+    builder.set(MEMBER_TIMEOUT, "2000");
+
+    serverLauncher = builder.build();
+    serverLauncher.start();
+
+    system = (InternalDistributedSystem) serverLauncher.getCache().getDistributedSystem();
+  }
+}

geode-core/src/integrationTest/java/org/apache/geode/distributed/LocatorLauncherIntegrationTestCase.java

@@ -93,7 +93,7 @@ protected LocatorLauncher givenLocatorLauncher() {
     return givenLocatorLauncher(newBuilder());
   }
 
-  private LocatorLauncher givenLocatorLauncher(final Builder builder) {
+  protected LocatorLauncher givenLocatorLauncher(final Builder builder) {
     return builder.build();
   }