Update to log4j2

Fix apache#1514 Update pom.xml with log4j2 dependencies * Use slf4j bindings for log4j2 * Keep log4j 1.2 jars for minicluster and test, because ZK needs it at runtime * Use log4j-1.2-api jar for log4j 1.2 dependencies for deprecated public APIs that can't yet be removed (particularly in core/mapred stuffs) * Add disruptor jar to class path to support high-performance asynchronous loggers * Exclude transitive dependencies for log4j 1.2, for better control over build class path, and to avoid multiple runtime slf4j bindings * Update assembly to include log4j2 jars, and to exclude log4j 1.2 jars Fix apache#1515 Create new AccumuloMonitorAppender to send to REST endpoint * Remove unnecessary extra port configuration property for log4j SocketServer endpoint in monitor * Update Monitor to store HTTP(s) URL for the service in ZK instead of just host and port; remove redundant log4j service advertisement using old log4j SocketServer port * Create new AccumuloMonitorAppender to read HTTP(s) URL from ZooCache, on demand, and send logs as JSON to the REST endpoint at that URL * Fix extra slash in MonitorUtilTest * Replace previous LogService in monitor with simpler RecentLogs implementation to cache received logs from REST endpoint * Simplify LogResource REST endpoints to use the RecentLogs APIs, with new endpoint to receive POSTed logs as JSON, and clean up the deduplication/counting/sanitizing code * Simplify StatusResource to get its info from the RecentLogs object * Update the web views to include logger names on the monitor page, and to simplify passing of the optional stacktrace details Fix apache#1516 Remove log4j1 config files and add log4j2 config files * Create basic log4j2 config files for Accumulo services and a default config file for clients such as the shell * Update accumulo-env.sh to configure logging using high-performance asynchronous loggers * Create service config file with console, rolling file, and monitor appenders, as well as an example separate rolling file appender for audit logs that can be easily toggled on * Create config file for console suitable for clients, especially the shell, complete with color highlighting * Add config for periodic auto-reconfiguring of log4j without restarting services (especially helpful as a substitute for the now defunct shell option for 'debug on')
ctubbsii · Feb 21, 2020 · 030daa1 · 030daa1
1 parent 2b0b971
commit 030daa1
Show file tree

Hide file tree

Showing 41 changed files with 675 additions and 1,078 deletions.
diff --git a/assemble/conf/accumulo-env.sh b/assemble/conf/accumulo-env.sh
@@ -85,20 +85,19 @@ case "$cmd" in
   *)       JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms64m') ;;
 esac
 
-## JVM options set for logging. Review logj4 properties files to see how they are used.
+## JVM options set for logging. Review log4j2.properties file to see how they are used.
 JAVA_OPTS=("${JAVA_OPTS[@]}"
   "-Daccumulo.log.dir=${ACCUMULO_LOG_DIR}"
-  "-Daccumulo.application=${cmd}${ACCUMULO_SERVICE_INSTANCE}_$(hostname)")
+  "-Daccumulo.application=${cmd}${ACCUMULO_SERVICE_INSTANCE}_$(hostname)"
+  "-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+)
 
 case "$cmd" in
-  monitor)
-    JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configuration=log4j-monitor.properties")
-    ;;
-  gc|master|tserver|tracer)
-    JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configuration=log4j-service.properties")
+  monitor|gc|master|tserver|tracer)
+    JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configurationFile=log4j2-service.properties")
     ;;
   *)
-    # let log4j use its default behavior (log4j.xml, log4j.properties)
+    # let log4j use its default behavior (log4j2.properties, etc.)
     true
     ;;
 esac

diff --git a/assemble/conf/log4j-monitor.properties b/assemble/conf/log4j-monitor.properties
diff --git a/assemble/conf/log4j-service.properties b/assemble/conf/log4j-service.properties
diff --git a/assemble/conf/log4j.properties b/assemble/conf/log4j.properties
diff --git a/assemble/conf/log4j2-service.properties b/assemble/conf/log4j2-service.properties
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## Log4j2 file that configures logging for all Accumulo services
+## The system properties referenced below are configured by accumulo-env.sh
+
+status = info
+dest = err
+name = AccumuloServiceLoggingProperties
+monitorInterval = 30
+
+packages = org.apache.accumulo.monitor.util.logging
+
+property.filename = ${sys:accumulo.log.dir}/${sys:accumulo.application}
+
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.filter.threshold.type = ThresholdFilter
+appender.console.filter.threshold.level = info
+
+appender.rolling.type = RollingFile
+appender.rolling.name = LogFiles
+appender.rolling.fileName = ${filename}.log
+appender.rolling.filePattern = ${filename}-%d{MM-dd-yy-HH}-%i.log.gz
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=100MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.max = 10
+
+appender.audit.type = RollingFile
+appender.audit.name = AuditLogFiles
+appender.audit.fileName = ${filename}.audit
+appender.audit.filePattern = ${filename}-%d{MM-dd-yy-HH}-%i.audit.gz
+appender.audit.layout.type = PatternLayout
+appender.audit.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.audit.policies.type = Policies
+appender.audit.policies.time.type = TimeBasedTriggeringPolicy
+appender.audit.policies.time.interval = 1
+appender.audit.policies.time.modulate = true
+appender.audit.policies.size.type = SizeBasedTriggeringPolicy
+appender.audit.policies.size.size=100MB
+appender.audit.strategy.type = DefaultRolloverStrategy
+appender.audit.strategy.max = 10
+
+appender.monitor.type = AccumuloMonitor
+appender.monitor.name = MonitorLog
+appender.monitor.filter.threshold.type = ThresholdFilter
+appender.monitor.filter.threshold.level = warn
+
+logger.zookeeper.name = org.apache.zookeeper
+logger.zookeeper.level = error
+
+# uncomment for separate audit logs
+#logger.audit.name = org.apache.accumulo.audit
+#logger.audit.level = info
+#logger.audit.additivity = false
+#logger.audit.appenderRef.audit.ref = AuditLogFiles
+
+rootLogger.level = debug
+rootLogger.appenderRef.console.ref = STDERR
+rootLogger.appenderRef.rolling.ref = LogFiles
+rootLogger.appenderRef.monitor.ref = MonitorLog
+
diff --git a/assemble/conf/log4j2.properties b/assemble/conf/log4j2.properties
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## Log4j2 file that configures logging for processes other than Accumulo services
+
+status = info
+dest = err
+name = AccumuloDefaultLoggingProperties
+monitorInterval = 30
+
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %style{%d{ISO8601}}{dim,cyan} %style{[}{red}%style{%-8c{2}}{dim,blue}%style{]}{red} %highlight{%-5p}%style{:}{red} %m%n
+
+logger.shellaudit.name = org.apache.accumulo.shell.Shell.audit
+logger.shellaudit.level = warn
+
+logger.zookeeper.name = org.apache.zookeeper
+logger.zookeeper.level = error
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = STDERR
+
diff --git a/assemble/pom.xml b/assemble/pom.xml
@@ -96,6 +96,11 @@
       <artifactId>protobuf-java</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>com.lmax</groupId>
+      <artifactId>disruptor</artifactId>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>com.sun.xml.bind</groupId>
       <artifactId>jaxb-core</artifactId>
@@ -166,11 +171,6 @@
       <artifactId>jline</artifactId>
       <optional>true</optional>
     </dependency>
-    <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <optional>true</optional>
-    </dependency>
     <dependency>
       <groupId>org.apache.accumulo</groupId>
       <artifactId>accumulo-core</artifactId>
@@ -272,6 +272,31 @@
       <artifactId>htrace-core4</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-web</artifactId>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
@@ -452,11 +477,6 @@
       <artifactId>slf4j-api</artifactId>
       <optional>true</optional>
     </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <optional>true</optional>
-    </dependency>
   </dependencies>
   <build>
     <pluginManagement>