From 030daa1d477b74e732a7aadc6c056041241d96c1 Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Fri, 21 Feb 2020 07:37:32 -0500 Subject: [PATCH] Update to log4j2 Fix #1514 Update pom.xml with log4j2 dependencies * Use slf4j bindings for log4j2 * Keep log4j 1.2 jars for minicluster and test, because ZK needs it at runtime * Use log4j-1.2-api jar for log4j 1.2 dependencies for deprecated public APIs that can't yet be removed (particularly in core/mapred stuffs) * Add disruptor jar to class path to support high-performance asynchronous loggers * Exclude transitive dependencies for log4j 1.2, for better control over build class path, and to avoid multiple runtime slf4j bindings * Update assembly to include log4j2 jars, and to exclude log4j 1.2 jars Fix #1515 Create new AccumuloMonitorAppender to send to REST endpoint * Remove unnecessary extra port configuration property for log4j SocketServer endpoint in monitor * Update Monitor to store HTTP(s) URL for the service in ZK instead of just host and port; remove redundant log4j service advertisement using old log4j SocketServer port * Create new AccumuloMonitorAppender to read HTTP(s) URL from ZooCache, on demand, and send logs as JSON to the REST endpoint at that URL * Fix extra slash in MonitorUtilTest * Replace previous LogService in monitor with simpler RecentLogs implementation to cache received logs from REST endpoint * Simplify LogResource REST endpoints to use the RecentLogs APIs, with new endpoint to receive POSTed logs as JSON, and clean up the deduplication/counting/sanitizing code * Simplify StatusResource to get its info from the RecentLogs object * Update the web views to include logger names on the monitor page, and to simplify passing of the optional stacktrace details Fix #1516 Remove log4j1 config files and add log4j2 config files * Create basic log4j2 config files for Accumulo services and a default config file for clients such as the shell * Update accumulo-env.sh to configure logging using high-performance asynchronous loggers * Create service config file with console, rolling file, and monitor appenders, as well as an example separate rolling file appender for audit logs that can be easily toggled on * Create config file for console suitable for clients, especially the shell, complete with color highlighting * Add config for periodic auto-reconfiguring of log4j without restarting services (especially helpful as a substitute for the now defunct shell option for 'debug on') --- assemble/conf/accumulo-env.sh | 15 +- assemble/conf/log4j-monitor.properties | 40 --- assemble/conf/log4j-service.properties | 62 ----- assemble/conf/log4j.properties | 42 ---- assemble/conf/log4j2-service.properties | 88 +++++++ assemble/conf/log4j2.properties | 41 ++++ assemble/pom.xml | 40 ++- assemble/src/main/assemblies/component.xml | 8 +- core/pom.xml | 16 +- .../org/apache/accumulo/core/Constants.java | 1 - .../apache/accumulo/core/conf/Property.java | 2 - .../accumulo/core/util/MonitorUtilTest.java | 4 +- hadoop-mapreduce/pom.xml | 4 +- iterator-test-harness/pom.xml | 4 +- minicluster/pom.xml | 19 +- .../MiniAccumuloConfigImpl.java | 1 - .../minicluster/MiniAccumuloClusterTest.java | 3 +- pom.xml | 40 ++- server/base/pom.xml | 12 +- .../server/monitor/DedupedLogEvent.java | 71 ------ .../accumulo/server/monitor/LogService.java | 197 --------------- server/gc/pom.xml | 8 +- server/master/pom.xml | 8 +- server/monitor/pom.xml | 30 ++- .../accumulo/monitor/EmbeddedWebServer.java | 22 +- .../org/apache/accumulo/monitor/Monitor.java | 35 ++- .../monitor/rest/logs/LogResource.java | 63 ++--- .../monitor/rest/logs/SanitizedLogEvent.java | 65 +++++ .../{LogEvent.java => SingleLogEvent.java} | 44 +--- .../monitor/rest/status/StatusResource.java | 22 +- .../monitor/util/AccumuloMonitorAppender.java | 228 ------------------ .../util/logging/AccumuloMonitorAppender.java | 147 +++++++++++ .../monitor/util/logging/RecentLogs.java | 90 +++++++ .../apache/accumulo/monitor/templates/log.ftl | 14 +- .../util/AccumuloMonitorAppenderTest.java | 208 ---------------- server/native/pom.xml | 6 +- server/tracer/pom.xml | 8 +- server/tserver/pom.xml | 8 +- shell/pom.xml | 10 +- start/pom.xml | 15 +- test/pom.xml | 12 +- 41 files changed, 675 insertions(+), 1078 deletions(-) delete mode 100644 assemble/conf/log4j-monitor.properties delete mode 100644 assemble/conf/log4j-service.properties delete mode 100644 assemble/conf/log4j.properties create mode 100644 assemble/conf/log4j2-service.properties create mode 100644 assemble/conf/log4j2.properties delete mode 100644 server/base/src/main/java/org/apache/accumulo/server/monitor/DedupedLogEvent.java delete mode 100644 server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java create mode 100644 server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SanitizedLogEvent.java rename server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/{LogEvent.java => SingleLogEvent.java} (53%) delete mode 100644 server/monitor/src/main/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppender.java create mode 100644 server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/AccumuloMonitorAppender.java create mode 100644 server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/RecentLogs.java delete mode 100644 server/monitor/src/test/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppenderTest.java diff --git a/assemble/conf/accumulo-env.sh b/assemble/conf/accumulo-env.sh index b131c5d6731..bbf7d986040 100644 --- a/assemble/conf/accumulo-env.sh +++ b/assemble/conf/accumulo-env.sh @@ -85,20 +85,19 @@ case "$cmd" in *) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms64m') ;; esac -## JVM options set for logging. Review logj4 properties files to see how they are used. +## JVM options set for logging. Review log4j2.properties file to see how they are used. JAVA_OPTS=("${JAVA_OPTS[@]}" "-Daccumulo.log.dir=${ACCUMULO_LOG_DIR}" - "-Daccumulo.application=${cmd}${ACCUMULO_SERVICE_INSTANCE}_$(hostname)") + "-Daccumulo.application=${cmd}${ACCUMULO_SERVICE_INSTANCE}_$(hostname)" + "-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" +) case "$cmd" in - monitor) - JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configuration=log4j-monitor.properties") - ;; - gc|master|tserver|tracer) - JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configuration=log4j-service.properties") + monitor|gc|master|tserver|tracer) + JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configurationFile=log4j2-service.properties") ;; *) - # let log4j use its default behavior (log4j.xml, log4j.properties) + # let log4j use its default behavior (log4j2.properties, etc.) true ;; esac diff --git a/assemble/conf/log4j-monitor.properties b/assemble/conf/log4j-monitor.properties deleted file mode 100644 index 0062d31399f..00000000000 --- a/assemble/conf/log4j-monitor.properties +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -## Log4j 1.2 file that configures logging for Accumulo Monitor -## The system properties referenced below are configured by accumulo-env.sh - -## Define a log file appender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.File=${accumulo.log.dir}/${accumulo.application}.log -log4j.appender.file.MaxFileSize=100MB -log4j.appender.file.MaxBackupIndex=10 -log4j.appender.file.Threshold=ALL -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n - -## Define an appender for the Accumulo Monitor to log to its own web GUI -log4j.appender.gui=org.apache.accumulo.server.monitor.LogService -log4j.appender.gui.Threshold=WARN - -## Append monitor logs to its own web GUI -log4j.logger.org.apache.accumulo=INHERITED, gui - -## Append most logs to file -log4j.rootLogger=INFO, file diff --git a/assemble/conf/log4j-service.properties b/assemble/conf/log4j-service.properties deleted file mode 100644 index ce80481247e..00000000000 --- a/assemble/conf/log4j-service.properties +++ /dev/null @@ -1,62 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -## Log4j 1.2 file that configures logging for all Accumulo services (Master, TabletServer, GC, and Tracer) except Monitor -## The system properties referenced below are configured by accumulo-env.sh - -## Define a log file appender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.File=${accumulo.log.dir}/${accumulo.application}.log -log4j.appender.file.MaxFileSize=100MB -log4j.appender.file.MaxBackupIndex=10 -log4j.appender.file.Threshold=ALL -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n - -## Define an appender to send important logs to the the primary Accumulo Monitor -## The primary monitor is the one currently holding a shared lock in ZooKeeper, -## and is typically the one that started first. -log4j.appender.monitor=org.apache.accumulo.monitor.util.AccumuloMonitorAppender -log4j.appender.monitor.Threshold=WARN - -## Uncomment to define a log file appender for audit logs -#log4j.appender.audit=org.apache.log4j.DailyRollingFileAppender -#log4j.appender.audit.File=${accumulo.log.dir}/${accumulo.application}.audit -#log4j.appender.audit.DatePattern='.'yyyy-MM-dd -#log4j.appender.audit.layout=org.apache.log4j.PatternLayout -#log4j.appender.audit.layout.ConversionPattern=%d{ISO8601} [%c{2}] %-5p: %m%n -#log4j.additivity.org.apache.accumulo.audit=false - -## Change this log level from OFF to one of the following to enable audit logging: -## INFO -## enables audit logging (inherit appenders from root logger) -## INFO, audit -## enables audit logging using the audit log appender -## (requires audit log file appender above to be uncommented) -log4j.logger.org.apache.accumulo.audit=OFF - -## Append logs to the primary Accumulo Monitor -log4j.logger.org.apache.accumulo=INHERITED, monitor - -## Constrain some particularly spammy loggers -log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO -log4j.logger.org.apache.zookeeper=ERROR - -## Append most logs to file -log4j.rootLogger=INFO, file diff --git a/assemble/conf/log4j.properties b/assemble/conf/log4j.properties deleted file mode 100644 index b1a7b57f8df..00000000000 --- a/assemble/conf/log4j.properties +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -## Log4j 1.2 file that configures logging for clients and utility commands - -## Define a console appender -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Target=System.err -log4j.appender.console.Threshold=ALL -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n -log4j.appender.console.layout=org.apache.log4j.PatternLayout - -## Hide audit logs generated by the shell -log4j.logger.org.apache.accumulo.shell.Shell.audit=WARN - -## Constrain some particularly spammy loggers -log4j.logger.org.apache.accumulo.core.file.rfile.bcfile.Compression=WARN -log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN -log4j.logger.org.apache.hadoop.io.compress=WARN -log4j.logger.org.apache.zookeeper=ERROR - -## Uncomment the following for detailed logging (for example, in the Shell) -#log4j.logger.org.apache.accumulo.core=TRACE - -## Append most logs to console -log4j.rootLogger=INFO, console diff --git a/assemble/conf/log4j2-service.properties b/assemble/conf/log4j2-service.properties new file mode 100644 index 00000000000..bfe62462304 --- /dev/null +++ b/assemble/conf/log4j2-service.properties @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## Log4j2 file that configures logging for all Accumulo services +## The system properties referenced below are configured by accumulo-env.sh + +status = info +dest = err +name = AccumuloServiceLoggingProperties +monitorInterval = 30 + +packages = org.apache.accumulo.monitor.util.logging + +property.filename = ${sys:accumulo.log.dir}/${sys:accumulo.application} + +appender.console.type = Console +appender.console.name = STDERR +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n +appender.console.filter.threshold.type = ThresholdFilter +appender.console.filter.threshold.level = info + +appender.rolling.type = RollingFile +appender.rolling.name = LogFiles +appender.rolling.fileName = ${filename}.log +appender.rolling.filePattern = ${filename}-%d{MM-dd-yy-HH}-%i.log.gz +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 10 + +appender.audit.type = RollingFile +appender.audit.name = AuditLogFiles +appender.audit.fileName = ${filename}.audit +appender.audit.filePattern = ${filename}-%d{MM-dd-yy-HH}-%i.audit.gz +appender.audit.layout.type = PatternLayout +appender.audit.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n +appender.audit.policies.type = Policies +appender.audit.policies.time.type = TimeBasedTriggeringPolicy +appender.audit.policies.time.interval = 1 +appender.audit.policies.time.modulate = true +appender.audit.policies.size.type = SizeBasedTriggeringPolicy +appender.audit.policies.size.size=100MB +appender.audit.strategy.type = DefaultRolloverStrategy +appender.audit.strategy.max = 10 + +appender.monitor.type = AccumuloMonitor +appender.monitor.name = MonitorLog +appender.monitor.filter.threshold.type = ThresholdFilter +appender.monitor.filter.threshold.level = warn + +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = error + +# uncomment for separate audit logs +#logger.audit.name = org.apache.accumulo.audit +#logger.audit.level = info +#logger.audit.additivity = false +#logger.audit.appenderRef.audit.ref = AuditLogFiles + +rootLogger.level = debug +rootLogger.appenderRef.console.ref = STDERR +rootLogger.appenderRef.rolling.ref = LogFiles +rootLogger.appenderRef.monitor.ref = MonitorLog + diff --git a/assemble/conf/log4j2.properties b/assemble/conf/log4j2.properties new file mode 100644 index 00000000000..6decfa4bfa5 --- /dev/null +++ b/assemble/conf/log4j2.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## Log4j2 file that configures logging for processes other than Accumulo services + +status = info +dest = err +name = AccumuloDefaultLoggingProperties +monitorInterval = 30 + +appender.console.type = Console +appender.console.name = STDERR +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %style{%d{ISO8601}}{dim,cyan} %style{[}{red}%style{%-8c{2}}{dim,blue}%style{]}{red} %highlight{%-5p}%style{:}{red} %m%n + +logger.shellaudit.name = org.apache.accumulo.shell.Shell.audit +logger.shellaudit.level = warn + +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = error + +rootLogger.level = info +rootLogger.appenderRef.console.ref = STDERR + diff --git a/assemble/pom.xml b/assemble/pom.xml index f76648dbace..401add8684f 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -96,6 +96,11 @@ protobuf-java true + + com.lmax + disruptor + true + com.sun.xml.bind jaxb-core @@ -166,11 +171,6 @@ jline true - - log4j - log4j - true - org.apache.accumulo accumulo-core @@ -272,6 +272,31 @@ htrace-core4 true + + org.apache.logging.log4j + log4j-1.2-api + true + + + org.apache.logging.log4j + log4j-api + true + + + org.apache.logging.log4j + log4j-core + true + + + org.apache.logging.log4j + log4j-slf4j-impl + true + + + org.apache.logging.log4j + log4j-web + true + org.apache.thrift libthrift @@ -452,11 +477,6 @@ slf4j-api true - - org.slf4j - slf4j-log4j12 - true - diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml index bdc4a8f39c4..7c38a680ceb 100644 --- a/assemble/src/main/assemblies/component.xml +++ b/assemble/src/main/assemblies/component.xml @@ -46,6 +46,7 @@ com.google.guava:failureaccess com.google.guava:guava com.google.protobuf:protobuf-java + com.lmax:disruptor com.sun.xml.bind commons-beanutils:commons-beanutils commons-cli:commons-cli @@ -60,7 +61,6 @@ javax.ws.rs:javax.ws.rs-api javax.xml.bind:jaxb-api jline:jline - log4j:log4j org.apache.commons:commons-collections4 org.apache.commons:commons-configuration2 org.apache.commons:commons-lang3 @@ -69,6 +69,11 @@ org.apache.commons:commons-vfs2 org.apache.htrace:htrace-core4 org.apache.htrace:htrace-core + org.apache.logging.log4j:log4j-1.2-api + org.apache.logging.log4j:log4j-api + org.apache.logging.log4j:log4j-core + org.apache.logging.log4j:log4j-slf4j-impl + org.apache.logging.log4j:log4j-web org.apache.thrift:libthrift org.eclipse.jetty:jetty-continuation org.eclipse.jetty:jetty-http @@ -105,7 +110,6 @@ org.javassist:javassist org.jboss.logging:jboss-logging org.slf4j:slf4j-api - org.slf4j:slf4j-log4j12 diff --git a/core/pom.xml b/core/pom.xml index 62f5b313da1..57c78b615f8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -73,10 +73,6 @@ jline jline - - log4j - log4j - org.apache.accumulo accumulo-start @@ -105,6 +101,10 @@ org.apache.htrace htrace-core + + org.apache.logging.log4j + log4j-1.2-api + org.apache.thrift libthrift @@ -145,13 +145,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index 51862517390..85fa56a1c9f 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -54,7 +54,6 @@ public class Constants { public static final String ZMONITOR = "/monitor"; public static final String ZMONITOR_LOCK = ZMONITOR + "/lock"; public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr"; - public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr"; public static final String ZCONFIG = "/config"; diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index f721cb7e236..ba4fd62b663 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -544,8 +544,6 @@ public enum Property { "Properties in this category affect the behavior of the monitor web server."), MONITOR_PORT("monitor.port.client", "9995", PropertyType.PORT, "The listening port for the monitor's http service"), - MONITOR_LOG4J_PORT("monitor.port.log4j", "4560", PropertyType.PORT, - "The listening port for the monitor's log4j logging collection."), MONITOR_SSL_KEYSTORE("monitor.ssl.keyStore", "", PropertyType.PATH, "The keystore for enabling monitor SSL."), @Sensitive diff --git a/core/src/test/java/org/apache/accumulo/core/util/MonitorUtilTest.java b/core/src/test/java/org/apache/accumulo/core/util/MonitorUtilTest.java index 004992ca159..84d996e1724 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/MonitorUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/MonitorUtilTest.java @@ -36,8 +36,8 @@ public void testNoNodeFound() throws Exception { ZooReader zr = mock(ZooReader.class); ClientContext context = mock(ClientContext.class); - expect(context.getZooKeeperRoot()).andReturn("/root/"); - expect(zr.getData("/root/" + Constants.ZMONITOR_HTTP_ADDR, null)) + expect(context.getZooKeeperRoot()).andReturn("/root"); + expect(zr.getData("/root" + Constants.ZMONITOR_HTTP_ADDR, null)) .andThrow(new NoNodeException()); replay(zr, context); diff --git a/hadoop-mapreduce/pom.xml b/hadoop-mapreduce/pom.xml index 67944db260f..290612db37e 100644 --- a/hadoop-mapreduce/pom.xml +++ b/hadoop-mapreduce/pom.xml @@ -76,8 +76,8 @@ test - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl test diff --git a/iterator-test-harness/pom.xml b/iterator-test-harness/pom.xml index e5ac6bdd61f..c024a78e111 100644 --- a/iterator-test-harness/pom.xml +++ b/iterator-test-harness/pom.xml @@ -57,8 +57,8 @@ runtime - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl test diff --git a/minicluster/pom.xml b/minicluster/pom.xml index c6be4979a8f..153794f53c2 100644 --- a/minicluster/pom.xml +++ b/minicluster/pom.xml @@ -104,6 +104,12 @@ org.slf4j slf4j-api + + + log4j + log4j + runtime + org.apache.hadoop hadoop-client-runtime @@ -125,13 +131,18 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-1.2-api test - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.easymock + easymock test diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java index a51d3e1ae48..7c11f8a7757 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java @@ -165,7 +165,6 @@ MiniAccumuloConfigImpl initialize() { mergePropWithRandomPort(Property.TSERV_CLIENTPORT.getKey()); mergePropWithRandomPort(Property.MONITOR_PORT.getKey()); mergePropWithRandomPort(Property.GC_PORT.getKey()); - mergePropWithRandomPort(Property.MONITOR_LOG4J_PORT.getKey()); mergePropWithRandomPort(Property.REPLICATION_RECEIPT_SERVICE_PORT.getKey()); mergePropWithRandomPort(Property.MASTER_REPLICATION_COORDINATOR_PORT.getKey()); diff --git a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java index 21ce4b9e2ea..124daf5d62a 100644 --- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java +++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java @@ -245,8 +245,7 @@ public void testRandomPorts() throws Exception { .configure(new Parameters().properties().setFile(accumuloProps)); PropertiesConfiguration conf = propsBuilder.getConfiguration(); for (Property randomPortProp : new Property[] {Property.TSERV_CLIENTPORT, Property.MONITOR_PORT, - Property.MONITOR_LOG4J_PORT, Property.MASTER_CLIENTPORT, Property.TRACE_PORT, - Property.GC_PORT}) { + Property.MASTER_CLIENTPORT, Property.TRACE_PORT, Property.GC_PORT}) { String value = conf.getString(randomPortProp.getKey()); assertNotNull("Found no value for " + randomPortProp, value); assertEquals("0", value); diff --git a/pom.xml b/pom.xml index ec322cf08f3..0b9d3289bed 100644 --- a/pom.xml +++ b/pom.xml @@ -235,6 +235,11 @@ protobuf-java 3.7.1 + + com.lmax + disruptor + 3.4.2 + com.sun.xml.bind jaxb-core @@ -487,6 +492,12 @@ org.apache.hadoop hadoop-minikdc ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + org.apache.hadoop @@ -519,6 +530,16 @@ org.apache.zookeeper zookeeper ${zookeeper.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.bouncycastle @@ -751,14 +772,11 @@ ${slf4j.version} - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - org.slf4j - slf4j-nop - ${slf4j.version} + org.apache.logging.log4j + log4j-bom + 2.13.0 + pom + import @@ -1087,7 +1105,6 @@ org.glassfish.jersey.inject:jersey-hk2:jar:* org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2:jar:* org.powermock:powermock-api-easymock:jar:* - org.slf4j:slf4j-log4j12:jar:* junit:junit:jar:* javax.servlet:javax.servlet-api:jar:* javax.el:javax.el-api:jar:* @@ -1097,6 +1114,11 @@ org.apache.accumulo:accumulo-native:tar.gz:* commons-beanutils:commons-beanutils:jar:* + + log4j:log4j:jar:* + org.apache.logging.log4j:log4j-1.2-api:jar:* + org.apache.logging.log4j:log4j-slf4j-impl:jar:* + org.apache.logging.log4j:log4j-web:jar:* diff --git a/server/base/pom.xml b/server/base/pom.xml index c7add0aced3..f5f2cfc8ad0 100644 --- a/server/base/pom.xml +++ b/server/base/pom.xml @@ -56,10 +56,6 @@ jline jline - - log4j - log4j - org.apache.accumulo accumulo-core @@ -107,13 +103,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/server/base/src/main/java/org/apache/accumulo/server/monitor/DedupedLogEvent.java b/server/base/src/main/java/org/apache/accumulo/server/monitor/DedupedLogEvent.java deleted file mode 100644 index 46a4648f47e..00000000000 --- a/server/base/src/main/java/org/apache/accumulo/server/monitor/DedupedLogEvent.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.monitor; - -import org.apache.log4j.spi.LoggingEvent; - -public class DedupedLogEvent { - - private LoggingEvent event; - private int count = 0; - private int hash = -1; - - public DedupedLogEvent(LoggingEvent event) { - this(event, 1); - } - - public DedupedLogEvent(LoggingEvent event, int count) { - this.event = event; - this.count = count; - } - - public LoggingEvent getEvent() { - return event; - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } - - @Override - public int hashCode() { - if (hash == -1) { - String eventId = - event.getMDC("application") + ":" + event.getLevel() + ":" + event.getMessage(); - hash = eventId.hashCode(); - } - return hash; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof DedupedLogEvent) - return this.event.equals(((DedupedLogEvent) obj).event); - return false; - } - - @Override - public String toString() { - return event.getMDC("application") + ":" + event.getLevel() + ":" + event.getMessage(); - } -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java deleted file mode 100644 index 146e7568df4..00000000000 --- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.monitor; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.util.Daemon; -import org.apache.accumulo.fate.zookeeper.ZooReaderWriter; -import org.apache.accumulo.server.ServerContext; -import org.apache.log4j.LogManager; -import org.apache.log4j.net.SocketNode; -import org.apache.log4j.spi.LoggingEvent; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -/** - * Hijack log4j and capture log events for display. - */ -public class LogService extends org.apache.log4j.AppenderSkeleton { - - private static final Logger log = LoggerFactory.getLogger(LogService.class); - - /** - * Read logging events forward to us over the net. - */ - static class SocketServer implements Runnable { - private ServerSocket server = null; - - @SuppressFBWarnings(value = "UNENCRYPTED_SERVER_SOCKET", - justification = "not obvious what credentials could be used to set up this log receiver" - + " socket in order to encrypt it, in case of errors; should probably remove this " - + "socket appender in the future and force users to watch their system logs") - public SocketServer(int port) { - try { - server = new ServerSocket(port); - } catch (IOException io) { - throw new UncheckedIOException(io); - } - } - - public int getLocalPort() { - return server.getLocalPort(); - } - - @Override - public void run() { - try { - while (true) { - log.debug("Waiting for log message senders"); - Socket socket = server.accept(); - log.debug("Got a new connection"); - Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository())); - t.start(); - } - } catch (IOException io) { - log.error("{}", io.getMessage(), io); - } - } - } - - /** - * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper - * - * @param context - * Server context - * @param hostAddress - * Address that monitor process is bound to - */ - public static void startLogListener(ServerContext context, String hostAddress) { - try { - AccumuloConfiguration conf = context.getConfiguration(); - SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT)[0]); - - // getLocalPort will return the actual ephemeral port used when '0' was provided. - String logForwardingAddr = hostAddress + ":" + server.getLocalPort(); - - log.debug("Setting monitor log4j log-forwarding address to: {}", logForwardingAddr); - - final String path = context.getZooKeeperRoot() + Constants.ZMONITOR_LOG4J_ADDR; - final ZooReaderWriter zoo = context.getZooReaderWriter(); - - // Delete before we try to re-create in case the previous session hasn't yet expired - try { - zoo.delete(path, -1); - } catch (KeeperException e) { - // We don't care if the node is already gone - if (!KeeperException.Code.NONODE.equals(e.code())) { - throw e; - } - } - - zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8)); - - new Daemon(server).start(); - } catch (Throwable t) { - log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t); - } - } - - private static LogService instance = null; - - public static synchronized LogService getInstance() { - if (instance == null) - return new LogService(); - return instance; - } - - private static final int MAX_LOGS = 50; - - private LinkedHashMap events = - new LinkedHashMap(MAX_LOGS + 1, (float) .75, true) { - - private static final long serialVersionUID = 1L; - - @Override - @SuppressWarnings("rawtypes") - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > MAX_LOGS; - } - }; - - public LogService() { - synchronized (LogService.class) { - instance = this; - } - } - - @Override - protected synchronized void append(LoggingEvent ev) { - Object application = ev.getMDC("application"); - if (application == null || application.toString().isEmpty()) - return; - - DedupedLogEvent dev = new DedupedLogEvent(ev); - - // if event is present, increase the count - if (events.containsKey(dev.toString())) { - DedupedLogEvent oldDev = events.remove(dev.toString()); - dev.setCount(oldDev.getCount() + 1); - } - events.put(dev.toString(), dev); - } - - @Override - public void close() { - events = null; - } - - @Override - public synchronized void doAppend(LoggingEvent event) { - super.doAppend(event); - } - - @Override - public boolean requiresLayout() { - return false; - } - - public synchronized List getEvents() { - return new ArrayList<>(events.values()); - } - - public synchronized void clear() { - events.clear(); - } -} diff --git a/server/gc/pom.xml b/server/gc/pom.xml index b015b1f9aa3..22c2bcaae09 100644 --- a/server/gc/pom.xml +++ b/server/gc/pom.xml @@ -83,13 +83,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/server/master/pom.xml b/server/master/pom.xml index 6bcfdd5b369..b0b76f63e76 100644 --- a/server/master/pom.xml +++ b/server/master/pom.xml @@ -99,13 +99,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/server/monitor/pom.xml b/server/monitor/pom.xml index 1cb8c0c1a0e..70122c86d09 100644 --- a/server/monitor/pom.xml +++ b/server/monitor/pom.xml @@ -40,6 +40,10 @@ auto-service true + + com.google.code.gson + gson + javax.servlet javax.servlet-api @@ -56,10 +60,6 @@ javax.xml.bind jaxb-api - - log4j - log4j - org.apache.accumulo accumulo-core @@ -80,6 +80,18 @@ org.apache.hadoop hadoop-client-api + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-web + org.apache.thrift libthrift @@ -151,6 +163,11 @@ junit test + + org.apache.logging.log4j + log4j-slf4j-impl + test + org.easymock easymock @@ -181,11 +198,6 @@ powermock-module-junit4 test - - org.slf4j - slf4j-log4j12 - test - diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java index fd3e942c40c..54b46acdddb 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java @@ -36,14 +36,21 @@ public class EmbeddedWebServer { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedWebServer.class); + private static final EnumSet requireForSecure = + EnumSet.of(Property.MONITOR_SSL_KEYSTORE, Property.MONITOR_SSL_KEYSTOREPASS, + Property.MONITOR_SSL_TRUSTSTORE, Property.MONITOR_SSL_TRUSTSTOREPASS); + private final Server server; private final ServerConnector connector; private final ServletContextHandler handler; + private final boolean secure; public EmbeddedWebServer(Monitor monitor, int port) { server = new Server(); final AccumuloConfiguration conf = monitor.getContext().getConfiguration(); - connector = new ServerConnector(server, getConnectionFactories(conf)); + secure = requireForSecure.stream().map(conf::get).allMatch(s -> s != null && !s.isEmpty()); + + connector = new ServerConnector(server, getConnectionFactories(conf, secure)); connector.setHost(monitor.getHostname()); connector.setPort(port); @@ -53,13 +60,10 @@ public EmbeddedWebServer(Monitor monitor, int port) { handler.setContextPath("/"); } - private static AbstractConnectionFactory[] getConnectionFactories(AccumuloConfiguration conf) { + private static AbstractConnectionFactory[] getConnectionFactories(AccumuloConfiguration conf, + boolean secure) { HttpConnectionFactory httpFactory = new HttpConnectionFactory(); - EnumSet requireForSecure = - EnumSet.of(Property.MONITOR_SSL_KEYSTORE, Property.MONITOR_SSL_KEYSTOREPASS, - Property.MONITOR_SSL_TRUSTSTORE, Property.MONITOR_SSL_TRUSTSTOREPASS); - - if (requireForSecure.stream().map(p -> conf.get(p)).anyMatch(s -> s == null || s.isEmpty())) { + if (!secure) { LOG.debug("Not configuring Jetty to use TLS"); return new AbstractConnectionFactory[] {httpFactory}; } else { @@ -108,6 +112,10 @@ public int getPort() { return connector.getLocalPort(); } + public boolean isSecure() { + return secure; + } + public void start() { try { server.addConnector(connector); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 5dc6e5223ee..358690375e2 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@ -22,6 +22,7 @@ import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly; import java.net.InetAddress; +import java.net.URL; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; @@ -65,11 +66,11 @@ import org.apache.accumulo.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy; +import org.apache.accumulo.monitor.util.logging.RecentLogs; import org.apache.accumulo.server.AbstractServer; import org.apache.accumulo.server.HighlyAvailableService; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServerOpts; -import org.apache.accumulo.server.monitor.LogService; import org.apache.accumulo.server.problems.ProblemReports; import org.apache.accumulo.server.problems.ProblemType; import org.apache.accumulo.server.util.Halt; @@ -450,19 +451,22 @@ public void run() { log.debug("Using {} to advertise monitor location in ZooKeeper", advertiseHost); try { - String monitorAddress = HostAndPort.fromParts(advertiseHost, server.getPort()).toString(); - context.getZooReaderWriter().putPersistentData( - context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(UTF_8), - NodeExistsPolicy.OVERWRITE); - log.info("Set monitor address in zookeeper to {}", monitorAddress); + URL url = new URL(server.isSecure() ? "https" : "http", advertiseHost, server.getPort(), "/"); + final String path = context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR; + final ZooReaderWriter zoo = context.getZooReaderWriter(); + // Delete before we try to re-create in case the previous session hasn't yet expired + try { + zoo.delete(path, -1); + } catch (KeeperException e) { + // We don't care if the node is already gone + if (KeeperException.Code.NONODE != e.code()) { + throw e; + } + } + zoo.putEphemeralData(path, url.toString().getBytes(UTF_8)); + log.info("Set monitor address in zookeeper to {}", url); } catch (Exception ex) { - log.error("Unable to set monitor HTTP address in zookeeper", ex); - } - - if (advertiseHost != null) { - LogService.startLogListener(context, advertiseHost); - } else { - log.warn("Not starting log4j listener as we could not determine address to use"); + log.error("Unable to advertise monitor HTTP address in zookeeper", ex); } new Daemon(new LoggingRunnable(log, new ZooKeeperStatus(context)), "ZooKeeperStatus").start(); @@ -562,6 +566,7 @@ public static class ScanStats { } private final Map allScans = new HashMap<>(); + private final RecentLogs recentLogs = new RecentLogs(); public Map getScans() { synchronized (allScans) { @@ -813,4 +818,8 @@ public boolean isActiveService() { return monitorInitialized.get(); } + public RecentLogs recentLogs() { + return recentLogs; + } + } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogResource.java index d98eeeb6bcf..002e73b73c6 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogResource.java @@ -18,18 +18,17 @@ */ package org.apache.accumulo.monitor.rest.logs; -import java.util.ArrayList; import java.util.List; +import javax.inject.Inject; +import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; -import org.apache.accumulo.server.monitor.DedupedLogEvent; -import org.apache.accumulo.server.monitor.LogService; -import org.apache.log4j.spi.LoggingEvent; +import org.apache.accumulo.monitor.Monitor; /** * Responsible for generating a new log JSON object @@ -40,49 +39,17 @@ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) public class LogResource { + @Inject + private Monitor monitor; + /** * Generates log event list as a JSON object * * @return log event array */ @GET - public List getRecentLogs() { - List dedupedLogEvents = LogService.getInstance().getEvents(); - ArrayList logEvents = new ArrayList<>(dedupedLogEvents.size()); - - for (DedupedLogEvent dev : dedupedLogEvents) { - LoggingEvent ev = dev.getEvent(); - Object application = ev.getMDC("application"); - if (application == null) - application = ""; - String msg = ev.getMessage().toString(); - // truncate if full hadoop errors get logged as a message - msg = sanitize(msg); - if (msg.length() > 300) - msg = msg.substring(0, 300); - - String[] stacktrace = ev.getThrowableStrRep(); - if (stacktrace != null) - for (int i = 0; i < stacktrace.length; i++) - stacktrace[i] = sanitize(stacktrace[i]); - - // Add a new log event to the list - logEvents.add(new LogEvent(ev.getTimeStamp(), application, dev.getCount(), - ev.getLevel().toString(), msg.trim(), stacktrace)); - } - return logEvents; - } - - private String sanitize(String s) { - StringBuilder text = new StringBuilder(); - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - int type = Character.getType(c); - boolean notPrintable = type == Character.UNASSIGNED || type == Character.LINE_SEPARATOR - || type == Character.NON_SPACING_MARK || type == Character.PRIVATE_USE; - text.append(notPrintable ? '?' : c); - } - return text.toString().replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); + public List getRecentLogs() { + return monitor.recentLogs().getSanitizedEvents(); } /** @@ -90,6 +57,18 @@ private String sanitize(String s) { */ @POST public void clearLogs() { - LogService.getInstance().clear(); + monitor.recentLogs().clearEvents(); + } + + /** + * REST call to append a log message + * + * @since 2.1.0 + */ + @POST + @Path("append") + @Consumes(MediaType.APPLICATION_JSON) + public void append(SingleLogEvent event) { + monitor.recentLogs().addEvent(event); } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SanitizedLogEvent.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SanitizedLogEvent.java new file mode 100644 index 00000000000..05736b09ae3 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SanitizedLogEvent.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.rest.logs; + +/** + * A log event type (with counts) suitable for display on the monitor at the REST endpoint, + * {@link LogResource#getRecentLogs()}. + */ +public class SanitizedLogEvent { + + // Variable names become JSON keys + public final long timestamp; + public final String application; + public final String logger; + public final String level; + public final String message; + public final String stacktrace; + public final int count; + + public SanitizedLogEvent(SingleLogEvent event, int count) { + this.timestamp = event.timestamp; + this.application = sanitize(event.application); + this.logger = sanitize(event.logger); + this.level = sanitize(event.level); + String msg = sanitize(event.message); + // truncate long messages + if (msg.length() > 300) + msg = msg.substring(0, 300).trim(); + this.message = msg; + this.stacktrace = sanitize(event.stacktrace); + this.count = count; + } + + private String sanitize(String s) { + if (s == null) + return null; + StringBuilder text = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + int type = Character.getType(c); + boolean notPrintable = type == Character.UNASSIGNED || type == Character.LINE_SEPARATOR + || type == Character.NON_SPACING_MARK || type == Character.PRIVATE_USE; + text.append(notPrintable ? '?' : c); + } + return text.toString().trim().replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", + ">"); + } + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogEvent.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SingleLogEvent.java similarity index 53% rename from server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogEvent.java rename to server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SingleLogEvent.java index 33407853420..c021d13bc39 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/LogEvent.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/logs/SingleLogEvent.java @@ -18,43 +18,21 @@ */ package org.apache.accumulo.monitor.rest.logs; +import org.apache.accumulo.monitor.util.logging.AccumuloMonitorAppender; + /** - * A single message logged - * - * @since 2.0.0 + * A basic POJO to serialize single log events as JSON for transmitting from the + * {@link AccumuloMonitorAppender} to the REST endpoint at + * {@link LogResource#append(SingleLogEvent)}. */ -public class LogEvent { +public class SingleLogEvent { // Variable names become JSON keys public long timestamp; - public Object application; - public int count; - public String level, message; - public String[] stacktrace; - - public LogEvent() {} + public String application; + public String logger; + public String level; + public String message; + public String stacktrace; - /** - * Stores a new log event - * - * @param timestamp - * log event timestamp - * @param application - * log event application - * @param count - * log event count - * @param level - * log event level - * @param message - * log event message - */ - public LogEvent(long timestamp, Object application, int count, String level, String message, - String[] stacktrace) { - this.timestamp = timestamp; - this.application = application; - this.count = count; - this.level = level; - this.message = message; - this.stacktrace = stacktrace; - } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java index 6388ea32f6f..656c17edf1b 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java @@ -28,9 +28,6 @@ import org.apache.accumulo.core.master.thrift.MasterMonitorInfo; import org.apache.accumulo.monitor.Monitor; -import org.apache.accumulo.server.monitor.DedupedLogEvent; -import org.apache.accumulo.server.monitor.LogService; -import org.apache.log4j.Level; /** * Generates the status for master, gc, and tservers as well as log and problem reports @@ -56,7 +53,6 @@ public enum Status { @GET public StatusInformation getTables() { - StatusInformation status; Status masterStatus; Status gcStatus; Status tServerStatus = Status.ERROR; @@ -98,20 +94,8 @@ public StatusInformation getTables() { tServerStatus = Status.ERROR; } - List logs = LogService.getInstance().getEvents(); - boolean logsHaveError = false; - for (DedupedLogEvent dedupedLogEvent : logs) { - if (dedupedLogEvent.getEvent().getLevel().isGreaterOrEqual(Level.ERROR)) { - logsHaveError = true; - break; - } - } - - int numProblems = monitor.getProblemSummary().entrySet().size(); - - status = new StatusInformation(masterStatus.toString(), gcStatus.toString(), - tServerStatus.toString(), logs.size(), logsHaveError, numProblems); - - return status; + return new StatusInformation(masterStatus.toString(), gcStatus.toString(), + tServerStatus.toString(), monitor.recentLogs().numEvents(), + monitor.recentLogs().eventsIncludeErrors(), monitor.getProblemSummary().entrySet().size()); } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppender.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppender.java deleted file mode 100644 index 7691afde72a..00000000000 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppender.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.util; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.conf.SiteConfiguration; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.fate.zookeeper.ZooCache; -import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; -import org.apache.accumulo.server.ServerContext; -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.AsyncAppender; -import org.apache.log4j.net.SocketAppender; - -public class AccumuloMonitorAppender extends AsyncAppender implements AutoCloseable { - - final ScheduledExecutorService executorService; - final AtomicBoolean trackerScheduled; - private int frequency = 0; - private MonitorTracker tracker = null; - - /** - * A Log4j Appender which follows the registered location of the active Accumulo monitor service, - * and forwards log messages to it - */ - public AccumuloMonitorAppender() { - // create the background thread to watch for updates to monitor location - trackerScheduled = new AtomicBoolean(false); - executorService = Executors.newSingleThreadScheduledExecutor(runnable -> { - Thread t = - new Thread(runnable, AccumuloMonitorAppender.class.getSimpleName() + " Location Tracker"); - t.setDaemon(true); - return t; - }); - } - - public void setFrequency(int millis) { - if (millis > 0) { - frequency = millis; - } - } - - public int getFrequency() { - return frequency; - } - - // this is just for testing - void setTracker(MonitorTracker monitorTracker) { - tracker = monitorTracker; - } - - @Override - public void activateOptions() { - // only schedule it once (in case options get activated more than once); not sure if this is - // possible - if (trackerScheduled.compareAndSet(false, true)) { - if (frequency <= 0) { - // use default rate of 5 seconds between each check - frequency = 5000; - } - if (tracker == null) { - tracker = - new MonitorTracker(this, new ZooCacheLocationSupplier(), new SocketAppenderFactory()); - } - executorService.scheduleWithFixedDelay(tracker, frequency, frequency, TimeUnit.MILLISECONDS); - } - super.activateOptions(); - } - - @Override - public void close() { - if (!executorService.isShutdown()) { - executorService.shutdownNow(); - } - super.close(); - } - - static class MonitorLocation { - private final String location; - private final long modId; - - public MonitorLocation(long modId, byte[] location) { - this.modId = modId; - this.location = location == null ? null : new String(location, UTF_8); - } - - public boolean hasLocation() { - return location != null; - } - - public String getLocation() { - return location; - } - - @Override - public boolean equals(Object obj) { - if (obj != null && obj instanceof MonitorLocation) { - MonitorLocation other = (MonitorLocation) obj; - return modId == other.modId && Objects.equals(location, other.location); - } - return false; - } - - @Override - public int hashCode() { - return Long.hashCode(modId); - } - } - - private static class ZooCacheLocationSupplier implements Supplier { - - // path and zooCache are lazily set the first time this tracker is run - // this allows the tracker to be constructed and scheduled during log4j initialization without - // triggering any actual logs from the Accumulo or ZooKeeper code - private ServerContext context = null; - private String path = null; - private ZooCache zooCache = null; - - @Override - public MonitorLocation get() { - // lazily set up path and zooCache (see comment in constructor) - if (this.context == null) { - this.context = new ServerContext(SiteConfiguration.auto()); - this.path = context.getZooKeeperRoot() + Constants.ZMONITOR_LOG4J_ADDR; - this.zooCache = context.getZooCache(); - } - - // get the current location from the cache and update if necessary - ZcStat stat = new ZcStat(); - byte[] loc = zooCache.get(path, stat); - // mzxid is 0 if location does not exist and the non-zero transaction id of the last - // modification otherwise - return new MonitorLocation(stat.getMzxid(), loc); - } - } - - private static class SocketAppenderFactory implements Function { - @Override - public AppenderSkeleton apply(MonitorLocation loc) { - int defaultPort = Integer.parseUnsignedInt(Property.MONITOR_LOG4J_PORT.getDefaultValue()); - HostAndPort remote = HostAndPort.fromString(loc.getLocation()); - - SocketAppender socketAppender = new SocketAppender(); - socketAppender.setApplication(System.getProperty("accumulo.application", "unknown")); - socketAppender.setRemoteHost(remote.getHost()); - socketAppender.setPort(remote.getPortOrDefault(defaultPort)); - - return socketAppender; - } - } - - static class MonitorTracker implements Runnable { - - private final AccumuloMonitorAppender parentAsyncAppender; - private final Supplier currentLocationSupplier; - private final Function appenderFactory; - - private MonitorLocation lastLocation; - private AppenderSkeleton lastSocketAppender; - - public MonitorTracker(AccumuloMonitorAppender appender, - Supplier currentLocationSupplier, - Function appenderFactory) { - this.parentAsyncAppender = Objects.requireNonNull(appender); - this.appenderFactory = Objects.requireNonNull(appenderFactory); - this.currentLocationSupplier = Objects.requireNonNull(currentLocationSupplier); - - this.lastLocation = new MonitorLocation(0, null); - this.lastSocketAppender = null; - } - - @Override - public void run() { - try { - MonitorLocation currentLocation = currentLocationSupplier.get(); - // detect change - if (!currentLocation.equals(lastLocation)) { - // clean up old appender - if (lastSocketAppender != null) { - parentAsyncAppender.removeAppender(lastSocketAppender); - lastSocketAppender.close(); - lastSocketAppender = null; - } - // create a new one - if (currentLocation.hasLocation()) { - lastSocketAppender = appenderFactory.apply(currentLocation); - lastSocketAppender.activateOptions(); - parentAsyncAppender.addAppender(lastSocketAppender); - } - // update the last location only if switching was successful - lastLocation = currentLocation; - } - } catch (Exception e) { - // dump any non-fatal problems to the console, but let it run again - e.printStackTrace(); - } - } - - } - -} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/AccumuloMonitorAppender.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/AccumuloMonitorAppender.java new file mode 100644 index 00000000000..5c2625c1559 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/AccumuloMonitorAppender.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.util.logging; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.Optional; +import java.util.function.Supplier; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; +import org.apache.accumulo.monitor.rest.logs.LogResource; +import org.apache.accumulo.monitor.rest.logs.SingleLogEvent; +import org.apache.accumulo.server.ServerContext; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; + +import com.google.gson.Gson; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * A custom Log4j2 Appender which follows the registered location of the active Accumulo monitor + * service, and forwards log messages to its log4j REST endpoint at + * {@link LogResource#append(SingleLogEvent)}. + */ +@Plugin(name = "AccumuloMonitor", category = Core.CATEGORY_NAME, + elementType = Appender.ELEMENT_TYPE, printObject = true) +public class AccumuloMonitorAppender extends AbstractAppender { + + @PluginBuilderFactory + public static > B newBuilder() { + return new Builder().asBuilder(); + } + + public static class Builder> extends AbstractAppender.Builder + implements org.apache.logging.log4j.core.util.Builder { + + @Override + public AccumuloMonitorAppender build() { + return new AccumuloMonitorAppender(getName(), getFilter(), isIgnoreExceptions(), + getPropertyArray()); + } + + } + + private final Gson gson = new Gson(); + private final HttpClient httpClient = HttpClient.newHttpClient(); + private final Supplier> monitorLocator; + + private ServerContext context; + private String path; + private Pair> lastResult = new Pair<>(0L, Optional.empty()); + + private AccumuloMonitorAppender(final String name, final Filter filter, + final boolean ignoreExceptions, final Property[] properties) { + super(name, filter, null, ignoreExceptions, properties); + final ZcStat stat = new ZcStat(); + monitorLocator = () -> { + // lazily set up context/path + if (context == null) { + context = new ServerContext(SiteConfiguration.auto()); + path = context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR; + } + // get the current location from the cache + byte[] loc = context.getZooCache().get(path, stat); + Pair> last = lastResult; + if (stat.getMzxid() != last.getFirst()) { + // only create new objects if there's a change + last = new Pair<>(stat.getMzxid(), Optional.ofNullable(loc) + .map(bytes -> URI.create(new String(bytes, UTF_8) + "rest/logs/append"))); + lastResult = last; + } + return last.getSecond(); + }; + } + + @Override + public void append(final LogEvent event) { + monitorLocator.get().ifPresent(uri -> { + try { + var pojo = new SingleLogEvent(); + pojo.timestamp = event.getTimeMillis(); + pojo.application = System.getProperty("accumulo.application", "unknown"); + pojo.logger = event.getLoggerName(); + pojo.level = event.getLevel().name(); + pojo.message = event.getMessage().getFormattedMessage(); + pojo.stacktrace = throwableToStacktrace(event.getThrown()); + + String jsonEvent = gson.toJson(pojo); + + var req = HttpRequest.newBuilder(uri).POST(BodyPublishers.ofString(jsonEvent, UTF_8)) + .setHeader("Content-Type", "application/json").build(); + httpClient.sendAsync(req, BodyHandlers.discarding()); + } catch (final Exception e) { + error("Unable to send HTTP in appender [" + getName() + "]", event, e); + } + }); + } + + @SuppressFBWarnings(value = "INFORMATION_EXPOSURE_THROUGH_AN_ERROR_MESSAGE", + justification = "throwable is intended to be printed to output stream, to send to monitor") + private static String throwableToStacktrace(Throwable t) { + if (t == null) + return null; + StringWriter writer = new StringWriter(); + t.printStackTrace(new PrintWriter(writer)); + return writer.toString(); + } + + @Override + public String toString() { + return "AccumuloMonitorAppender{" + "name=" + getName() + ", state=" + getState() + '}'; + } + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/RecentLogs.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/RecentLogs.java new file mode 100644 index 00000000000..04fd02f2cc0 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/util/logging/RecentLogs.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.util.logging; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.accumulo.monitor.rest.logs.LogResource; +import org.apache.accumulo.monitor.rest.logs.SanitizedLogEvent; +import org.apache.accumulo.monitor.rest.logs.SingleLogEvent; + +/** + * A recent logs cache for the monitor that holds log messages received from + * {@link AccumuloMonitorAppender} instances at the monitor's REST endpoint + * {@link LogResource#append(SingleLogEvent)} until retrieved at the REST endpoint + * {@link LogResource#getRecentLogs()} or cleared via the REST endpoint + * {@link LogResource#clearLogs()}. + */ +public class RecentLogs { + + private static final int MAX_LOGS = 50; + + /** + * Internal class for keeping the current count and most recent event that matches a given cache + * key (derived from the event's application, logger, level, and message fields). + */ + private static class DedupedEvent { + private final SingleLogEvent event; + private final int count; + + private DedupedEvent(SingleLogEvent event, int count) { + this.event = event; + this.count = count; + } + } + + private final LinkedHashMap events = + new LinkedHashMap<>(MAX_LOGS + 1, (float) .75, true) { + + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_LOGS; + } + }; + + public synchronized void addEvent(SingleLogEvent event) { + String key = event.application + ":" + event.logger + ":" + event.level + ":" + event.message; + int count = events.containsKey(key) ? events.remove(key).count + 1 : 1; + events.put(key, new DedupedEvent(event, count)); + } + + public synchronized void clearEvents() { + events.clear(); + } + + public synchronized int numEvents() { + return events.size(); + } + + public synchronized boolean eventsIncludeErrors() { + return events.values().stream().anyMatch( + x -> x.event.level.equalsIgnoreCase("ERROR") || x.event.level.equalsIgnoreCase("FATAL")); + } + + public synchronized List getSanitizedEvents() { + return events.values().stream().map(ev -> new SanitizedLogEvent(ev.event, ev.count)) + .collect(Collectors.toList()); + } + +} diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/log.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/log.ftl index 98f68ec29be..66d62f2e035 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/log.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/log.ftl @@ -41,6 +41,7 @@ } }, { "data": "application" }, + { "data": "logger" }, { "data": "count" }, { "data": "level", "type": "html", @@ -75,7 +76,7 @@ } else { tr.addClass( 'details' ); - row.child( formatStacktrace(row.data().stacktrace) ).show(); + row.child( "
" + row.data().stacktrace + "
" ).show(); // Add to the 'open' array if ( idx === -1 ) { @@ -100,16 +101,6 @@ }); }); // end document ready - // format stacktrace - function formatStacktrace(d) { - var str = new String("
");
-          $.each(d, function( index, value ) {
-            str = str + value + "
";
-          });
-          str = str + "
"; - return str; - } - /** * Used to refresh the table */ @@ -125,6 +116,7 @@ Timestamp Application + Logger Count Level Message diff --git a/server/monitor/src/test/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppenderTest.java b/server/monitor/src/test/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppenderTest.java deleted file mode 100644 index 77ccea1b6ac..00000000000 --- a/server/monitor/src/test/java/org/apache/accumulo/monitor/util/AccumuloMonitorAppenderTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.util; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.util.Enumeration; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.accumulo.monitor.util.AccumuloMonitorAppender.MonitorLocation; -import org.apache.accumulo.monitor.util.AccumuloMonitorAppender.MonitorTracker; -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.spi.LoggingEvent; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.Timeout; - -public class AccumuloMonitorAppenderTest { - - @Rule - public Timeout timeout = new Timeout(10, TimeUnit.SECONDS); - - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void testActivateOptions() { - try (AccumuloMonitorAppender appender = new AccumuloMonitorAppender()) { - appender.executorService.shutdown(); - // simulate tracker having already been scheduled - appender.trackerScheduled.compareAndSet(false, true); - appender.activateOptions(); - // activateOptions should not trigger a RejectedExecutionException, because we tricked it into - // thinking it was already called, and therefore it did not - // schedule the tracker after shutting down - } - - exception.expect(RejectedExecutionException.class); - try (AccumuloMonitorAppender appender = new AccumuloMonitorAppender()) { - appender.executorService.shutdown(); - appender.activateOptions(); - fail("Calling activateOptions should have triggered a RejectedExecutionException"); - // this ensures that the activateOptions correctly attempts to schedule a worker - } - } - - @Test - public void testExecutorService() throws InterruptedException, ExecutionException { - ScheduledExecutorService executorService = null; - AtomicLong counter = new AtomicLong(2); - try (AccumuloMonitorAppender appender = new AccumuloMonitorAppender()) { - executorService = appender.executorService; - - // make sure executor service is started and running - assertFalse(executorService.isShutdown()); - assertFalse(executorService.isTerminated()); - - // make sure executor service executes tasks - ScheduledFuture future = - executorService.schedule(() -> counter.getAndIncrement(), 1, TimeUnit.MILLISECONDS); - assertEquals(Long.valueOf(2), future.get()); - assertEquals(3, counter.get()); - - // schedule a task that won't finish - executorService.schedule(() -> counter.getAndIncrement(), 1, TimeUnit.DAYS); - - // make sure executor service is still running - assertFalse(executorService.isShutdown()); - assertFalse(executorService.isTerminated()); - } - // verify that closing the appender shuts down the executor service threads - assertTrue(executorService.isShutdown()); - executorService.awaitTermination(5, TimeUnit.SECONDS); - assertTrue(executorService.isTerminated()); - - // verify executor service did not wait for scheduled task to run - assertEquals(3, counter.get()); - } - - @Test - public void testMonitorTracker() throws InterruptedException { - AtomicLong currentLoc = new AtomicLong(0); - Supplier locSupplier = () -> { - long loc = currentLoc.get(); - // for simplicity, create the location name from a number (0 represents no location) - byte[] location = loc == 0 ? null : ("loc" + loc).getBytes(UTF_8); - return new MonitorLocation(loc, location); - }; - Function appenderFactory = newLocation -> { - - return new AppenderSkeleton() { - - { - this.name = "Appender for " + newLocation.getLocation(); - } - - @Override - public boolean requiresLayout() { - return false; - } - - @Override - public void close() {} - - @Override - protected void append(LoggingEvent event) {} - - }; - - }; - - try (AccumuloMonitorAppender parent = new AccumuloMonitorAppender()) { - parent.setFrequency(1); // make it check frequently (every 1 millisecond) - parent.setTracker(new MonitorTracker(parent, locSupplier, appenderFactory)); - parent.activateOptions(); - - // initially there are no appenders - assertNull(parent.getAllAppenders()); - updateLocAndVerify(currentLoc, parent, 0); - updateLocAndVerify(currentLoc, parent, 10); - - // verify it's the same after a few times - // this verifies the logic in the tracker's run method which compares current location with - // last to see if a change occurred - AppenderSkeleton lastAppender = (AppenderSkeleton) parent.getAllAppenders().nextElement(); - for (int x = 0; x < 10; x++) { - Thread.sleep(10); - AppenderSkeleton currentAppender = - (AppenderSkeleton) parent.getAllAppenders().nextElement(); - assertSame(lastAppender, currentAppender); - } - - updateLocAndVerify(currentLoc, parent, 3); - updateLocAndVerify(currentLoc, parent, 0); - updateLocAndVerify(currentLoc, parent, 0); - updateLocAndVerify(currentLoc, parent, 12); - updateLocAndVerify(currentLoc, parent, 0); - updateLocAndVerify(currentLoc, parent, 335); - - updateLocAndVerify(currentLoc, parent, 0); - // verify we removed all the appenders - assertFalse(parent.getAllAppenders().hasMoreElements()); - } - } - - private static void updateLocAndVerify(AtomicLong currentLoc, AccumuloMonitorAppender parent, - int newLoc) throws InterruptedException { - // set the new location - currentLoc.set(newLoc); - // wait for the appender to notice the change - while (!verifyAppender(parent, newLoc == 0 ? null : ("loc" + newLoc))) { - Thread.sleep(10); - } - } - - private static boolean verifyAppender(AccumuloMonitorAppender parent, String newLocName) { - Enumeration childAppenders = parent.getAllAppenders(); - if (newLocName == null) { - return childAppenders == null || !childAppenders.hasMoreElements(); - } - if (childAppenders == null || !childAppenders.hasMoreElements()) { - return false; - } - AppenderSkeleton child = null; - try { - child = (AppenderSkeleton) childAppenders.nextElement(); - } catch (NoSuchElementException e) { - // Enumeration not thread safe; appender for old loc was removed after we did hasMoreElements - return false; - } - if (childAppenders.hasMoreElements()) { - fail("Appender should never have more than one child"); - } - return ("Appender for " + newLocName).equals(child.getName()); - } - -} diff --git a/server/native/pom.xml b/server/native/pom.xml index 3303e9700e4..37ed30de32a 100644 --- a/server/native/pom.xml +++ b/server/native/pom.xml @@ -46,13 +46,13 @@ test - org.slf4j - slf4j-api + org.apache.logging.log4j + log4j-slf4j-impl test org.slf4j - slf4j-log4j12 + slf4j-api test diff --git a/server/tracer/pom.xml b/server/tracer/pom.xml index 18ae981e275..0af4b54009f 100644 --- a/server/tracer/pom.xml +++ b/server/tracer/pom.xml @@ -87,13 +87,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/server/tserver/pom.xml b/server/tserver/pom.xml index c4d17f11362..362149add48 100644 --- a/server/tserver/pom.xml +++ b/server/tserver/pom.xml @@ -107,13 +107,13 @@ test - org.easymock - easymock + org.apache.logging.log4j + log4j-slf4j-impl test - org.slf4j - slf4j-log4j12 + org.easymock + easymock test diff --git a/shell/pom.xml b/shell/pom.xml index db05a3a0062..7fdb6cfbe94 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -105,6 +105,11 @@ junit test + + org.apache.logging.log4j + log4j-slf4j-impl + test + org.easymock easymock @@ -115,10 +120,5 @@ powermock-api-easymock test - - org.slf4j - slf4j-log4j12 - test - diff --git a/start/pom.xml b/start/pom.xml index c224c9c79cd..8f7f4fd7a5e 100644 --- a/start/pom.xml +++ b/start/pom.xml @@ -76,18 +76,23 @@ test - org.powermock - powermock-api-easymock + org.apache.logging.log4j + log4j-1.2-api + test + + + org.apache.logging.log4j + log4j-slf4j-impl test org.powermock - powermock-module-junit4 + powermock-api-easymock test - org.slf4j - slf4j-log4j12 + org.powermock + powermock-module-junit4 test diff --git a/test/pom.xml b/test/pom.xml index 0dbae0e87ac..eff48141ae0 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -70,10 +70,6 @@ junit junit - - log4j - log4j - org.apache.accumulo accumulo-core @@ -154,6 +150,10 @@ org.apache.htrace htrace-core + + org.apache.logging.log4j + log4j-1.2-api + org.apache.thrift libthrift @@ -193,8 +193,8 @@ test - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl test