From 26b329486b690f1784f780601d90b09cb5974451 Mon Sep 17 00:00:00 2001 From: Aleksey Zvolinsky Date: Mon, 2 Nov 2015 08:28:10 +0200 Subject: [PATCH 1/4] LOG4J2-649: Purging dynamically created routing sub-appenders 1. Added PurgePolicy interface 2. Added IdlePurgePolicy which is purging sub-appenders that were not in use last X minutes 3. Added ability to access sub-appenders, so custom purging controlled outside log4j2 also possible. --- .../appender/routing/IdlePurgePolicy.java | 80 +++++++++++++++++++ .../core/appender/routing/PurgePolicy.java | 31 +++++++ .../appender/routing/RoutingAppender.java | 33 +++++++- 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java create mode 100644 log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java new file mode 100644 index 00000000000..e54bff24b25 --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java @@ -0,0 +1,80 @@ +package org.apache.logging.log4j.core.appender.routing; + +import java.util.Calendar; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; +import org.apache.logging.log4j.status.StatusLogger; + +/** + * + * Policy is purging appenders that were not in use specified time in minutes + * + */ +@Plugin(name = "IdlePurgePolicy", category = "Core", printObject = true) +public class IdlePurgePolicy implements PurgePolicy { + + private static final Logger LOGGER = StatusLogger.getLogger(); + private int timeToLive; + private final ConcurrentMap appendersUsage = new ConcurrentHashMap<>(); + private RoutingAppender routingAppender; + + public IdlePurgePolicy(int timeToLive) { + this.timeToLive = timeToLive; + } + + @Override + public void initialize(RoutingAppender routingAppender) { + this.routingAppender = routingAppender; + } + + /** + * Purging appenders that were not in use specified time + * + */ + @Override + public void purge() { + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.MINUTE, -timeToLive); + + for (Entry entry : appendersUsage.entrySet()) { + if(calendar.after(entry.getValue())) { + appendersUsage.remove(entry.getKey()); + routingAppender.deleteAppender(entry.getKey()); + } + } + } + + @Override + public void update(String key, LogEvent event) { + appendersUsage.put(key, Calendar.getInstance()); + } + + /** + * Create the Routes. + * @param pattern The pattern. + * @param routes An array of Route elements. + * @return The Routes container. + */ + @PluginFactory + public static PurgePolicy createPurgePolicy( + @PluginAttribute("timetolive") final String time) { + + if (time == null) { + LOGGER.error("A timeToLive is required"); + return null; + } + + final int timeToLive = Integers.parseInt(time); + + return new IdlePurgePolicy(timeToLive); + } + +} diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java new file mode 100644 index 00000000000..cfac2fa2070 --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java @@ -0,0 +1,31 @@ +package org.apache.logging.log4j.core.appender.routing; + +import org.apache.logging.log4j.core.LogEvent; + +/** + * + * Policy for purging routed appenders + * + */ +public interface PurgePolicy { + + /** + * Activate purging appenders + */ + void purge(); + + /** + * + * @param routed appender key + * @param event + */ + void update(String key, LogEvent event); + + /** + * Initialize with routing appender + * + * @param routingAppender + */ + void initialize(RoutingAppender routingAppender); + +} diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java index d9b2818769b..bc83edcbb1a 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java @@ -16,6 +16,7 @@ */ package org.apache.logging.log4j.core.appender.routing; +import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,13 +53,18 @@ public final class RoutingAppender extends AbstractAppender { private final Configuration config; private final ConcurrentMap appenders = new ConcurrentHashMap<>(); private final RewritePolicy rewritePolicy; + private final PurgePolicy purgePolicy; private RoutingAppender(final String name, final Filter filter, final boolean ignoreExceptions, final Routes routes, - final RewritePolicy rewritePolicy, final Configuration config) { + final RewritePolicy rewritePolicy, final Configuration config, PurgePolicy purgePolicy) { super(name, filter, null, ignoreExceptions); this.routes = routes; this.config = config; this.rewritePolicy = rewritePolicy; + this.purgePolicy = purgePolicy; + if(this.purgePolicy != null) { + this.purgePolicy.initialize(this); + } Route defRoute = null; for (final Route route : routes.getRoutes()) { if (route.getKey() == null) { @@ -111,9 +117,14 @@ public void append(LogEvent event) { if (control != null) { control.callAppender(event); } + + if(purgePolicy != null) { + purgePolicy.update(key, event); + purgePolicy.purge(); + } } - private synchronized AppenderControl getControl(final String key, final LogEvent event) { + private synchronized AppenderControl getControl(final String key, final LogEvent event) { AppenderControl control = appenders.get(key); if (control != null) { return control; @@ -162,6 +173,21 @@ private Appender createAppender(final Route route, final LogEvent event) { LOGGER.error("No Appender was configured for route " + route.getKey()); return null; } + + public Map getAppenders() { + return Collections.unmodifiableMap(appenders); + } + + /** + * Delete specified appender + * + * @param key + */ + public void deleteAppender(String key) { + LOGGER.debug("Stopping route with key" + key); + AppenderControl control = appenders.remove(key); + control.getAppender().stop(); + } /** * Create a RoutingAppender. @@ -181,6 +207,7 @@ public static RoutingAppender createAppender( @PluginElement("Routes") final Routes routes, @PluginConfiguration final Configuration config, @PluginElement("RewritePolicy") final RewritePolicy rewritePolicy, + @PluginElement("PurgePolicy") final PurgePolicy purgePolicy, @PluginElement("Filter") final Filter filter) { final boolean ignoreExceptions = Booleans.parseBoolean(ignore, true); @@ -192,6 +219,6 @@ public static RoutingAppender createAppender( LOGGER.error("No routes defined for RoutingAppender"); return null; } - return new RoutingAppender(name, filter, ignoreExceptions, routes, rewritePolicy, config); + return new RoutingAppender(name, filter, ignoreExceptions, routes, rewritePolicy, config, purgePolicy); } } From 16891c62efc863e49265c7ea3691b205a91a4680 Mon Sep 17 00:00:00 2001 From: Aleksey Zvolinsky Date: Tue, 3 Nov 2015 16:40:32 +0200 Subject: [PATCH 2/4] Added timeUnit and applied camel case for attributes --- .../appender/routing/IdlePurgePolicy.java | 22 ++++++++++++++----- .../appender/routing/RoutingAppender.java | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java index e54bff24b25..b47a3d90179 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java @@ -4,6 +4,7 @@ import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LogEvent; @@ -23,11 +24,15 @@ public class IdlePurgePolicy implements PurgePolicy { private static final Logger LOGGER = StatusLogger.getLogger(); private int timeToLive; + private TimeUnit timeUnit; + private final ConcurrentMap appendersUsage = new ConcurrentHashMap<>(); private RoutingAppender routingAppender; - public IdlePurgePolicy(int timeToLive) { + public IdlePurgePolicy(int timeToLive, TimeUnit timeUnit) { this.timeToLive = timeToLive; + this.timeUnit = timeUnit; + } @Override @@ -42,10 +47,10 @@ public void initialize(RoutingAppender routingAppender) { @Override public void purge() { Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.MINUTE, -timeToLive); + calendar.setTimeInMillis(calendar.getTimeInMillis()-timeUnit.toMillis(timeToLive)); for (Entry entry : appendersUsage.entrySet()) { - if(calendar.after(entry.getValue())) { + if(calendar.after(entry.getValue())) { appendersUsage.remove(entry.getKey()); routingAppender.deleteAppender(entry.getKey()); } @@ -65,7 +70,8 @@ public void update(String key, LogEvent event) { */ @PluginFactory public static PurgePolicy createPurgePolicy( - @PluginAttribute("timetolive") final String time) { + @PluginAttribute("timeToLive") final String time, + @PluginAttribute("timeUnit") final String timeUnit) { if (time == null) { LOGGER.error("A timeToLive is required"); @@ -74,7 +80,13 @@ public static PurgePolicy createPurgePolicy( final int timeToLive = Integers.parseInt(time); - return new IdlePurgePolicy(timeToLive); + if(timeUnit == null) { + LOGGER.error("A timeUnit is required"); + return null; + } + TimeUnit unit = TimeUnit.valueOf(timeUnit.toUpperCase()); + + return new IdlePurgePolicy(timeToLive, unit); } } diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java index bc83edcbb1a..09bc1526f52 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java @@ -184,7 +184,7 @@ public Map getAppenders() { * @param key */ public void deleteAppender(String key) { - LOGGER.debug("Stopping route with key" + key); + LOGGER.debug("Stopping route with key " + key); AppenderControl control = appenders.remove(key); control.getAppender().stop(); } From ca94da87bf203fd7d4767918455b866611628f46 Mon Sep 17 00:00:00 2001 From: Aleksey Zvolinsky Date: Tue, 3 Nov 2015 23:14:31 +0200 Subject: [PATCH 3/4] Added tests for RoutingAppender purging feature --- .../RoutingAppenderWithPurgingTest.java | 108 ++++++++++++++++++ .../test/resources/log4j-routing-purge.xml | 71 ++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java create mode 100644 log4j-core/src/test/resources/log4j-routing-purge.xml diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java new file mode 100644 index 00000000000..6cea9f9af73 --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.core.appender.routing; + +import static org.junit.Assert.*; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.EventLogger; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.junit.CleanFiles; +import org.apache.logging.log4j.junit.LoggerContextRule; +import org.apache.logging.log4j.message.StructuredDataMessage; +import org.apache.logging.log4j.test.appender.ListAppender; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +/** + * Testing Routing appender purge facilities + */ +public class RoutingAppenderWithPurgingTest { + private static final String CONFIG = "log4j-routing-purge.xml"; + private static final String IDLE_LOG_FILE1 = "target/routing-purge-idle/routingtest-1.log"; + private static final String IDLE_LOG_FILE2 = "target/routing-purge-idle/routingtest-2.log"; + private static final String IDLE_LOG_FILE3 = "target/routing-purge-idle/routingtest-3.log"; + private static final String MANUAL_LOG_FILE1 = "target/routing-purge-manual/routingtest-1.log"; + private static final String MANUAL_LOG_FILE2 = "target/routing-purge-manual/routingtest-2.log"; + private static final String MANUAL_LOG_FILE3 = "target/routing-purge-manual/routingtest-3.log"; + + + private ListAppender app; + private RoutingAppender routingAppenderIdle; + private RoutingAppender routingAppenderManual; + + @Rule + public LoggerContextRule init = new LoggerContextRule(CONFIG); + + @Rule + public CleanFiles files = new CleanFiles(IDLE_LOG_FILE1, IDLE_LOG_FILE2, IDLE_LOG_FILE3, + MANUAL_LOG_FILE1, MANUAL_LOG_FILE2, MANUAL_LOG_FILE3); + + + @Before + public void setUp() throws Exception { + this.app = this.init.getListAppender("List"); + this.routingAppenderIdle = this.init.getRequiredAppender("RoutingPurgeIdle", RoutingAppender.class); + this.routingAppenderManual = this.init.getRequiredAppender("RoutingPurgeManual", RoutingAppender.class); + } + + @After + public void tearDown() throws Exception { + this.app.clear(); + } + + @Test + public void routingTest() throws InterruptedException { + StructuredDataMessage msg = new StructuredDataMessage("1", "This is a test 1", "Service"); + EventLogger.logEvent(msg); + final List list = app.getEvents(); + assertNotNull("No events generated", list); + assertTrue("Incorrect number of events. Expected 1, got " + list.size(), list.size() == 1); + msg = new StructuredDataMessage("2", "This is a test 2", "Service"); + EventLogger.logEvent(msg); + msg = new StructuredDataMessage("3", "This is a test 3", "Service"); + EventLogger.logEvent(msg); + String[] files = {IDLE_LOG_FILE1, IDLE_LOG_FILE2, IDLE_LOG_FILE3, MANUAL_LOG_FILE1, MANUAL_LOG_FILE2, MANUAL_LOG_FILE3}; + assertFileExistance(files); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 3, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders manual purge.", 3, routingAppenderManual.getAppenders().size()); + + Thread.sleep(4000); + EventLogger.logEvent(msg); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 1, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders with manual purge.", 3, routingAppenderManual.getAppenders().size()); + + routingAppenderManual.deleteAppender("1"); + routingAppenderManual.deleteAppender("2"); + routingAppenderManual.deleteAppender("3"); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 1, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders with manual purge.", 0, routingAppenderManual.getAppenders().size()); + } + + private void assertFileExistance(String... files) { + for (String file : files) { + assertTrue("File should exist - " + file + " file ", new File(file).exists()); + } + } +} diff --git a/log4j-core/src/test/resources/log4j-routing-purge.xml b/log4j-core/src/test/resources/log4j-routing-purge.xml new file mode 100644 index 00000000000..41647dc22f9 --- /dev/null +++ b/log4j-core/src/test/resources/log4j-routing-purge.xml @@ -0,0 +1,71 @@ + + + + + target/routing-purge-idle/routingtest-$${sd:id}.log + target/routing-purge-manual/routingtest-$${sd:id}.log + + + + + + + + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 997be17bde105bb054dc420240d31972d53427dd Mon Sep 17 00:00:00 2001 From: Aleksey Zvolinsky Date: Mon, 16 Nov 2015 15:50:54 +0200 Subject: [PATCH 4/4] Replaced usage of Calendar and added licenses 1. Replaced usage of Calendar objects with System.currentTimeMillis() method 2. Added missing license headers --- .../appender/routing/IdlePurgePolicy.java | 28 ++++++++++++++----- .../core/appender/routing/PurgePolicy.java | 16 +++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java index b47a3d90179..2637121b1e3 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java @@ -1,6 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ package org.apache.logging.log4j.core.appender.routing; -import java.util.Calendar; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -26,7 +41,7 @@ public class IdlePurgePolicy implements PurgePolicy { private int timeToLive; private TimeUnit timeUnit; - private final ConcurrentMap appendersUsage = new ConcurrentHashMap<>(); + private final ConcurrentMap appendersUsage = new ConcurrentHashMap<>(); private RoutingAppender routingAppender; public IdlePurgePolicy(int timeToLive, TimeUnit timeUnit) { @@ -46,11 +61,10 @@ public void initialize(RoutingAppender routingAppender) { */ @Override public void purge() { - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(calendar.getTimeInMillis()-timeUnit.toMillis(timeToLive)); + long expiredTime = System.currentTimeMillis() - timeUnit.toMillis(timeToLive); - for (Entry entry : appendersUsage.entrySet()) { - if(calendar.after(entry.getValue())) { + for (Entry entry : appendersUsage.entrySet()) { + if(expiredTime > entry.getValue()) { appendersUsage.remove(entry.getKey()); routingAppender.deleteAppender(entry.getKey()); } @@ -59,7 +73,7 @@ public void purge() { @Override public void update(String key, LogEvent event) { - appendersUsage.put(key, Calendar.getInstance()); + appendersUsage.put(key, System.currentTimeMillis()); } /** diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java index cfac2fa2070..a90b6a8aa73 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ package org.apache.logging.log4j.core.appender.routing; import org.apache.logging.log4j.core.LogEvent;