From f3500217f0ba9753fc3f04e657cf25e0bb2c492c Mon Sep 17 00:00:00 2001 From: Norbert Bartels Date: Tue, 21 Jul 2015 23:24:36 +0200 Subject: [PATCH 1/2] LOG4J2-524: autodelete older files while rollover (based on work of @arun0009) --- .../core/appender/RollingFileAppender.java | 2 +- .../RollingRandomAccessFileAppender.java | 2 +- .../rolling/DefaultRolloverStrategy.java | 99 ++++++++++++++++++- 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingFileAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingFileAppender.java index 2e67ff81a3e..2f14a9b674b 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingFileAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingFileAppender.java @@ -173,7 +173,7 @@ public static RollingFileAppender createAppender( } if (strategy == null) { - strategy = DefaultRolloverStrategy.createStrategy(null, null, null, + strategy = DefaultRolloverStrategy.createStrategy(null, null, null, null, String.valueOf(Deflater.DEFAULT_COMPRESSION), config); } diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingRandomAccessFileAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingRandomAccessFileAppender.java index 4efdaa806a7..84df54455c5 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingRandomAccessFileAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/RollingRandomAccessFileAppender.java @@ -195,7 +195,7 @@ public static RollingRandomAccessFileAppender createAppender( } if (strategy == null) { - strategy = DefaultRolloverStrategy.createStrategy(null, null, null, + strategy = DefaultRolloverStrategy.createStrategy(null, null, null, null, String.valueOf(Deflater.DEFAULT_COMPRESSION), config); } diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java index 6da1b77c072..ba2d15d886c 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java @@ -17,7 +17,12 @@ package org.apache.logging.log4j.core.appender.rolling; import java.io.File; +import java.io.FileFilter; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -163,11 +168,13 @@ abstract Action createCompressAction(String renameTo, String compressedName, boo private static final int MIN_WINDOW_SIZE = 1; private static final int DEFAULT_WINDOW_SIZE = 7; + private static final int NO_MAX_AGE = 0; /** * Create the DefaultRolloverStrategy. * @param max The maximum number of files to keep. * @param min The minimum number of files to keep. + * @param maxAge The maximum days to keep files, use 0 to keep files forever * @param fileIndex If set to "max" (the default), files with a higher index will be newer than files with a * smaller index. If set to "min", file renaming and the counter will follow the Fixed Window strategy. * @param compressionLevelStr The compression level, 0 (less) through 9 (more); applies only to ZIP files. @@ -178,6 +185,7 @@ abstract Action createCompressAction(String renameTo, String compressedName, boo public static DefaultRolloverStrategy createStrategy( @PluginAttribute("max") final String max, @PluginAttribute("min") final String min, + @PluginAttribute("maxAge") final String maxAge, @PluginAttribute("fileIndex") final String fileIndex, @PluginAttribute("compressionLevel") final String compressionLevelStr, @PluginConfiguration final Configuration config) { @@ -198,8 +206,14 @@ public static DefaultRolloverStrategy createStrategy( LOGGER.error("Maximum window size must be greater than the minimum windows size. Set to " + maxIndex); } } + int maxAgeIndex = NO_MAX_AGE; + if (maxAge != null) { + int maxAgeInt = Integer.parseInt(maxAge); + // max age may not be negative + maxAgeIndex = (maxAgeInt > NO_MAX_AGE) ? maxAgeInt : NO_MAX_AGE; + } final int compressionLevel = Integers.parseInt(compressionLevelStr, Deflater.DEFAULT_COMPRESSION); - return new DefaultRolloverStrategy(minIndex, maxIndex, useMax, compressionLevel, config.getStrSubstitutor()); + return new DefaultRolloverStrategy(minIndex, maxIndex, maxAgeIndex, useMax, compressionLevel, config.getStrSubstitutor()); } /** @@ -214,19 +228,26 @@ public static DefaultRolloverStrategy createStrategy( private final boolean useMax; private final StrSubstitutor subst; private final int compressionLevel; + private final int maxAgeIndex; /** * Constructs a new instance. * @param minIndex The minimum index. * @param maxIndex The maximum index. + * @param maxAgeIndex The maximum age of log files. */ - protected DefaultRolloverStrategy(final int minIndex, final int maxIndex, final boolean useMax, final int compressionLevel, final StrSubstitutor subst) { + protected DefaultRolloverStrategy(final int minIndex, final int maxIndex, final int maxAgeIndex, final boolean useMax, final int compressionLevel, final StrSubstitutor subst) { this.minIndex = minIndex; this.maxIndex = maxIndex; this.useMax = useMax; + this.maxAgeIndex = maxAgeIndex; this.compressionLevel = compressionLevel; this.subst = subst; } + + public int getMaxAgeIndex() { + return this.maxAgeIndex; + } public int getCompressionLevel() { return this.compressionLevel; @@ -244,6 +265,48 @@ private int purge(final int lowIndex, final int highIndex, final RollingFileMana return useMax ? purgeAscending(lowIndex, highIndex, manager) : purgeDescending(lowIndex, highIndex, manager); } + + /** + * Purge files older than defined maxAge. + * + * If file older than current date - maxAge delete them or else keep it. + * If maxAgeIndex is 0, no file is deleted + * + * @param maxAgeIndex maxAge Index + * @param manager The RollingFileManager + * @return true if purge was successful and rollover should be attempted. + */ + private void purgeMaxAgeFiles(final int maxAgeIndex, final RollingFileManager manager) { + + if (maxAgeIndex == NO_MAX_AGE) { + return; + } + + // LOG4J2-531: directory scan & rollover must use same format + String filename = manager.getFileName(); + File file = new File(filename); + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.DATE, -maxAgeIndex); + Date cutoffDate = cal.getTime(); + + if (file.getParentFile().exists()) { + filename = file.getName().replaceAll("\\..*", ""); + + File[] files = file.getParentFile().listFiles( + new StartsWithFileFilter(filename, false)); + + for (File purgeCandidate : files) { + try { + BasicFileAttributes attr = Files.readAttributes(purgeCandidate.toPath(), BasicFileAttributes.class); + if (new Date(attr.creationTime().toMillis()).before(cutoffDate)) { + purgeCandidate.delete(); + } + } catch(Exception e) { + LOGGER.warn("unable to get basic file attributes : ", e); + } + } + } + } /** * Purge and rename old log files in preparation for rollover. The oldest file will have the smallest index, @@ -473,7 +536,8 @@ private int suffixLength(final String lowFilename) { */ @Override public RolloverDescription rollover(final RollingFileManager manager) throws SecurityException { - if (maxIndex < 0) { + purgeMaxAgeFiles(maxAgeIndex, manager); + if (maxIndex < 0) { return null; } final long startNanos = System.nanoTime(); @@ -509,7 +573,34 @@ public RolloverDescription rollover(final RollingFileManager manager) throws Sec @Override public String toString() { - return "DefaultRolloverStrategy(min=" + minIndex + ", max=" + maxIndex + ')'; + return "DefaultRolloverStrategy(min=" + minIndex + ", max=" + maxIndex + ", maxAgeIndex=" + maxAgeIndex + ')'; + } + + class StartsWithFileFilter implements FileFilter { + private final String startsWith; + private final boolean inclDirs; + + /** + * + */ + public StartsWithFileFilter(String startsWith, + boolean includeDirectories) { + super(); + this.startsWith = startsWith.toUpperCase(); + inclDirs = includeDirectories; + } + + /* + * (non-Javadoc) + * + * @see java.io.FileFilter#accept(java.io.File) + */ + public boolean accept(File pathname) { + if (!inclDirs && pathname.isDirectory()) { + return false; + } else + return pathname.getName().toUpperCase().startsWith(startsWith); + } } } From f8a9c5b717244b22eb9e0eaa2ee216e3391bc87b Mon Sep 17 00:00:00 2001 From: Norbert Bartels Date: Wed, 22 Jul 2015 01:51:42 +0200 Subject: [PATCH 2/2] LOG4J-524: unittest added, not working correctly atm --- .../rolling/DefaultRolloverStrategy.java | 11 +- .../rolling/RollingAppenderMaxAgeTest.java | 124 ++++++++++++++++++ .../test/resources/log4j-rolling-maxage.xml | 59 +++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 log4j-core/src/test/java/org/apache/logging/log4j/core/appender/rolling/RollingAppenderMaxAgeTest.java create mode 100644 log4j-core/src/test/resources/log4j-rolling-maxage.xml diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java index ba2d15d886c..fea987c1243 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/rolling/DefaultRolloverStrategy.java @@ -282,15 +282,22 @@ private void purgeMaxAgeFiles(final int maxAgeIndex, final RollingFileManager ma return; } + final StringBuilder buf = new StringBuilder(); + // LOG4J2-531: directory scan & rollover must use same format - String filename = manager.getFileName(); + manager.getPatternProcessor().formatFileName(buf, 0); + + String filename = buf.toString(); File file = new File(filename); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, -maxAgeIndex); Date cutoffDate = cal.getTime(); if (file.getParentFile().exists()) { - filename = file.getName().replaceAll("\\..*", ""); + // find the file prefix and ignore everything beyond the first subst element + // this is a bit dirty, but no other way to get the rolling files is known + String shortname = file.getName().substring(0, file.getName().indexOf("$")); + filename = shortname.replaceAll("\\..*", ""); File[] files = file.getParentFile().listFiles( new StartsWithFileFilter(filename, false)); diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/rolling/RollingAppenderMaxAgeTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/rolling/RollingAppenderMaxAgeTest.java new file mode 100644 index 00000000000..4fc9ba79d0b --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/rolling/RollingAppenderMaxAgeTest.java @@ -0,0 +1,124 @@ +/* + * Copyright 2015 Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.logging.log4j.core.appender.rolling; + +import java.io.File; +import java.nio.file.Files; +import static java.nio.file.LinkOption.NOFOLLOW_LINKS; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import org.apache.logging.log4j.Logger; +import static org.apache.logging.log4j.hamcrest.Descriptors.that; +import static org.apache.logging.log4j.hamcrest.FileMatchers.hasName; +import org.apache.logging.log4j.junit.InitialLoggerContext; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasItemInArray; +import org.junit.After; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class RollingAppenderMaxAgeTest { + + private static final String DIR = "target/rolling1"; + + private final String fileExtension; + + private Logger logger; + + @Parameterized.Parameters(name = "{0} \u2192 {1}") + public static Collection data() { + return Arrays.asList( + new Object[][]{ + { "log4j-rolling-maxage.xml", ".gz" }, + } + ); + } + + @Rule + public InitialLoggerContext init; + + public RollingAppenderMaxAgeTest(final String configFile, final String fileExtension) { + this.fileExtension = fileExtension; + this.init = new InitialLoggerContext(configFile); + } + + @Before + public void setUp() throws Exception { + this.logger = this.init.getLogger(RollingAppenderSizeTest.class.getName()); + } + + @After + public void tearDown() throws Exception { + deleteDir(); + } + + @Test + public void testAppender() throws Exception { + for (int i=0; i < 100; ++i) { + logger.debug("This is test message number " + i); + } + final File dir = new File(DIR); + assertTrue("Directory not created", dir.exists() && dir.listFiles().length > 0); + final File[] files = dir.listFiles(); + Thread.sleep(50); + assertNotNull(files); + assertEquals(8, files.length); + // change creation time + + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.DATE, -5); + FileTime fileTime = FileTime.fromMillis(cal.getTimeInMillis()); + + for (File file : files) { + if (file.getName().endsWith(".zip")) { + BasicFileAttributeView attributes = Files.getFileAttributeView(file.toPath(), BasicFileAttributeView.class); + attributes.setTimes(fileTime, fileTime, fileTime); + } + } + + for (int i=0; i < 2; ++i) { + logger.debug("This is test message number " + i); + Thread.sleep(50); + } + + final File dir2 = new File(DIR); + final File[] filesNew = dir2.listFiles(); + assertNotNull(filesNew); + assertEquals(2, files.length); + } + + private static void deleteDir() { + final File dir = new File(DIR); + if (dir.exists()) { + final File[] files = dir.listFiles(); + for (final File file : files) { + file.delete(); + } + dir.delete(); + } + } +} diff --git a/log4j-core/src/test/resources/log4j-rolling-maxage.xml b/log4j-core/src/test/resources/log4j-rolling-maxage.xml new file mode 100644 index 00000000000..0866997cf44 --- /dev/null +++ b/log4j-core/src/test/resources/log4j-rolling-maxage.xml @@ -0,0 +1,59 @@ + + + + + target/rolling1/rollingtest.log + + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + + + + + + > + + + + > + + + + + + + \ No newline at end of file