Skip to content

Commit

Permalink
adding maxArchies support in local file handler (+ a few fixes/enhanc…
Browse files Browse the repository at this point in the history 
…ements in behavior)
  • Loading branch information
@rmannibucau
rmannibucau committed Sep 5, 2022
1 parent 2b68f86 commit 7335046
Show file tree
Hide file tree
Showing 4 changed files with 188 additions and 68 deletions.
5 changes: 3 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.7.0</version>
<version>5.9.0</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand All @@ -56,7 +56,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.30</version>
<version>1.7.36</version>
</dependency>
</dependencies>
</dependencyManagement>
Expand All @@ -80,6 +80,7 @@
<target>11</target>
<release>11</release>
<encoding>UTF-8</encoding>
<parameters>true</parameters>
</configuration>
</plugin>
<plugin>
Expand Down
1 change: 1 addition & 0 deletions src/main/minisite/content/jul-integration.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Here is its configuration - all are prefixed with `io.yupiik.logging.jul.handler
| archiveOlderThan | -1 | how many days files are kept before being compressed
| purgeOlderThan | -1 | how many days files are kept before being deleted, note: it applies on archives and not log files so 2 days of archiving and 3 days of purge makes it deleted after 5 days.
| compressionLevel | -1 | In case of zip archiving the zip compression level (-1 for off or 0-9).
| maxArchives | -1 | Max number of archives (zip/gzip) to keep, ignored if negative (you can review `io.yupiik.logging.jul.handler.LocalFileHandlerTest.purgeMaxArchive` for some sample configuration).
|===

== Pattern formatter
Expand Down
168 changes: 102 additions & 66 deletions yupiik-logging-jul/src/main/java/io/yupiik/logging/jul/handler/LocalFileHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,16 @@

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.sql.Timestamp;
import java.time.Clock;
import java.time.Duration;
import java.time.LocalDate;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
Expand All @@ -40,6 +38,7 @@
import java.util.logging.LogManager;
import java.util.logging.LogRecord;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
Expand All @@ -49,7 +48,7 @@

// from https://github.com/apache/tomee/blob/master/tomee/tomee-juli/src/main/java/org/apache/tomee/jul/handler/rotating/LocalFileHandler.java
public class LocalFileHandler extends Handler {
private static final int BUFFER_SIZE = 8102;
private final Clock clock;

private long limit = 0;
private int bufferSize = -1;
Expand All @@ -59,6 +58,7 @@ public class LocalFileHandler extends Handler {
private String archiveFormat = "gzip";
private long dateCheckInterval;
private long archiveExpiryDuration;
private int maxArchives = -1;
private int compressionLevel;
private long purgeExpiryDuration;
private File archiveDir;
Expand All @@ -68,6 +68,8 @@ public class LocalFileHandler extends Handler {
private volatile String date;
private volatile PrintWriter writer;
private volatile int written;
private volatile File currentFile;

private final ReadWriteLock writerLock = new ReentrantReadWriteLock();
private final Lock backgroundTaskLock = new ReentrantLock();
private volatile boolean closed;
Expand All @@ -76,6 +78,11 @@ public class LocalFileHandler extends Handler {
private boolean truncateIfExists;

public LocalFileHandler() {
this(Clock.systemDefaultZone());
}

public LocalFileHandler(final Clock clock) {
this.clock = clock;
configure();
}

Expand Down Expand Up @@ -112,6 +119,7 @@ private void configure() {
archiveFilenameRegex = Pattern.compile(fileNameReg + "\\." + archiveFormat);

purgeExpiryDuration = getProperty(className + ".purgeOlderThan", v -> Duration.parse(v).toMillis(), () -> -1L);
maxArchives = getProperty(className + ".maxArchives", Integer::parseInt, () -> -1);

try {
bufferSize = getProperty(className + ".bufferSize", Integer::parseInt, () -> -1);
Expand All @@ -120,11 +128,11 @@ private void configure() {
}

//setErrorManager(new ErrorManager());
lastTimestamp = System.currentTimeMillis();
lastTimestamp = clock.instant().toEpochMilli();
}

protected String currentDate() {
return new Timestamp(System.currentTimeMillis()).toString().substring(0, 10);
return LocalDate.ofInstant(clock.instant(), clock.getZone()).toString();
}

@Override
Expand All @@ -133,7 +141,7 @@ public void publish(final LogRecord record) {
return;
}

final long now = System.currentTimeMillis();
final long now = clock.instant().toEpochMilli();
// just do it once / sec if we have a lot of log, can make some log appearing in the wrong file but better than doing it each time
if (dateCheckInterval < 0 || now - lastTimestamp > dateCheckInterval) { // using as much as possible volatile to avoid to lock too much
lastTimestamp = now;
Expand Down Expand Up @@ -221,6 +229,7 @@ public void close() {
writer.write(getFormatter().getTail(this));
writer.flush();
writer.close();
currentFile = null;
writer = null;
} catch (final Exception e) {
reportError(null, e, ErrorManager.CLOSE_FAILURE);
Expand All @@ -246,11 +255,11 @@ public void flush() {
}

protected synchronized void openWriter() {
final long beforeRotation = System.currentTimeMillis();
final var now = clock.instant();
final long beforeRotation = now.toEpochMilli();

writerLock.writeLock().lock();
OutputStream fos;
OutputStream os = null;
OutputStream fos = null;
try {
File pathname;
do {
Expand All @@ -259,22 +268,26 @@ protected synchronized void openWriter() {
if (!parent.isDirectory() && !parent.mkdirs()) {
reportError("Unable to create [" + parent + "]", null, ErrorManager.OPEN_FAILURE);
writer = null;
currentFile = null;
return;
}
currentIndex++;
} while (!overwrite && pathname.isFile()); // loop to ensure we don't overwrite existing files

final String encoding = getEncoding();
fos = new FileOutputStream(pathname, !truncateIfExists);
os = new CountingStream(bufferSize > 0 ? new BufferedOutputStream(fos, bufferSize) : fos);
writer = new PrintWriter((encoding != null) ? new OutputStreamWriter(os, encoding) : new OutputStreamWriter(os), false);
final var os = new CountingStream(bufferSize > 0 ? new BufferedOutputStream(fos, bufferSize) : fos);
final var encoding = getEncoding();
final var streamWriter = (encoding != null) ? new OutputStreamWriter(os, encoding) : new OutputStreamWriter(os);
writer = new PrintWriter(streamWriter, false);
writer.write(getFormatter().getHead(this));
currentFile = pathname;
} catch (final Exception e) {
reportError(null, e, ErrorManager.OPEN_FAILURE);
writer = null;
if (os != null) {
currentFile = null;
if (fos != null) {
try {
os.close();
fos.close();
} catch (final IOException e1) {
// no-op
}
Expand All @@ -294,49 +307,84 @@ protected synchronized void openWriter() {
}

private void evict(final long now) {
if (purgeExpiryDuration > 0) { // purging archives
final File[] archives = archiveDir.listFiles((dir, name) -> archiveFilenameRegex.matcher(name).matches());

if (archives != null) {
for (final File archive : archives) {
try {
final BasicFileAttributes attr = Files.readAttributes(archive.toPath(), BasicFileAttributes.class);
if (now - attr.creationTime().toMillis() > purgeExpiryDuration) {
if (!Files.deleteIfExists(archive.toPath())) {
// dont try to delete on exit cause we will find it again
reportError("Can't delete " + archive.getAbsolutePath() + ".", null, ErrorManager.GENERIC_FAILURE);
}
if (purgeExpiryDuration > 0) {
purgeArchives(now);
}
if (archiveExpiryDuration > 0) {
archiveIfNeeded(now);
}
if (maxArchives > 0) {
deleteUndesiredArchives();
}
}

private void purgeArchives(final long now) {
final var archives = listArchives();
if (archives != null) {
for (final var archive : archives) {
try {
final var attr = Files.readAttributes(archive.toPath(), BasicFileAttributes.class);
if (now - attr.creationTime().toMillis() > purgeExpiryDuration) {
if (!Files.deleteIfExists(archive.toPath())) {
// dont try to delete on exit cause we will find it again
reportError("Can't delete " + archive.getAbsolutePath() + ".", null, ErrorManager.GENERIC_FAILURE);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
}
}
if (archiveExpiryDuration > 0) { // archiving log files
final File[] logs = new File(formatFilename(filenamePattern, "0000-00-00", 0)).getParentFile()
.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File dir, final String name) {
return filenameRegex.matcher(name).matches();
}
});

if (logs != null) {
for (final File file : logs) {
try {
final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
if (attr.creationTime().toMillis() < now && now - attr.lastModifiedTime().toMillis() > archiveExpiryDuration) {
createArchive(file);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}

private void archiveIfNeeded(final long now) {
final File[] logs = new File(formatFilename(filenamePattern, "0000-00-00", 0)).getParentFile()
.listFiles((dir, name) -> filenameRegex.matcher(name).matches());

if (logs != null) {
for (final var file : logs) {
try {
final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
if (!file.equals(currentFile) && shouldArchive(now, attr)) {
createArchive(file);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
}
}
}

private void deleteUndesiredArchives() {
final var archives = listArchives();
if (archives != null) {
final var toDelete = archives.length - maxArchives;
if (toDelete > 0) {
final var sorted = Stream.of(archives)
.sorted((a, b) -> { // older first
try {
return Files.readAttributes(a.toPath(), BasicFileAttributes.class).creationTime()
.compareTo(Files.readAttributes(b.toPath(), BasicFileAttributes.class).creationTime());
} catch (final IOException ie) {
getErrorManager().error(ie.getMessage(), ie, ErrorManager.GENERIC_FAILURE);
return a.getName().compareTo(b.getName());
}
}).toArray(File[]::new);
Stream.of(sorted)
.limit(toDelete)
.forEach(File::delete);
}
}
}

private boolean shouldArchive(final long now, final BasicFileAttributes attr) {
return attr.creationTime().toMillis() < now && now - attr.lastModifiedTime().toMillis() > archiveExpiryDuration;
}

private File[] listArchives() {
return archiveDir.listFiles((dir, name) -> archiveFilenameRegex.matcher(name).matches());
}

private String formatFilename(final String pattern, final String date, final int index) {
return String.format(pattern, date, index);
}
Expand All @@ -353,25 +401,20 @@ private void createArchive(final File source) {
}

if (archiveFormat.equalsIgnoreCase("gzip")) {
try (final OutputStream outputStream = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(target)))) {
final byte[] buffer = new byte[BUFFER_SIZE];
try (final FileInputStream inputStream = new FileInputStream(source)) {
copyStream(inputStream, outputStream, buffer);
} catch (final IOException e) {
throw new IllegalStateException(e);
}
try (final var outputStream = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(target)))) {
Files.copy(source.toPath(), outputStream);
} catch (final IOException e) {
throw new IllegalStateException(e);
}
} else { // consider file defines a zip whatever extension it is
try (final ZipOutputStream outputStream = new ZipOutputStream(new FileOutputStream(target))) {
try (final var outputStream = new ZipOutputStream(new FileOutputStream(target))) {
outputStream.setLevel(compressionLevel);

final byte[] buffer = new byte[BUFFER_SIZE];
try (final FileInputStream inputStream = new FileInputStream(source)) {
try {
final ZipEntry zipEntry = new ZipEntry(source.getName());
outputStream.putNextEntry(zipEntry);
copyStream(inputStream, outputStream, buffer);
Files.copy(source.toPath(), outputStream);
outputStream.closeEntry();
} catch (final IOException e) {
throw new IllegalStateException(e);
}
Expand All @@ -388,13 +431,6 @@ private void createArchive(final File source) {
}
}

private static void copyStream(final InputStream inputStream, final OutputStream outputStream, final byte[] buffer) throws IOException {
int n;
while ((n = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, n);
}
}

protected <T> T getProperty(final String name, final Function<String, T> mapper, final Supplier<T> defaultValue) {
final var value = LogManager.getLogManager().getProperty(name);
if (value == null) {
Expand Down

0 comments on commit 7335046

Please sign in to comment.