Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Automatic purge of daemon logs #213

Merged
merged 3 commits into from
Nov 13, 2020
Merged

Automatic purge of daemon logs #213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6ea2ff9
Use human readable durations instead of milliseconds
gnodet Nov 12, 2020
3f1ecac
Provide an automatic purge of daemon logs, fixes #196
gnodet Nov 12, 2020
85d2189
Rewrite and simplify the TimeUtils class, add a bit of doc on the Env…
gnodet Nov 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/src/main/java/org/mvndaemon/mvnd/client/DaemonClientConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public void dispatch(Message message) throws DaemonException.ConnectException {
}

public List<Message> receive() throws ConnectException, StaleAddressException {
int maxKeepAliveMs = parameters.keepAliveMs() * parameters.maxLostKeepAlive();
long maxKeepAliveMs = parameters.keepAlive().toMillis() * parameters.maxLostKeepAlive();
while (true) {
try {
final Message m = queue.poll(maxKeepAliveMs, TimeUnit.MILLISECONDS);
Expand Down
20 changes: 16 additions & 4 deletions client/src/main/java/org/mvndaemon/mvnd/client/DaemonParameters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -41,6 +42,7 @@
import org.mvndaemon.mvnd.common.BuildProperties;
import org.mvndaemon.mvnd.common.Environment;
import org.mvndaemon.mvnd.common.Os;
import org.mvndaemon.mvnd.common.TimeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -49,6 +51,8 @@
*/
public class DaemonParameters {

public static final String LOG_EXTENSION = ".log";

private static final Logger LOG = LoggerFactory.getLogger(DaemonParameters.class);
private static final String EXT_CLASS_PATH = "maven.ext.class.path";
private static final String EXTENSIONS_FILENAME = ".mvn/extensions.xml";
Expand Down Expand Up @@ -182,11 +186,11 @@ public Path registry() {
}

public Path daemonLog(String daemon) {
return daemonStorage().resolve("daemon-" + daemon + ".log");
return daemonStorage().resolve("daemon-" + daemon + LOG_EXTENSION);
}

public Path daemonOutLog(String daemon) {
return daemonStorage().resolve("daemon-" + daemon + ".out.log");
return daemonStorage().resolve("daemon-" + daemon + ".out" + LOG_EXTENSION);
}

public Path multiModuleProjectDirectory() {
Expand Down Expand Up @@ -268,8 +272,8 @@ public DaemonParameters cd(Path newUserDir) {
.put(Environment.USER_DIR, newUserDir));
}

public int keepAliveMs() {
return property(Environment.DAEMON_KEEP_ALIVE_MS).orFail().asInt();
public Duration keepAlive() {
return property(Environment.DAEMON_KEEP_ALIVE).orFail().asDuration();
}

public int maxLostKeepAlive() {
Expand All @@ -284,6 +288,10 @@ public int rollingWindowSize() {
return property(Environment.MVND_ROLLING_WINDOW_SIZE).orFail().asInt();
}

public Duration purgeLogPeriod() {
return property(Environment.MVND_LOG_PURGE_PERIOD).orFail().asDuration();
}

public static String findDefaultMultimoduleProjectDirectory(Path pwd) {
Path dir = pwd;
do {
Expand Down Expand Up @@ -579,5 +587,9 @@ public int asInt(IntUnaryOperator function) {
return function.applyAsInt(asInt());
}

public Duration asDuration() {
return TimeUtils.toDuration(get());
}

}
}
131 changes: 117 additions & 14 deletions client/src/main/java/org/mvndaemon/mvnd/client/DefaultClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,22 @@
*/
package org.mvndaemon.mvnd.client;

import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.fusesource.jansi.Ansi;
import org.jline.utils.AttributedString;
import org.jline.utils.AttributedStyle;
Expand All @@ -33,11 +42,14 @@
import org.mvndaemon.mvnd.common.Message;
import org.mvndaemon.mvnd.common.Message.BuildException;
import org.mvndaemon.mvnd.common.OsUtils;
import org.mvndaemon.mvnd.common.TimeUtils;
import org.mvndaemon.mvnd.common.logging.ClientOutput;
import org.mvndaemon.mvnd.common.logging.TerminalOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.mvndaemon.mvnd.client.DaemonParameters.LOG_EXTENSION;

public class DefaultClient implements Client {

private static final Logger LOGGER = LoggerFactory.getLogger(DefaultClient.class);
Expand Down Expand Up @@ -183,6 +195,12 @@ public ExecutionResult execute(ClientOutput output, List<String> argv) {
}
return new DefaultResult(argv, null);
}
boolean purge = args.remove("--purge");
if (purge) {
String result = purgeLogs();
output.accept(Message.display(result != null ? result : "Nothing to purge"));
return new DefaultResult(argv, null);
}

if (args.stream().noneMatch(arg -> arg.startsWith("-T") || arg.equals("--threads"))) {
args.add("--threads");
Expand Down Expand Up @@ -216,27 +234,112 @@ public ExecutionResult execute(ClientOutput output, List<String> argv) {

output.accept(Message.buildStatus("Build request sent"));

while (true) {
final List<Message> messages = daemon.receive();
output.accept(messages);
for (Message m : messages) {
switch (m.getType()) {
case Message.CANCEL_BUILD:
return new DefaultResult(argv,
new InterruptedException("The build was canceled"));
case Message.BUILD_EXCEPTION:
final BuildException e = (BuildException) m;
return new DefaultResult(argv,
new Exception(e.getClassName() + ": " + e.getMessage() + "\n" + e.getStackTrace()));
case Message.BUILD_STOPPED:
return new DefaultResult(argv, null);
// We've sent the request, so it gives us a bit of time to purge the logs
AtomicReference<String> purgeMessage = new AtomicReference<>();
Thread purgeLog = new Thread(() -> {
purgeMessage.set(purgeLogs());
}, "Log purge");
purgeLog.setDaemon(true);
purgeLog.start();

try {
while (true) {
final List<Message> messages = daemon.receive();
output.accept(messages);
for (Message m : messages) {
switch (m.getType()) {
case Message.CANCEL_BUILD:
return new DefaultResult(argv,
new InterruptedException("The build was canceled"));
case Message.BUILD_EXCEPTION:
final BuildException e = (BuildException) m;
return new DefaultResult(argv,
new Exception(e.getClassName() + ": " + e.getMessage() + "\n" + e.getStackTrace()));
case Message.BUILD_STOPPED:
return new DefaultResult(argv, null);
}
}
}
} finally {
String msg = purgeMessage.get();
if (msg != null) {
output.accept(Message.display(msg));
}
}
}
}
}

private String purgeLogs() {
Path storage = parameters.daemonStorage();
Duration purgeLogPeriod = parameters.purgeLogPeriod();
if (!Files.isDirectory(storage) || !TimeUtils.isPositive(purgeLogPeriod)) {
return null;
}
String date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(Instant.now());
Path log = storage.resolve("purge-" + date + ".log");
List<Path> deleted = new ArrayList<>();
List<Throwable> exceptions = new ArrayList<>();
FileTime limit = FileTime.from(Instant.now().minus(purgeLogPeriod));
try {
Files.list(storage)
.filter(p -> p.getFileName().toString().endsWith(LOG_EXTENSION))
.filter(p -> !log.equals(p))
.filter(p -> {
try {
FileTime lmt = Files.getLastModifiedTime(p);
return lmt.compareTo(limit) < 0;
} catch (IOException e) {
exceptions.add(e);
return false;
}
})
.forEach(p -> {
try {
Files.delete(p);
deleted.add(p);
} catch (IOException e) {
exceptions.add(e);
}
});
} catch (Exception e) {
exceptions.add(e);
}
if (exceptions.isEmpty() && deleted.isEmpty()) {
return null;
}
String logMessage;
try (PrintWriter w = new PrintWriter(Files.newBufferedWriter(log,
StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE))) {
w.printf("Purge executed at %s%n", Instant.now().toString());
if (deleted.isEmpty()) {
w.printf("No files deleted.%n");
} else {
w.printf("Deleted files:%n");
for (Path p : deleted) {
w.printf(" %s%n", p.toString());
}
}
if (!exceptions.isEmpty()) {
w.printf("%d exception(s) occurred during the purge", exceptions.size());
for (Throwable t : exceptions) {
t.printStackTrace(w);
}
}
char[] buf = new char[80];
Arrays.fill(buf, '=');
w.printf("%s%n", new String(buf));
logMessage = "log available in " + log.toString();
} catch (IOException e) {
logMessage = "an exception occurred when writing log to " + log.toString() + ": " + e.toString();
}
if (exceptions.isEmpty()) {
return String.format("Purged %d log files (%s)", deleted.size(), logMessage);
} else {
return String.format("Purged %d log files with %d exceptions (%s)", deleted.size(), exceptions.size(), logMessage);
}
}

private static class DefaultResult implements ExecutionResult {

private final Exception exception;
Expand Down
32 changes: 21 additions & 11 deletions common/src/main/java/org/mvndaemon/mvnd/common/Environment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2019 the original author or authors.
* Copyright 2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,14 +17,20 @@

import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Collection;
import java.util.Locale;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

/**
* Collects system properties and environment variables used by mvnd client or server.
*
* Duration properties such as {@link #DAEMON_IDLE_TIMEOUT}, {@link #DAEMON_KEEP_ALIVE},
* {@link #DAEMON_EXPIRATION_CHECK_DELAY} or {@link #MVND_LOG_PURGE_PERIOD} are expressed
* in a human readable format such as {@code 2h30m}, {@code 600ms} or {@code 10 seconds}.
* The available units are <i>d/day/days</i>, <i>h/hour/hours</i>, <i>m/min/minute/minutes</i>,
* <i>s/sec/second/seconds</i> and <i>ms/millis/msec/milliseconds</i>.
*/
public enum Environment {
//
Expand Down Expand Up @@ -68,14 +74,18 @@ public String asCommandLineProperty(String value) {
* The number of log lines to display for each Maven module that is built in parallel.
*/
MVND_ROLLING_WINDOW_SIZE("mvnd.rollingWindowSize", null, "0", false),
/**
* The automatic log purge period
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A short description of the supported syntax and a couple of duration examples would be nice here and on other Duration options.

We should slowly start adding @since so that users see how old is which option.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense, I'll add one.
I think we can start using @since when we'll switch to a non 0.0.x version, which kinda assumes, very early code and non stabilised api.

*/
MVND_LOG_PURGE_PERIOD("mvnd.logPurgePeriod", null, "1w", false),
/**
* The path to the daemon registry
*/
DAEMON_REGISTRY("daemon.registry", null, null, false),
MVND_NO_DAEMON("mvnd.noDaemon", "MVND_NO_DAEMON", "false", true),
DAEMON_DEBUG("daemon.debug", null, false, true),
DAEMON_IDLE_TIMEOUT_MS("daemon.idleTimeoutMs", null, TimeUnit.HOURS.toMillis(3), true),
DAEMON_KEEP_ALIVE_MS("daemon.keepAliveMs", null, TimeUnit.SECONDS.toMillis(1), true),
DAEMON_IDLE_TIMEOUT("daemon.idleTimeout", null, "3 hours", true),
DAEMON_KEEP_ALIVE("daemon.keepAlive", null, "1 sec", true),
DAEMON_MAX_LOST_KEEP_ALIVE("daemon.maxLostKeepAlive", null, 3, false),
/**
* The minimum number of threads to use when constructing the default {@code -T} parameter for the daemon.
Expand All @@ -102,7 +112,7 @@ public String asCommandLineProperty(String value) {
/**
* The maven builder name to use. Ignored if the user passes
*
* @{@code -b} or @{@code --builder} on the command line
* {@code -b} or {@code --builder} on the command line
*/
MVND_BUILDER("mvnd.builder", null, "smart", false) {
@Override
Expand Down Expand Up @@ -146,17 +156,13 @@ public String asCommandLineProperty(String value) {
/**
* Interval to check if the daemon should expire
*/
DAEMON_EXPIRATION_CHECK_DELAY_MS("daemon.expirationCheckDelayMs", null, TimeUnit.SECONDS.toMillis(10), true),
DAEMON_EXPIRATION_CHECK_DELAY("daemon.expirationCheckDelay", null, "10 seconds", true),
/**
* Period after which idle daemons will shut down
*/
DAEMON_DUPLICATE_DAEMON_GRACE_PERIOD_MS("daemon.duplicateDaemonGracePeriodMs", null, TimeUnit.SECONDS.toMillis(10), true),
DAEMON_DUPLICATE_DAEMON_GRACE_PERIOD("daemon.duplicateDaemonGracePeriod", null, "10 seconds", true),
;

public static final int DEFAULT_IDLE_TIMEOUT = (int) TimeUnit.HOURS.toMillis(3);

public static final int DEFAULT_KEEP_ALIVE = (int) TimeUnit.SECONDS.toMillis(1);

static Properties properties = System.getProperties();

public static void setProperties(Properties properties) {
Expand Down Expand Up @@ -219,6 +225,10 @@ public Path asPath() {
return Paths.get(result);
}

public Duration asDuration() {
return TimeUtils.toDuration(asString());
}

public String asCommandLineProperty(String value) {
return "-D" + property + "=" + value;
}
Expand Down