Permalink
Browse files

Added retention policy.

  • Loading branch information...
1 parent 8d222ab commit 7e538fb80103d35161386a28a0dbc77d161fc3af @rbpark committed Jun 7, 2012
@@ -40,6 +40,7 @@
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -156,7 +157,26 @@ private Job loadJob(JobDescriptor desc,
s.remove(dep);
return s;
}
+
+ /**
+ * Dangerous hack. Should be called on the outside.
+ * @param executions
+ */
+ public void cleanupLogs(List<JobExecution> executions) throws IOException{
+
+ Iterator<JobExecution> iter = executions.iterator();
+ while (iter.hasNext()) {
+ JobExecution exec = iter.next();
+ File file = exec.getLogDir();
+ if (file.exists()) {
+ logger.info("Cleaning log " + file.getPath());
+ FileUtils.deleteDirectory(file);
+ }
+ }
+ }
+
+
public List<JobExecution> loadRecentJobExecutions(int count) throws IOException {
// load job executions for all jobs
File logDir = new File(_logDir);
@@ -222,12 +242,15 @@ private Job loadJob(JobDescriptor desc,
}
String logFile = jobName + File.separator + execDir.getName() + File.separator
+ jobName + "." + execDir.getName() + ".log";
+
execs.add(new JobExecution(jobName,
start == null ? dirDate : start,
end,
succeeded,
false,
- logFile));
+ logFile,
+ execDir
+ ));
}
}
return execs;
@@ -16,6 +16,8 @@
package azkaban.jobs;
+import java.io.File;
+
import org.joda.time.DateTime;
import org.joda.time.Duration;
@@ -33,6 +35,7 @@
private boolean succeeded;
private String log;
private final boolean ignoreDependencies;
+ private File logDir = null;
public JobExecution(String id, DateTime start, DateTime end, boolean succeeded, boolean ignoreDependecies, String log) {
super();
@@ -44,6 +47,18 @@ public JobExecution(String id, DateTime start, DateTime end, boolean succeeded,
this.ignoreDependencies = ignoreDependecies;
}
+ public JobExecution(String id, DateTime start, DateTime end, boolean succeeded, boolean ignoreDependecies, String log, File logDir) {
+ super();
+ this.id = id;
+ this.startTime = start;
+ this.endTime = end;
+ this.succeeded = succeeded;
+ this.log = log;
+ this.ignoreDependencies = ignoreDependecies;
+ this.setLogDir(logDir);
+ }
+
+
public JobExecution(String jobName, DateTime start, boolean ignoreDependecies) {
this(jobName, start, null, false, ignoreDependecies, null);
}
@@ -97,4 +112,12 @@ public String getLog() {
public boolean isDependencyIgnored() {
return ignoreDependencies;
}
+
+ public File getLogDir() {
+ return logDir;
+ }
+
+ public void setLogDir(File logDir) {
+ this.logDir = logDir;
+ }
}
@@ -78,10 +78,19 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se
page.add("tab", "logs");
}
+ int logRetention = jdesc.getProps().getInt("log.retention", -1);
+
// Reduce
List<JobExecution> execs = jobManager.loadJobExecutions(jobId);
- if (execs.size() > 500) {
- execs = execs.subList(0, 500);
+ if (logRetention > 0 && execs.size() > logRetention + 50) {
+ synchronized(jdesc) {
+ if (execs.size() > logRetention + 50) {
+ logger.info("Retention cleaning");
+ List<JobExecution> clearList = execs.subList(logRetention, execs.size());
+ jobManager.cleanupLogs(clearList);
+ execs = execs.subList(0, logRetention);
+ }
+ }
}
int successes = 0;

0 comments on commit 7e538fb

Please sign in to comment.