Skip to content
Permalink
Browse files

Added a startup hook to umbra harvests to allow cleanup of umbra prio…

…r to harvesting.
  • Loading branch information
csrster committed May 29, 2019
1 parent 7ea1ec2 commit 13599b102fcc614e6dee7c575786f68e22af492d
@@ -0,0 +1,141 @@
package dk.netarkivet.harvester.heritrix3;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Arrays;
import java.util.Map;

import org.netarchivesuite.heritrix3wrapper.LaunchResultHandlerAbstract;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BlockingCommandLauncher {

/** The logger for this class. */
private static final Logger log = LoggerFactory.getLogger(BlockingCommandLauncher.class);

public ProcessBuilder processBuilder;

public Map<String, String> env;

public Process process;

public BlockingCommandLauncher(File basedir, String[] cmd) {
processBuilder = new ProcessBuilder(Arrays.asList(cmd));
processBuilder.directory(basedir);
env = processBuilder.environment();
}

public Integer start(LaunchResultHandlerAbstract resultHandler) throws IOException {

process = processBuilder.start();

BufferedReader outputReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
Thread outputSinkThread = new Thread(new OutputSinkThread(outputReader, resultHandler));
outputSinkThread.setDaemon(true);
outputSinkThread.start();

BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
Thread errorSinkThread = new Thread(new ErrorSinkThread(errorReader, resultHandler));
errorSinkThread.setDaemon(true);
errorSinkThread.start();

int exitValue = Integer.MIN_VALUE;
try {
exitValue = process.waitFor();
} catch (InterruptedException e) {
log.error("Interrupted error:", e);
}
if (resultHandler != null) {
resultHandler.exitValue(exitValue);
}
return exitValue;
}

protected class OutputSinkThread implements Runnable {
LaunchResultHandlerAbstract resultHandler;
BufferedReader reader;

protected OutputSinkThread(BufferedReader reader, LaunchResultHandlerAbstract resultHandler) {
this.reader = reader;
this.resultHandler = resultHandler;
}

@Override
public void run() {
try {
if (resultHandler == null) {
return;
}
String line;
try {
while ((line = reader.readLine()) != null) {
resultHandler.output(line);
}
} catch (IOException e) {
resultHandler.output(getStackTrace(e));
} finally {
resultHandler.closeOutput();
}
} finally {
try {
reader.close();
} catch (IOException e) {
log.warn("Exception on closing reader", e);
}
}
}
}

protected String getStackTrace(IOException e) {
String stackTrace = null;
try (StringWriter stringWriter = new StringWriter();
PrintWriter writer = new PrintWriter(stringWriter)) {
e.printStackTrace(writer);
writer.flush();
stackTrace = stringWriter.toString();
} catch (IOException ex) {
log.warn("Error from string writer...", ex);
}
return stackTrace;
}

protected class ErrorSinkThread implements Runnable {
LaunchResultHandlerAbstract resultHandler;
BufferedReader reader;
protected ErrorSinkThread(BufferedReader reader, LaunchResultHandlerAbstract resultHandler) {
this.reader = reader;
this.resultHandler = resultHandler;
}

@Override
public void run() {
try {
if (resultHandler == null) {
return;
}
String line;
try {
while ((line = reader.readLine()) != null) {
resultHandler.error(line);
}
} catch (IOException e) {
resultHandler.error(getStackTrace(e));
} finally {
resultHandler.closeError();
}
} finally {
try {
reader.close();
} catch (IOException e) {
log.warn("Exception on closing reader", e);
}
}
}
}

}
@@ -50,7 +50,10 @@
*/
public static String UMBRA_HOPS_SHOULD_PROCESS = "settings.harvester.harvesting.umbra.hopsShouldProcess";


/**
* Path to a script to be executed before heritrix is started for every umbra enabled job.
*/
public static String UMBRA_PRESTART_SCRIPT = "settings.harvester.harvesting.umbra.startupHook";

/*
* The static initialiser is called when the class is loaded. It will add default values for all settings defined in

0 comments on commit 13599b1

Please sign in to comment.
You can’t perform that action at this time.