Permalink
Browse files

Allow to stop currently running warc import (stop button)

  • Loading branch information...
reger24 committed Aug 20, 2017
1 parent 6cec2cd commit 51a4e03c938364628d8813aa7d66e618cddf0a85
Showing with 26 additions and 12 deletions.
  1. +13 −10 htroot/IndexImportWarc_p.html
  2. +3 −0 htroot/IndexImportWarc_p.java
  3. +10 −2 source/net/yacy/document/importer/WarcImporter.java
@@ -37,16 +37,19 @@ <h2>Web Archive File Import</h2>
<br />
::
<form><fieldset><legend>Import Process</legend>
<dl>
<dt>Thread:</dt><dd>#[thread]#</dd>
<dt>Warc File:</dt><dd>#[warcfile]#</dd>
<dt>Processed:</dt><dd>#[count]# Entries</dd>
<dt>Speed:</dt><dd>#[speed]# pages per second</dd>
<dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd>
<dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd>
</dl>
</fieldset></form>
<form>
<fieldset><legend>Import Process</legend>
<dl>
<dt>Thread:</dt><dd>#[thread]#</dd>
<dt>Warc File:</dt><dd>#[warcfile]#</dd>
<dt>Processed:</dt><dd>#[count]# Entries</dd>
<dt>Speed:</dt><dd>#[speed]# pages per second</dd>
<dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd>
<dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd>
</dl>
</fieldset>
<input name="abort" type="submit" class="btn btn-danger" value="Stop"/>
</form>
#(/import)#
#%env/templates/footer.template%#
@@ -46,6 +46,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("import_runningMinutes", (WarcImporter.job.runningTime() / 60) % 60);
prop.put("import_remainingHours", (WarcImporter.job.remainingTime() / 60) / 60);
prop.put("import_remainingMinutes", (WarcImporter.job.remainingTime() / 60) % 60);
if (post != null && post.containsKey("abort")) {
WarcImporter.job.quit();
}
} else {
prop.put("import", 0);
if (post != null) {
@@ -58,7 +58,7 @@
*/
public class WarcImporter extends Thread implements Importer {
static public Importer job; // static object to assure only one importer is running (if started from a servlet, this object is used to store the thread)
static public WarcImporter job; // static object to assure only one importer is running (if started from a servlet, this object is used to store the thread)
private final InputStream source; // current input warc archive
private String name; // file name of input source
@@ -67,6 +67,7 @@
private long startTime; // (for statistic)
private final long sourceSize; // length of the input source (for statistic)
private long consumed; // bytes consumed from input source (for statistic)
private boolean abort = false; // flag to signal stop of import
public WarcImporter(InputStream f) {
source = f;
@@ -107,7 +108,7 @@ public void indexWarcRecords(InputStream f) throws IOException {
WarcReader localwarcReader = WarcReaderFactory.getReader(f);
WarcRecord wrec = localwarcReader.getNextRecord();
while (wrec != null) {
while (wrec != null && !abort) {
HeaderLine hl = wrec.getHeader(WarcConstants.FN_WARC_TYPE);
if (hl != null && hl.value.equals(WarcConstants.RT_RESPONSE)) { // filter responses
@@ -185,6 +186,13 @@ public void run() {
ConcurrentLog.info("WarcImporter", ex.getMessage());
}
}
/**
* Set the flag to stop import
*/
public void quit() {
this.abort = true;
}
/**
* Filename of the input source

0 comments on commit 51a4e03

Please sign in to comment.