Permalink
Browse files

Added a stop condition to the Recrawl busy thread

  • Loading branch information...
luccioman committed Jan 9, 2018
1 parent 421728d commit b2af25b14f6c10609d4fb6cc6dffa5b34eb4ea17
Showing with 47 additions and 24 deletions.
  1. +2 −2 htroot/IndexReIndexMonitor_p.java
  2. +45 −22 source/net/yacy/crawler/RecrawlBusyThread.java
@@ -122,7 +122,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
inclerrdoc = post.getBoolean("includefailedurls");
}
if (recrawlbt == null) {
if (recrawlbt == null || recrawlbt.shutdownInProgress()) {
prop.put("recrawljobrunning_simulationResult", 0);
if (post.containsKey("recrawlnow") && sb.index.fulltext().connectedLocalSolr()) {
sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null,
@@ -165,7 +165,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
// just post status of recrawlThread
if (recrawlbt != null && !recrawlbt.shutdownInProgress()) { // provide status
prop.put("recrawljobrunning", 1);
prop.put("recrawljobrunning_docCount", ((RecrawlBusyThread) recrawlbt).urlsfound);
prop.put("recrawljobrunning_docCount", ((RecrawlBusyThread) recrawlbt).getUrlsToRecrawl());
prop.put("recrawljobrunning_recrawlquerytext", ((RecrawlBusyThread) recrawlbt).getQuery());
prop.put("recrawljobrunning_includefailedurls", ((RecrawlBusyThread) recrawlbt).getIncludeFailed());
} else {
@@ -45,7 +45,7 @@
* and feeds the found urls to the crawler to recrawl the documents.
* This is intended to keep the index up-to-date
* Currently the doucments are selected by expired fresh_date_dt field
* an added to the crawler in smaller chunks (see chunksize) as long as no other crawl is runnin.
* an added to the crawler in smaller chunks (see chunksize) as long as no other crawl is running.
*/
public class RecrawlBusyThread extends AbstractBusyThread {
@@ -66,12 +66,18 @@
private int chunkstart = 0;
private final int chunksize;
final Switchboard sb;
private final Switchboard sb;
/** buffer of urls to recrawl */
private final Set<DigestURL> urlstack;
public long urlsfound = 0;
/** The total number of candidate URLs found for recrawl */
private long urlsToRecrawl = 0;
private String solrSortBy;
/** Set to true when more URLs are still to be processed */
private boolean moreToRecrawl = true;
/**
* @param xsb
@@ -92,10 +98,10 @@ public RecrawlBusyThread(final Switchboard xsb, final String query, final boolea
this.sb = xsb;
this.currentQuery = query;
this.includefailed = includeFailed;
urlstack = new HashSet<DigestURL>();
this.urlstack = new HashSet<DigestURL>();
// workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues
// org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues.
solrSortBy = null; // CollectionSchema.load_date_dt.getSolrFieldName() + " asc";
this.solrSortBy = null; // CollectionSchema.load_date_dt.getSolrFieldName() + " asc";
this.chunksize = sb.getConfigInt(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 200);
}
@@ -191,10 +197,21 @@ public boolean job() {
return false;
}
boolean didSomething = false;
if (this.urlstack.isEmpty()) {
return processSingleQuery();
if(!this.moreToRecrawl) {
/* We do not remove the thread from the Switchboard worker threads using serverSwitch.terminateThread(String,boolean),
* because we want to be able to provide a report after its termination */
terminate(false);
} else {
this.moreToRecrawl = processSingleQuery();
/* Even if no more URLs are to recrawl, the job has done something by searching the Solr index */
didSomething = true;
}
} else {
didSomething = feedToCrawler();
}
return feedToCrawler();
return didSomething;
}
@@ -208,21 +225,22 @@ private boolean processSingleQuery() {
}
SolrDocumentList docList = null;
SolrConnector solrConnector = sb.index.fulltext().getDefaultConnector();
if (!solrConnector.isClosed()) {
try {
// query all or only httpstatus=200 depending on includefailed flag
docList = solrConnector.getDocumentListByQuery(RecrawlBusyThread.buildSelectionQuery(this.currentQuery, this.includefailed),
this.solrSortBy, this.chunkstart, this.chunksize, CollectionSchema.sku.getSolrFieldName());
this.urlsfound = docList.getNumFound();
} catch (Throwable e) {
this.urlsfound = 0;
}
} else {
this.urlsfound =0;
if (solrConnector.isClosed()) {
this.urlsToRecrawl = 0;
return false;
}
try {
// query all or only httpstatus=200 depending on includefailed flag
docList = solrConnector.getDocumentListByQuery(RecrawlBusyThread.buildSelectionQuery(this.currentQuery, this.includefailed),
this.solrSortBy, this.chunkstart, this.chunksize, CollectionSchema.sku.getSolrFieldName());
this.urlsToRecrawl = docList.getNumFound();
} catch (final Throwable e) {
this.urlsToRecrawl = 0;
}
if (docList != null) {
for (SolrDocument doc : docList) {
for (final SolrDocument doc : docList) {
try {
this.urlstack.add(new DigestURL((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())));
} catch (MalformedURLException ex) {
@@ -237,10 +255,8 @@ private boolean processSingleQuery() {
this.chunkstart = this.chunkstart + this.chunksize;
}
if (this.urlsfound <= this.chunkstart) {
this.chunkstart = 0;
if (docList == null || docList.size() < this.chunksize) {
return false;
// TODO: add a stop condition
}
return true;
}
@@ -249,6 +265,13 @@ private boolean processSingleQuery() {
public int getJobCount() {
return this.urlstack.size();
}
/**
* @return The total number of candidate URLs found for recrawl
*/
public long getUrlsToRecrawl() {
return this.urlsToRecrawl;
}
@Override
public void freemem() {

0 comments on commit b2af25b

Please sign in to comment.