Permalink
Browse files

Fixed count of filtered results from local solr.

Was inadequately modified in my previous related commits (making next
pages buttons unavailable in Search portal mode), as
SearchEvent.local_solr_available did not count the total filtered
results but only the ones within the currently fetched result page(s).
  • Loading branch information...
luccioman committed Aug 31, 2017
1 parent 30c2f50 commit a28428047a048a9eb1e3800b90374dbe56a91668
@@ -726,7 +726,7 @@ public static serverObjects respond(
+ "remote_rwi_available(" + theSearch.remote_rwi_available.get() + "), "
+ "remote_rwi_stored(" + theSearch.remote_rwi_stored.get() + "), "
+ "remote_rwi_peerCount(" + theSearch.remote_rwi_peerCount.get() + "), "
+ "local_solr_available(" + theSearch.local_solr_available.get() + "), "
+ "local_solr_evicted(" + theSearch.local_solr_evicted.get() + "), "
+ "local_solr_stored(" + theSearch.local_solr_stored.get() + "), "
+ "remote_solr_available(" + theSearch.remote_solr_available.get() + "), "
+ "remote_solr_stored(" + theSearch.remote_solr_stored.get() + "), "
@@ -118,7 +118,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("statistics_itemscount", Formatter.number(Math.min((item < 0) ? theSearch.query.neededResults() : item + 1, theSearch.getResultCount())));
prop.put("statistics_itemsperpage", Formatter.number(theSearch.query.itemsPerPage));
prop.put("statistics_totalcount", Formatter.number(theSearch.getResultCount(), true));
prop.put("statistics_localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_available.get(), true));
prop.put("statistics_localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_stored.get() - theSearch.local_solr_evicted.get(), true));
prop.put("statistics_remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("statistics_remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("statistics_navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
@@ -40,7 +40,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("itemsperpage", theSearch.query.itemsPerPage);
prop.put("totalcount", Formatter.number(theSearch.getResultCount(), true));
prop.put("localResourceSize", Formatter.number(theSearch.local_rwi_stored.get() + theSearch.local_solr_stored.get(), true));
prop.put("localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_available.get(), true));
prop.put("localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_stored.get() - theSearch.local_solr_evicted.get(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.remote_rwi_stored.get() + theSearch.remote_solr_stored.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
@@ -241,10 +241,10 @@
/** the number of peers which contributed to the remote search result */
public final AtomicInteger remote_rwi_peerCount;
/** the number of hits generated/ranked by the local search in solr, after filtering */
public final AtomicInteger local_solr_available;
/** The number of results evicted from local solr results after filtering up to the current query offset */
public final AtomicInteger local_solr_evicted;
/** the number of existing hits by the local search in solr, before any supplementary filtering */
/** the total number of existing hits by the local search in solr, before any supplementary filtering */
public final AtomicInteger local_solr_stored;
/** the number of hits imported from remote peers (rwi/solr mixed), after filtering */
@@ -265,7 +265,7 @@
public int getResultCount() {
return Math.max(
this.local_rwi_available.get() + this.remote_rwi_available.get() +
this.remote_solr_available.get() + this.local_solr_available.get(),
this.remote_solr_available.get() + this.local_solr_stored.get() - this.local_solr_evicted.get(),
imageViewed.size() + sizeSpare()
);
}
@@ -355,7 +355,7 @@ protected SearchEvent(
this.remoteStoredDocMaxSize = -1;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available = new AtomicInteger(0);
this.local_solr_evicted = new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_rwi_stored = new AtomicInteger(0);
this.remote_rwi_available = new AtomicInteger(0); // the number of result contributions from all the remote dht peers
@@ -974,13 +974,19 @@ public void addNodes(
// check url mask
if (!iEntry.matches(this.query.urlMaskPattern)) {
if (log.isFine()) log.fine("dropped Node: url mask does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
// doublecheck for urls
if (this.urlhashes.has(iEntry.hash())) {
if (log.isFine()) log.fine("dropped Node: double check");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
@@ -993,6 +999,9 @@ public void addNodes(
Bitfield flags = iEntry.flags();
if (!this.testFlags(flags)) {
if (log.isFine()) log.fine("dropped Node: flag test");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
@@ -1003,13 +1012,19 @@ public void addNodes(
(this.query.contentdom == ContentDomain.IMAGE && !(flags.get(Tokenizer.flag_cat_hasimage))) ||
(this.query.contentdom == ContentDomain.APP && !(flags.get(Tokenizer.flag_cat_hasapp))))) {
if (log.isFine()) log.fine("dropped Node: content domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
// filter out media links in text search, if wanted
String ext = MultiProtocolURL.getFileExtension(iEntry.url().getFileName());
if (this.query.contentdom == ContentDomain.TEXT && Classification.isImageExtension(ext) && this.excludeintext_image) {
if (log.isFine()) log.fine("dropped Node: file name domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
@@ -1018,33 +1033,48 @@ public void addNodes(
if ( this.query.modifier.sitehash == null ) {
if (this.query.siteexcludes != null && this.query.siteexcludes.contains(hosthash)) {
if (log.isFine()) log.fine("dropped Node: siteexclude");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
} else {
// filter out all domains that do not match with the site constraint
if (iEntry.url().getHost().indexOf(this.query.modifier.sitehost) < 0) {
if (log.isFine()) log.fine("dropped Node: sitehost");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
if (this.query.modifier.language != null) {
if (!this.query.modifier.language.equals(iEntry.language())) {
if (log.isFine()) log.fine("dropped Node: language");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
if (this.query.modifier.author != null) {
if (!this.query.modifier.author.equals(iEntry.dc_creator())) {
if (log.isFine()) log.fine ("dropped Node: author");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
if (this.query.modifier.keyword != null) {
if (iEntry.dc_subject().indexOf(this.query.modifier.keyword) < 0) {
if (log.isFine()) log.fine ("dropped Node: keyword");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
@@ -1069,7 +1099,9 @@ public void addNodes(
}
}
// increase counter for statistics
if (local) this.local_solr_available.incrementAndGet(); else this.remote_solr_available.incrementAndGet();
if (!local) {
this.remote_solr_available.incrementAndGet();
}
}
} catch (final SpaceExceededException e ) {
}

0 comments on commit a284280

Please sign in to comment.