Skip to content

Commit

Permalink
Fixed count of filtered results from local solr.
Browse files Browse the repository at this point in the history 
Was inadequately modified in my previous related commits (making next
pages buttons unavailable in Search portal mode), as
SearchEvent.local_solr_available did not count the total filtered
results but only the ones within the currently fetched result page(s).
  • Loading branch information
@luccioman
luccioman committed Aug 31, 2017
1 parent 30c2f50 commit a284280
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 9 deletions.
2 changes: 1 addition & 1 deletion htroot/yacysearch.java
View file
Expand Up @@ -726,7 +726,7 @@ public static serverObjects respond(
+ "remote_rwi_available(" + theSearch.remote_rwi_available.get() + "), "
+ "remote_rwi_stored(" + theSearch.remote_rwi_stored.get() + "), "
+ "remote_rwi_peerCount(" + theSearch.remote_rwi_peerCount.get() + "), "
+ "local_solr_available(" + theSearch.local_solr_available.get() + "), "
+ "local_solr_evicted(" + theSearch.local_solr_evicted.get() + "), "
+ "local_solr_stored(" + theSearch.local_solr_stored.get() + "), "
+ "remote_solr_available(" + theSearch.remote_solr_available.get() + "), "
+ "remote_solr_stored(" + theSearch.remote_solr_stored.get() + "), "
Expand Down
2 changes: 1 addition & 1 deletion htroot/yacysearchitem.java
View file
Expand Up @@ -118,7 +118,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("statistics_itemscount", Formatter.number(Math.min((item < 0) ? theSearch.query.neededResults() : item + 1, theSearch.getResultCount())));
prop.put("statistics_itemsperpage", Formatter.number(theSearch.query.itemsPerPage));
prop.put("statistics_totalcount", Formatter.number(theSearch.getResultCount(), true));
prop.put("statistics_localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_available.get(), true));
prop.put("statistics_localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_stored.get() - theSearch.local_solr_evicted.get(), true));
prop.put("statistics_remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("statistics_remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("statistics_navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
Expand Down
2 changes: 1 addition & 1 deletion htroot/yacysearchlatestinfo.java
View file
Expand Up @@ -40,7 +40,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
prop.put("itemsperpage", theSearch.query.itemsPerPage);
prop.put("totalcount", Formatter.number(theSearch.getResultCount(), true));
prop.put("localResourceSize", Formatter.number(theSearch.local_rwi_stored.get() + theSearch.local_solr_stored.get(), true));
prop.put("localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_available.get(), true));
prop.put("localIndexCount", Formatter.number(theSearch.local_rwi_available.get() + theSearch.local_solr_stored.get() - theSearch.local_solr_evicted.get(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.remote_rwi_stored.get() + theSearch.remote_solr_stored.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
Expand Down
44 changes: 38 additions & 6 deletions source/net/yacy/search/query/SearchEvent.java
View file
Expand Up @@ -241,10 +241,10 @@ public final class SearchEvent {
/** the number of peers which contributed to the remote search result */
public final AtomicInteger remote_rwi_peerCount;

/** the number of hits generated/ranked by the local search in solr, after filtering */
public final AtomicInteger local_solr_available;
/** The number of results evicted from local solr results after filtering up to the current query offset */
public final AtomicInteger local_solr_evicted;

/** the number of existing hits by the local search in solr, before any supplementary filtering */
/** the total number of existing hits by the local search in solr, before any supplementary filtering */
public final AtomicInteger local_solr_stored;

/** the number of hits imported from remote peers (rwi/solr mixed), after filtering */
Expand All @@ -265,7 +265,7 @@ public final class SearchEvent {
public int getResultCount() {
return Math.max(
this.local_rwi_available.get() + this.remote_rwi_available.get() +
this.remote_solr_available.get() + this.local_solr_available.get(),
this.remote_solr_available.get() + this.local_solr_stored.get() - this.local_solr_evicted.get(),
imageViewed.size() + sizeSpare()
);
}
Expand Down Expand Up @@ -355,7 +355,7 @@ protected SearchEvent(
this.remoteStoredDocMaxSize = -1;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available = new AtomicInteger(0);
this.local_solr_evicted = new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_rwi_stored = new AtomicInteger(0);
this.remote_rwi_available = new AtomicInteger(0); // the number of result contributions from all the remote dht peers
Expand Down Expand Up @@ -974,13 +974,19 @@ public void addNodes(
// check url mask
if (!iEntry.matches(this.query.urlMaskPattern)) {
if (log.isFine()) log.fine("dropped Node: url mask does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}

// doublecheck for urls
if (this.urlhashes.has(iEntry.hash())) {
if (log.isFine()) log.fine("dropped Node: double check");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}

Expand All @@ -993,6 +999,9 @@ public void addNodes(
Bitfield flags = iEntry.flags();
if (!this.testFlags(flags)) {
if (log.isFine()) log.fine("dropped Node: flag test");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}

Expand All @@ -1003,13 +1012,19 @@ public void addNodes(
(this.query.contentdom == ContentDomain.IMAGE && !(flags.get(Tokenizer.flag_cat_hasimage))) ||
(this.query.contentdom == ContentDomain.APP && !(flags.get(Tokenizer.flag_cat_hasapp))))) {
if (log.isFine()) log.fine("dropped Node: content domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}

// filter out media links in text search, if wanted
String ext = MultiProtocolURL.getFileExtension(iEntry.url().getFileName());
if (this.query.contentdom == ContentDomain.TEXT && Classification.isImageExtension(ext) && this.excludeintext_image) {
if (log.isFine()) log.fine("dropped Node: file name domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}

Expand All @@ -1018,33 +1033,48 @@ public void addNodes(
if ( this.query.modifier.sitehash == null ) {
if (this.query.siteexcludes != null && this.query.siteexcludes.contains(hosthash)) {
if (log.isFine()) log.fine("dropped Node: siteexclude");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
} else {
// filter out all domains that do not match with the site constraint
if (iEntry.url().getHost().indexOf(this.query.modifier.sitehost) < 0) {
if (log.isFine()) log.fine("dropped Node: sitehost");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}

if (this.query.modifier.language != null) {
if (!this.query.modifier.language.equals(iEntry.language())) {
if (log.isFine()) log.fine("dropped Node: language");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}

if (this.query.modifier.author != null) {
if (!this.query.modifier.author.equals(iEntry.dc_creator())) {
if (log.isFine()) log.fine ("dropped Node: author");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}

if (this.query.modifier.keyword != null) {
if (iEntry.dc_subject().indexOf(this.query.modifier.keyword) < 0) {
if (log.isFine()) log.fine ("dropped Node: keyword");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
continue pollloop;
}
}
Expand All @@ -1069,7 +1099,9 @@ public void addNodes(
}
}
// increase counter for statistics
if (local) this.local_solr_available.incrementAndGet(); else this.remote_solr_available.incrementAndGet();
if (!local) {
this.remote_solr_available.incrementAndGet();
}
}
} catch (final SpaceExceededException e ) {
}
Expand Down

0 comments on commit a284280

Please sign in to comment.