Skip to content

Commit

Permalink
Enabled displaying results after 14th page for local search queries.
Browse files Browse the repository at this point in the history 
Fixes issue #90 for local queries only: Stealth mode, Portal mode or
Intranet mode. 
For P2p mode, the issue would probably be difficult to solve with
reasonable performance. This is still to dig.

Also switched some InterreputedException catch log messages to warn
level as this is normal behavior when shutting down a peer.

Fixed yacysearch buttons navbar behavior to deal correctly with total
results count or offset over 1000. Also improved the buttons navbar to
be able to navigate over 10th page for local queries.
  • Loading branch information
@luccioman
luccioman committed Dec 20, 2016
1 parent a3886c6 commit c25e48e
Show file tree
Hide file tree
Showing 7 changed files with 155 additions and 60 deletions.
108 changes: 70 additions & 38 deletions htroot/js/yacysearch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,72 @@ function fadeOutBar() {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;backgroundColor:transparent;");
}

function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase) {
if (totalcount == 0) return;
if (offset >= 0) document.getElementById("offset").innerHTML = offset;
if (offset >= 0) document.getElementById("startRecord").setAttribute('value', offset - 1);
if (itemscount >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
/**
* @returns pagination buttons
*/
function renderPaginationButtons(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
var resnav = "<ul class=\"pagination\">";
var thispage = Math.floor(offset / itemsperpage);
var firstPage = thispage - (thispage % 10);
if (thispage == 0) {
resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
} else {
resnav += "<li><a id=\"prevpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
resnav += "\">&laquo;</a></li>";
}

var totalPagesNb = Math.floor(1 + ((totalcount - 1) / itemsperpage));
var numberofpages = Math.min(10, totalPagesNb - firstPage);
if (!numberofpages) numberofpages = 10;
for (i = firstPage; i < (firstPage + numberofpages); i++) {
if (i == thispage) {
resnav += "<li class=\"active\"><a href=\"#\">";
resnav += (i + 1);
resnav += "</a></li>";
} else {
resnav += "<li><a href=\"";
resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
resnav += "\">" + (i + 1) + "</a></li>";
}
}
if ((localQuery && thispage >= (totalPagesNb - 1)) || (!localQuery && thispage >= (numberofpages - 1))) {
resnav += "<li class=\"disabled\"><a href=\"#\">&raquo;</a></li>";
} else {
resnav += "<li><a id=\"nextpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
resnav += "\">&raquo;</a>";
}
resnav += "</ul>";
return resnav;
}

/**
* Parses a string representing an integer value
* @param strIntValue formatted string
* @returns the number value or undefined when the string is undefined, or NaN when the string is not a number
*/
function parseFormattedInt(strIntValue) {
var inValue;
if(strIntValue != null && strIntValue.replace != null) {
/* Remove thousands separator and try to parse as integer */
intValue = parseInt(strIntValue.replace(/[\.\,]/g,''))
}
return intValue;
}

function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
var totalcountIntValue = parseFormattedInt(totalcount);
var offsetIntValue = parseFormattedInt(offset);
var itemscountIntValue = parseFormattedInt(itemscount);
var itemsperpageIntValue = parseFormattedInt(itemsperpage);

if (totalcountIntValue == 0) {
return;
}
if (offsetIntValue >= 0) document.getElementById("offset").innerHTML = offset;
if (offsetIntValue >= 0) document.getElementById("startRecord").setAttribute('value', offsetIntValue - 1);
if (itemscountIntValue >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
document.getElementById("totalcount").firstChild.nodeValue = totalcount;
if (document.getElementById("localResourceSize") != null) document.getElementById("localResourceSize").firstChild.nodeValue = localResourceSize;
if (document.getElementById("remoteResourceSize") != null) document.getElementById("remoteResourceSize").firstChild.nodeValue = remoteResourceSize;
Expand All @@ -52,7 +113,7 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
// compose page navigation

if (document.getElementById("progressbar").getAttribute('class') != "progress-bar progress-bar-success") {
var percent = 100 * (itemscount - offset + 1) / itemsperpage;
var percent = 100 * (itemscountIntValue - offsetIntValue + 1) / itemsperpageIntValue;
if (percent == 100) {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;");
document.getElementById("progressbar").setAttribute('class',"progress-bar progress-bar-success");
Expand All @@ -62,38 +123,9 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
}
var resnavElement = document.getElementById("resNav");
if (resnavElement != null) {
var resnav = "<ul class=\"pagination\">";
thispage = Math.floor(offset / itemsperpage);
if (thispage == 0) {
resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
} else {
resnav += "<li><a id=\"prevpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
resnav += "\">&laquo;</a></li>";
}

numberofpages = Math.floor(Math.min(9, 1 + ((totalcount.replace(/\./g,'') - 1) / itemsperpage)));
if (!numberofpages) numberofpages = 9;
for (i = 0; i < numberofpages; i++) {
if (i == thispage) {
resnav += "<li class=\"active\"><a href=\"#\">";
resnav += (i + 1);
resnav += "</a></li>";
} else {
resnav += "<li><a href=\"";
resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
resnav += "\">" + (i + 1) + "</a></li>";
}
}
if (thispage >= numberofpages) {
resnav += "<li><a href=\"#\">&raquo;</a></li>";
} else {
resnav += "<li><a id=\"nextpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
resnav += "\">&raquo;</a>";
}
resnav += "</ul>";
resnavElement.innerHTML = resnav;
resnavElement.innerHTML = renderPaginationButtons(offsetIntValue, itemscountIntValue, itemsperpageIntValue,
totalcountIntValue, parseFormattedInt(localResourceSize), parseFormattedInt(remoteResourceSize), parseFormattedInt(remoteIndexCount),
parseFormattedInt(remotePeerCount), navurlbase, localQuery);
}
}

Expand Down
3 changes: 2 additions & 1 deletion htroot/yacysearch.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ <h4 class="linktitle">
self.xmlHttpReq.onreadystatechange = function() {
if (self.xmlHttpReq.readyState == 4) {
var rsp = eval("(" + self.xmlHttpReq.responseText + ")");
statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase);

statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase, #[localQuery]#);
}
};
self.xmlHttpReq.send(null);
Expand Down
2 changes: 2 additions & 0 deletions htroot/yacysearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ public static serverObjects respond(
prop.put("indexof", "off");
prop.put("constraint", "");
prop.put("depth", "0");
prop.put("localQuery", "0");
prop.put(
"search.verify",
(post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));
Expand Down Expand Up @@ -849,6 +850,7 @@ public static serverObjects respond(
}

prop.put("depth", "0");
prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");

}
prop.put("focus", focus ? 1 : 0); // focus search field
Expand Down
2 changes: 2 additions & 0 deletions htroot/yacysearchitem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("references", "0");
prop.put("rssreferences", "0");
prop.put("dynamic", "0");
prop.put("localQuery", "0");

// find search event
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
Expand All @@ -119,6 +120,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");

long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
Expand Down
8 changes: 7 additions & 1 deletion source/net/yacy/search/query/QueryParams.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,13 @@ public QueryParams(
this.contentdom = contentdom;
this.timezoneOffset = timezoneOffset;
this.itemsPerPage = Math.min((specialRights) ? 10000 : 1000, itemsPerPage);
this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
if(domType == Searchdom.LOCAL) {
/* No offset restriction on local index only requests, as only itemsPerPage will be loaded */
this.offset = Math.max(0, offset);
} else {
/* Offset has to be limited on requests mixing local and remote results, because all results before offset are loaded */
this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
}
try {
this.urlMaskString = urlMask;
// solr doesn't like slashes, backslashes or doublepoints; remove them // urlmask = ".*\\." + ft + "(\\?.*)?";
Expand Down
69 changes: 54 additions & 15 deletions source/net/yacy/search/query/SearchEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ public final class SearchEvent {
private byte[] IAmaxcounthash, IAneardhthash;
public Thread rwiProcess;
public Thread localsolrsearch;
/** Offset of the next local Solr index request
* Example : last local request with offset=10 and itemsPerPage=20, sets this attribute to 30. */
private int localsolroffset;
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
public final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates
Expand Down Expand Up @@ -323,9 +325,9 @@ protected SearchEvent(

// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), this.query.offset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset = this.query.itemsPerPage;
this.localsolroffset = this.query.offset + this.query.itemsPerPage;

// start a local RWI search concurrently
this.rwiProcess = null;
Expand Down Expand Up @@ -1576,31 +1578,68 @@ public URIMetadataNode oneResult(final int item, final long timeout) {
// (happens if a search pages is accessed a second time)
final long finishTime = timeout == Long.MAX_VALUE ? Long.MAX_VALUE : System.currentTimeMillis() + timeout;
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "started, item = " + item + ", available = " + this.getResultCount(), 0, 0), false);

// wait until a local solr is finished, we must do that to be able to check if we need more
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join(100);} catch (final InterruptedException e) {}}
if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
// load remaining solr results now
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {
try {
this.localsolrsearch.join(100);
} catch (final InterruptedException e) {
log.warn("Wait for local solr search was interrupted.");
}
}
if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {
try {
this.localsolrsearch.join();
} catch (final InterruptedException e) {
log.warn("Wait for local solr search was interrupted.");
}
}
if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
/* Request mixing remote and local Solr results : load remaining local solr results now.
* For local only search, a new SearchEvent should be created, starting directly at the requested offset,
* thus allowing to handle last pages of large resultsets
*/
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,
this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image),
this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
}

// now pull results as long as needed and as long as possible
if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) try {Thread.sleep(100);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
while ( this.resultList.sizeAvailable() <= item &&
if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) {
try {
Thread.sleep(100);
} catch (final InterruptedException e) {
log.warn("Remote search results wait was interrupted.");
}
}

final int resultListIndex;
if (this.remote) {
resultListIndex = item;
} else {
resultListIndex = item - (this.localsolroffset - this.query.itemsPerPage);
}
while ( this.resultList.sizeAvailable() <= resultListIndex &&
(this.rwiQueueSize() > 0 || this.nodeStack.sizeQueue() > 0 ||
(!this.feedingIsFinished() && System.currentTimeMillis() < finishTime))) {
if (!drainStacksToResult()) try {Thread.sleep(10);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
if (!drainStacksToResult()) {
try {
Thread.sleep(10);
} catch (final InterruptedException e) {
log.warn("Search results wait was interrupted.");
}
}
}

// check if we have a success
if (this.resultList.sizeAvailable() > item) {
if (this.resultList.sizeAvailable() > resultListIndex) {
// we have the wanted result already in the result array .. return that
final URIMetadataNode re = this.resultList.element(item).getElement();
final URIMetadataNode re = this.resultList.element(resultListIndex).getElement();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "fetched, item = " + item + ", available = " + this.getResultCount() + ": " + re.urlstring(), 0, 0), false);

/*
Expand Down
23 changes: 18 additions & 5 deletions source/net/yacy/search/query/SearchEventCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,24 @@ public static SearchEvent getEvent(
event = null;
} else {
if (event != null) {
//re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime();
// replace the current result offset
event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
if(query.isLocal()) {
/* Searching the local index only : we do not reuse the cached event each time the page size or offset changes.
* This allow to request last result pages of large result sets (larger than SearchEvent.max_results_node)
* without the need to retrieve all the beginning pages */
if(event.query.offset != query.offset || event.query.itemsPerPage != query.itemsPerPage) {
synchronized (lastEvents) {
lastEvents.remove(id);
}
cacheDelete++;
event = null;
}
} else {
//re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime();
// replace the current result offset
event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
}
}
}
if (event == null) {
Expand Down

0 comments on commit c25e48e

Please sign in to comment.