Skip to content

Commit

Permalink
smaller bug fixes for search behavior; should produce less unnecessar…
Browse files Browse the repository at this point in the history 
…y removals and an exact number of results as shown in counter

should also be a little bit faster

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8057 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Nov 18, 2011
1 parent a3eebfd commit 804e488
Show file tree
Hide file tree
Showing 9 changed files with 33 additions and 41 deletions.
24 changes: 12 additions & 12 deletions htroot/ConfigPortal.java
View file
@@ -1,4 +1,4 @@
// ConfigPortal.java
// ConfigPortal.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
Expand All @@ -10,7 +10,7 @@
//$LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
Expand Down Expand Up @@ -38,14 +38,14 @@ public class ConfigPortal {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;

if (post != null) {
// AUTHENTICATE
if (!header.containsKey(RequestHeader.AUTHORIZATION)) {
prop.putHTML("AUTHENTICATE","log-in");
return prop;
}

if (post.containsKey("popup")) {
final String popup = post.get("popup", "status");
if ("front".equals(popup)) {
Expand All @@ -62,7 +62,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
// store this call as api call
sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);

sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
Expand All @@ -80,7 +80,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
sb.setConfig("search.verify", post.get("search.verify", "ifexist"));
sb.setConfig("search.verify.delete", post.getBoolean("search.verify.delete", false));
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
// construct navigation String
String nav = "";
if (post.getBoolean("search.navigation.hosts", false)) nav += "hosts,";
Expand Down Expand Up @@ -109,8 +109,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig("search.result.show.parser", true);
sb.setConfig("search.result.show.pictures", true);
sb.setConfig("search.verify", "iffresh");
sb.setConfig("search.verify.delete", "true");
}
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
}
}

prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, ""));
Expand All @@ -121,7 +121,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0);
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);

prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
Expand All @@ -138,7 +138,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0);
prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0);
prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0);
prop.put("search.verify.delete", sb.getConfigBool("search.verify.delete", true) ? 1 : 0);
prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0);

final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
prop.put("popupFront", 0);
Expand All @@ -156,14 +156,14 @@ public static serverObjects respond(final RequestHeader header, final serverObje
}

prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10));

final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
prop.put("selected_blank", "_blank".equals(target) ? 1 : 0);
prop.put("selected_self", "_self".equals(target) ? 1 : 0);
prop.put("selected_parent", "_parent".equals(target) ? 1 : 0);
prop.put("selected_top", "_top".equals(target) ? 1 : 0);
prop.put("selected_searchresult", "searchresult".equals(target) ? 1 : 0);

String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress();
if (myaddress == null) {
myaddress = "localhost:" + sb.getConfig("port", "8090");
Expand Down
2 changes: 1 addition & 1 deletion htroot/yacysearch.java
View file
Expand Up @@ -597,7 +597,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
indexSegment,
ranking,
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);

Expand Down
5 changes: 1 addition & 4 deletions source/net/yacy/cora/date/GenericFormatter.java
View file
Expand Up @@ -133,10 +133,7 @@ public Date parse(final String timeString, final String UTCOffset) {
if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); }
try {
return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset));
} catch (final java.text.ParseException e) {
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
return new Date();
} catch (final java.lang.NumberFormatException e) {
} catch (final Throwable e) {
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
return new Date();
}
Expand Down
11 changes: 2 additions & 9 deletions source/net/yacy/document/parser/images/genericImageParser.java
View file
Expand Up @@ -28,7 +28,6 @@

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -227,14 +226,8 @@ public static ImageInfo parseJavaImage(
try {
ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
image = ImageIO.read(sourceStream);
} catch (final EOFException e) {
Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
} catch (final IOException e) {
Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
} catch (final Throwable e) { // may appear in case of an OutOfMemoryError
Log.logException(e);
} catch (final Throwable e) {
//Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
}
if (image == null) throw new Parser.Failure("ImageIO returned NULL", location);
Expand Down
1 change: 1 addition & 0 deletions source/net/yacy/search/SwitchboardConstants.java
View file
Expand Up @@ -437,6 +437,7 @@ public final class SwitchboardConstants {

public static final String SEARCH_ITEMS = "search.items";
public static final String SEARCH_TARGET = "search.target";
public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";

/**
* system tray
Expand Down
13 changes: 7 additions & 6 deletions source/net/yacy/search/query/RWIProcess.java
View file
Expand Up @@ -35,6 +35,7 @@
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

import net.yacy.cora.document.ASCII;
Expand Down Expand Up @@ -79,7 +80,7 @@ public final class RWIProcess extends Thread {
private int remote_resourceSize, remote_indexCount, remote_peerCount;
private int local_indexCount;
private final WeakPriorityBlockingQueue<WordReferenceVars> stack;
private int feeders;
private final AtomicInteger feeders;
private final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
//private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process

Expand Down Expand Up @@ -123,7 +124,7 @@ public RWIProcess(final QueryParams query, final ReferenceOrder order, final int
this.protocolNavigator = new ConcurrentScoreMap<String>();
this.filetypeNavigator = new ConcurrentScoreMap<String>();
this.ref = new ConcurrentScoreMap<String>();
this.feeders = 1;
this.feeders = new AtomicInteger(1);
this.startTime = System.currentTimeMillis();
}

Expand Down Expand Up @@ -288,16 +289,16 @@ public void add(
* method to signal the incoming stack that one feeder has terminated
*/
public void oneFeederTerminated() {
this.feeders--;
assert this.feeders >= 0 : "feeders = " + this.feeders;
final int c = this.feeders.decrementAndGet();
assert c >= 0 : "feeders = " + c;
}

public void moreFeeders(final int countMoreFeeders) {
this.feeders += countMoreFeeders;
this.feeders.addAndGet(countMoreFeeders);
}

public boolean feedingIsFinished() {
return System.currentTimeMillis() - this.startTime > 50 && this.feeders == 0;
return this.feeders.get() == 0;
}

private boolean testFlags(final WordReference ientry) {
Expand Down
5 changes: 2 additions & 3 deletions source/net/yacy/search/query/SearchEvent.java
View file
Expand Up @@ -371,14 +371,13 @@ public HeuristicResult getHeuristic(final byte[] urlhash) {
}

public ResultEntry oneResult(final int item, final long timeout) {
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
(this.query.domType == QueryParams.Searchdom.CLUSTER)) {
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
// this is a search using remote search threads. Also the local
// search thread is started as background process
if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {
// in case that the local search takes longer than some other
// remote search requests, wait that the local process terminates first
try {this.localSearchThread.join();} catch (final InterruptedException e) {}
try {this.localSearchThread.join(300);} catch (final InterruptedException e) {}
}
}
return this.resultFetcher.oneResult(item, timeout);
Expand Down
3 changes: 2 additions & 1 deletion source/net/yacy/search/query/SearchEventCache.java
View file
Expand Up @@ -37,6 +37,7 @@
import net.yacy.peers.SeedDB;
import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment;
import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.snippet.ContentDomain;
Expand Down Expand Up @@ -178,7 +179,7 @@ public static SearchEvent getEvent(
Log.logInfo("SearchEventCache", "getEvent: " + SearchEventCache.lastEvents.size() + " in cache; " + countAliveThreads() + " alive");

// start a new event
final boolean delete = Switchboard.getSwitchboard() == null | Switchboard.getSwitchboard().getConfigBool("search.verify.delete", true);
final boolean delete = Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true);
event = new SearchEvent(query, peers, workTables, preselectedPeerHashes, generateAbstracts, loader, remote_maxcount, remote_maxtime, burstRobinsonPercent, burstMultiwordPercent, delete);
MemoryControl.request(100 * 1024 * 1024, false); // this may trigger a short memory status which causes a reducing of cache space of other threads
}
Expand Down
10 changes: 5 additions & 5 deletions source/net/yacy/search/query/SnippetProcess.java
View file
Expand Up @@ -147,7 +147,8 @@ public ResultEntry oneResult(final int item, final long timeout) {
final long waittimeout = System.currentTimeMillis() + 300;
while (
item == 0 &&
this.result.sizeAvailable() < this.query.neededResults() + this.query.itemsPerPage &&
(!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) &&
this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ &&
System.currentTimeMillis() < waittimeout &&
anyWorkerAlive()
) {
Expand Down Expand Up @@ -324,7 +325,7 @@ public void deployWorker(int deployCount, final int neededResults) {
}

private boolean anyWorkerAlive() {
if (this.workerThreads == null) return false;
if (this.workerThreads == null || this.workerThreads.length == 0) return false;
synchronized(this.workerThreads) {
for (final Worker workerThread : this.workerThreads) {
if ((workerThread != null) &&
Expand Down Expand Up @@ -377,7 +378,7 @@ public void run() {

// check if we have enough
if (SnippetProcess.this.result.sizeAvailable() >= this.neededResults) {
//Log.logWarning("ResultFetcher", ResultFetcher.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
//Log.logWarning("ResultFetcher", SnippetProcess.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
break;
}

Expand All @@ -391,7 +392,7 @@ public void run() {
page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis()));
//if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis());
if (page == null) {
//System.out.println("page == null");
//Log.logWarning("ResultFetcher", "page == null");
break; // no more available
}
if (SnippetProcess.this.query.filterfailurls && SnippetProcess.this.workTables.failURLsContains(page.hash())) continue;
Expand All @@ -405,7 +406,6 @@ public void run() {
if (sd != null) solrContent = this.solr.getScheme().solrGetText(sd);
}


loops++;
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
if (resultEntry == null) continue; // the entry had some problems, cannot be used
Expand Down

0 comments on commit 804e488

Please sign in to comment.