Skip to content

Commit

Permalink
small enhancements in search result computation speed
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6039 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Jun 9, 2009
1 parent e0b3984 commit 2c5554c
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 41 deletions.
10 changes: 9 additions & 1 deletion source/de/anomic/kelondro/text/metadataPrototype/URLMetadataRow.java
View file
Expand Up @@ -121,6 +121,7 @@ public class URLMetadataRow implements Metadata {
private final String snippet;
private WordReference word; // this is only used if the url is transported via remote search requests
private final long ranking; // during generation of a search result this value is set
private Components comp;

public URLMetadataRow(
final yacyURL url,
Expand Down Expand Up @@ -176,6 +177,7 @@ public URLMetadataRow(
this.snippet = null;
this.word = null;
this.ranking = 0;
this.comp = null;
}

private void encodeDate(final int col, final Date d) {
Expand Down Expand Up @@ -206,6 +208,7 @@ public URLMetadataRow(final Row.Entry entry, final WordReference searchedWord, f
this.snippet = null;
this.word = searchedWord;
this.ranking = ranking;
this.comp = null;
}

public URLMetadataRow(final Properties prop) {
Expand Down Expand Up @@ -270,6 +273,7 @@ public URLMetadataRow(final Properties prop) {
this.word = new WordReferenceRow(Base64Order.enhancedCoder.decodeString(prop.getProperty("wi", ""), "de.anomic.index.indexURLEntry.indexURLEntry()"));
}
this.ranking = 0;
this.comp = null;
}

public static URLMetadataRow importEntry(final String propStr) {
Expand Down Expand Up @@ -345,14 +349,18 @@ public long ranking() {
}

public Components metadata() {
// avoid double computation of metadata elements
if (this.comp != null) return this.comp;
// parse elements from comp string;
final ArrayList<String> cl = FileUtils.strings(this.entry.getCol("comp", null), "UTF-8");
return new Components(
this.comp = new Components(
(cl.size() > 0) ? (cl.get(0)).trim() : "",
hash(),
(cl.size() > 1) ? (cl.get(1)).trim() : "",
(cl.size() > 2) ? (cl.get(2)).trim() : "",
(cl.size() > 3) ? (cl.get(3)).trim() : "",
(cl.size() > 4) ? (cl.get(4)).trim() : "");
return this.comp;
}

public Date moddate() {
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/plasma/plasmaSearchEvent.java
View file
Expand Up @@ -541,7 +541,7 @@ public void run() {
if (page == null) {
if (!anyRemoteSearchAlive()) break; // we cannot expect more results
// if we did not get another entry, sleep some time and try again
try {Thread.sleep(100);} catch (final InterruptedException e1) {}
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (result.exists(page.hash().hashCode())) continue;
Expand Down
79 changes: 40 additions & 39 deletions source/de/anomic/plasma/plasmaSearchRankingProcess.java
View file
Expand Up @@ -32,6 +32,7 @@
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -65,7 +66,7 @@ public final class plasmaSearchRankingProcess {

private final SortStack<WordReferenceVars> stack;
private final HashMap<String, SortStack<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
private final HashMap<String, String> handover; // key = urlhash, value = urlstring; used for double-check of urls that had been handed over to search process
private final HashSet<String> handover; // key = urlhash; used for double-check of urls that had been handed over to search process
private final plasmaSearchQuery query;
private final int maxentries;
private int remote_peerCount, remote_indexCount, remote_resourceSize, local_resourceSize;
Expand All @@ -91,7 +92,7 @@ public plasmaSearchRankingProcess(
this.localSearchInclusion = null;
this.stack = new SortStack<WordReferenceVars>(maxentries);
this.doubleDomCache = new HashMap<String, SortStack<WordReferenceVars>>();
this.handover = new HashMap<String, String>();
this.handover = new HashSet<String>();
this.order = (query == null) ? null : new ReferenceOrder(query.ranking, query.targetlang);
this.query = query;
this.maxentries = maxentries;
Expand Down Expand Up @@ -317,47 +318,47 @@ private SortStack<WordReferenceVars>.stackElement bestRWI(final boolean skipDoub
public URLMetadataRow bestURL(final boolean skipDoubleDom) {
// returns from the current RWI list the best URL entry and removes this entry from the list
while ((stack.size() > 0) || (size() > 0)) {
if (((stack.size() == 0) && (size() == 0))) break;
final SortStack<WordReferenceVars>.stackElement obrwi = bestRWI(skipDoubleDom);
if (obrwi == null) continue; // *** ? this happened and the thread was suspended silently. cause?
final URLMetadataRow u = indexSegment.urlMetadata().load(obrwi.element.metadataHash(), obrwi.element, obrwi.weight.longValue());
if (u != null) {
final URLMetadataRow.Components metadata = u.metadata();
if (((stack.size() == 0) && (size() == 0))) break;
final SortStack<WordReferenceVars>.stackElement obrwi = bestRWI(skipDoubleDom);
if (obrwi == null) continue; // *** ? this happened and the thread was suspended silently. cause?
final URLMetadataRow u = indexSegment.urlMetadata().load(obrwi.element.metadataHash(), obrwi.element, obrwi.weight.longValue());
if (u != null) {
final URLMetadataRow.Components metadata = u.metadata();

// evaluate information of metadata for navigation
// author navigation:
String author = metadata.dc_creator();
if (author != null && author.length() > 0) {
// add author to the author navigator
String authorhash = new String(Word.word2hash(author));
System.out.println("*** DEBUG authorhash = " + authorhash + ", query.authorhash = " + this.query.authorhash + ", author = " + author);

// check if we already are filtering for authors
if (this.query.authorhash != null && !this.query.authorhash.equals(authorhash)) {
continue;
}

// add author to the author navigator
AuthorInfo in = this.authorNavigator.get(authorhash);
if (in == null) {
this.authorNavigator.put(authorhash, new AuthorInfo(author));
} else {
in.inc();
this.authorNavigator.put(authorhash, in);
}
} else if (this.query.authorhash != null) {
continue;
}
// evaluate information of metadata for navigation
// author navigation:
String author = metadata.dc_creator();
if (author != null && author.length() > 0) {
// add author to the author navigator
String authorhash = new String(Word.word2hash(author));
//System.out.println("*** DEBUG authorhash = " + authorhash + ", query.authorhash = " + this.query.authorhash + ", author = " + author);

// get the url
if (metadata.url() != null) {
String urlstring = metadata.url().toNormalform(true, true);
if (urlstring == null || !urlstring.matches(query.urlMask)) continue;
this.handover.put(u.hash(), metadata.url().toNormalform(true, false)); // remember that we handed over this url
return u;
// check if we already are filtering for authors
if (this.query.authorhash != null && !this.query.authorhash.equals(authorhash)) {
continue;
}

// add author to the author navigator
AuthorInfo in = this.authorNavigator.get(authorhash);
if (in == null) {
this.authorNavigator.put(authorhash, new AuthorInfo(author));
} else {
in.inc();
this.authorNavigator.put(authorhash, in);
}
} else if (this.query.authorhash != null) {
continue;
}
misses.add(obrwi.element.metadataHash());

// get the url
if (metadata.url() != null) {
String urlstring = metadata.url().toNormalform(true, true);
if (urlstring == null || !urlstring.matches(query.urlMask)) continue;
this.handover.add(u.hash()); // remember that we handed over this url
return u;
}
}
misses.add(obrwi.element.metadataHash());
}
return null;
}
Expand Down

0 comments on commit 2c5554c

Please sign in to comment.