Skip to content

Commit

Permalink
first attempt to implement a secondary search
Browse files Browse the repository at this point in the history 
this is a set of search processes that shall enrich search results
with specialized requests to realize a combination of search results
from different peers.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2571 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Sep 13, 2006
1 parent 2a06ce5 commit cf9884e
Show file tree
Hide file tree
Showing 8 changed files with 280 additions and 159 deletions.
15 changes: 3 additions & 12 deletions htroot/yacy/search.java
View file
Expand Up @@ -47,14 +47,12 @@
// javac -classpath .:../../Classes search.java
// if the shell's current path is htroot/yacy

import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import de.anomic.http.httpHeader;
import de.anomic.index.indexContainer;
import de.anomic.index.indexEntryAttribute;
import de.anomic.index.indexURL;
import de.anomic.plasma.plasmaCrawlLURL;
import de.anomic.plasma.plasmaSearchEvent;
Expand Down Expand Up @@ -108,10 +106,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
}

// prepare search
final HashSet keyhashes = new HashSet(query.length() / indexEntryAttribute.wordHashLength);
for (int i = 0; i < (query.length() / indexEntryAttribute.wordHashLength); i++) {
keyhashes.add(query.substring(i * indexEntryAttribute.wordHashLength, (i + 1) * indexEntryAttribute.wordHashLength));
}
final Set keyhashes = plasmaSearchQuery.hashes2Set(query);
final long timestamp = System.currentTimeMillis();

plasmaSearchQuery squery = new plasmaSearchQuery(keyhashes, maxdist, prefer, count, duetime, filter);
Expand All @@ -129,11 +124,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve

// retrieve index containers from search request
plasmaSearchEvent theSearch = new plasmaSearchEvent(squery, rankingProfile, localTiming, remoteTiming, true, yacyCore.log, sb.wordIndex, sb.urlPool.loadedURL, sb.snippetCache);
Set urlselection = null;
if ((urls.length() > 0) && (urls.length() % 12 == 0)) {
for (int i = 0; i < (urls.length() / 12); i++) urlselection.add(urls.substring(i * 12, (i + 1 * 12)));
}
Map containers = theSearch.localSearchContainers(urlselection);
Map containers = theSearch.localSearchContainers(plasmaSearchQuery.hashes2Set(urls));

// set statistic details of search result and find best result index set
String maxcounthash = null, neardhthash = null;
Expand Down Expand Up @@ -168,7 +159,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
indexContainer localResults = theSearch.localSearchJoin(containers.values());
int joincount = localResults.size();
prop.put("joincount", Integer.toString(joincount));
plasmaSearchResult acc = theSearch.order(localResults);
plasmaSearchResult acc = theSearch.orderFinal(localResults);

// generate compressed index for maxcounthash
// this is not needed if the search is restricted to specific urls, because it is a re-search
Expand Down
166 changes: 79 additions & 87 deletions source/de/anomic/kelondro/kelondroMSetTools.java
View file
Expand Up @@ -48,8 +48,8 @@
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;

Expand Down Expand Up @@ -85,7 +85,7 @@ public static int log2a(int x) {
// - join by iterative tests (where we distinguish left-right and right-left tests)


public static TreeMap joinConstructive(Collection maps) {
public static TreeMap joinConstructive(Collection maps, boolean concatStrings) {
// this joins all TreeMap(s) contained in maps

// first order entities by their size
Expand Down Expand Up @@ -116,7 +116,7 @@ public static TreeMap joinConstructive(Collection maps) {
k = (Long) orderMap.firstKey(); // the next smallest...
mapA = joinResult;
mapB = (TreeMap) orderMap.remove(k);
joinResult = joinConstructiveByTestSetInMap(mapB, mapA.keySet());
joinResult = joinConstructiveByTest(mapA, mapB, concatStrings);
// free resources
mapA = null;
mapB = null;
Expand All @@ -127,72 +127,63 @@ public static TreeMap joinConstructive(Collection maps) {
return joinResult;
}

public static TreeMap joinConstructive(TreeMap map, TreeSet set) {
// comparators must be equal
if ((map == null) || (set == null)) return null;
if (map.comparator() != set.comparator()) return null;
if ((map.size() == 0) || (set.size() == 0)) return new TreeMap(map.comparator());

// decide which method to use
int high = ((map.size() > set.size()) ? map.size() : set.size());
int low = ((map.size() > set.size()) ? set.size() : map.size());
int stepsEnum = 10 * (high + low - 1);
int stepsTest = 12 * log2a(high) * low;
public static TreeMap joinConstructive(TreeMap map1, TreeMap map2, boolean concatStrings) {
// comparators must be equal
if ((map1 == null) || (map2 == null)) return null;
if (map1.comparator() != map2.comparator()) return null;
if ((map1.size() == 0) || (map2.size() == 0)) return new TreeMap(map1.comparator());

// start most efficient method
if (stepsEnum > stepsTest) {
if (map.size() > set.size()) return joinConstructiveByTestSetInMap(map, set);
return joinConstructiveByTestMapInSet(map, set);
}
return joinConstructiveByEnumeration(map, set);
}
// decide which method to use
int high = ((map1.size() > map2.size()) ? map1.size() : map2.size());
int low = ((map1.size() > map2.size()) ? map2.size() : map1.size());
int stepsEnum = 10 * (high + low - 1);
int stepsTest = 12 * log2a(high) * low;

private static TreeMap joinConstructiveByTestSetInMap(TreeMap map, Set set) {
Iterator si = set.iterator();
TreeMap result = new TreeMap(map.comparator());
Object o;
while (si.hasNext()) {
o = si.next();
if (map.containsKey(o)) result.put(o, map.get(o));
}
return result;
// start most efficient method
if (stepsEnum > stepsTest) {
if (map1.size() > map2.size()) return joinConstructiveByTest(map2, map1, concatStrings);
return joinConstructiveByTest(map1, map2, concatStrings);
}
return joinConstructiveByEnumeration(map1, map2, concatStrings);
}

private static TreeMap joinConstructiveByTestMapInSet(Map map, TreeSet set) {
Iterator mi = map.keySet().iterator();
TreeMap result = new TreeMap(set.comparator());
Object o;
while (mi.hasNext()) {
o = mi.next();
if (set.contains(o)) result.put(o, map.get(o));
}
return result;

private static TreeMap joinConstructiveByTest(TreeMap small, TreeMap large, boolean concatStrings) {
Iterator mi = small.entrySet().iterator();
TreeMap result = new TreeMap(large.comparator());
Map.Entry mentry1;
Object mobj2;
while (mi.hasNext()) {
mentry1 = (Map.Entry) mi.next();
mobj2 = large.get(mentry1.getKey());
if (mobj2 != null) result.put(mentry1.getKey(), (concatStrings) ? ((String) mentry1.getValue() + (String) mobj2) : mentry1.getValue());
}
return result;
}

private static TreeMap joinConstructiveByEnumeration(TreeMap map, TreeSet set) {
// implement pairvise enumeration
Comparator comp = map.comparator();
Iterator mi = map.keySet().iterator();
Iterator si = set.iterator();
TreeMap result = new TreeMap(map.comparator());
int c;
if ((mi.hasNext()) && (si.hasNext())) {
Object mobj = mi.next();
Object sobj = si.next();
while (true) {
c = compare(mobj, sobj, comp);
if (c < 0) {
if (mi.hasNext()) mobj = mi.next(); else break;
} else if (c > 0) {
if (si.hasNext()) sobj = si.next(); else break;
} else {
result.put(mobj, map.get(mobj));
if (mi.hasNext()) mobj = mi.next(); else break;
if (si.hasNext()) sobj = si.next(); else break;
}
}
}
return result;
private static TreeMap joinConstructiveByEnumeration(TreeMap map1, TreeMap map2, boolean concatStrings) {
// implement pairvise enumeration
Comparator comp = map1.comparator();
Iterator mi1 = map1.entrySet().iterator();
Iterator mi2 = map2.entrySet().iterator();
TreeMap result = new TreeMap(map1.comparator());
int c;
if ((mi1.hasNext()) && (mi2.hasNext())) {
Map.Entry mentry1 = (Map.Entry) mi1.next();
Map.Entry mentry2 = (Map.Entry) mi2.next();
while (true) {
c = compare(mentry1.getKey(), mentry2.getKey(), comp);
if (c < 0) {
if (mi1.hasNext()) mentry1 = (Map.Entry) mi1.next(); else break;
} else if (c > 0) {
if (mi2.hasNext()) mentry2 = (Map.Entry) mi2.next(); else break;
} else {
result.put(mentry1.getKey(), (concatStrings) ? ((String) mentry1.getValue() + (String) mentry2.getValue()) : mentry1.getValue());
if (mi1.hasNext()) mentry1 = (Map.Entry) mi1.next(); else break;
if (mi2.hasNext()) mentry2 = (Map.Entry) mi2.next(); else break;
}
}
}
return result;
}

// now the same for set-set
Expand Down Expand Up @@ -268,7 +259,7 @@ public static TreeMap excludeConstructive(TreeMap map, TreeSet set) {
// return excludeConstructiveByEnumeration(map, set);
}

private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, TreeSet set) {
private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, Set set) {
Iterator mi = map.keySet().iterator();
TreeMap result = new TreeMap(map.comparator());
Object o;
Expand All @@ -279,7 +270,8 @@ private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, TreeSet se
return result;
}

private static TreeMap excludeConstructiveByEnumeration(TreeMap map, TreeSet set) {
/*
private static TreeMap excludeConstructiveByEnumeration(TreeMap map, TreeSet set) {
// returns map without the elements in set
// enumerates objects
Comparator comp = map.comparator();
Expand Down Expand Up @@ -317,7 +309,7 @@ private static TreeMap excludeConstructiveByEnumeration(TreeMap map, TreeSet se
}
return result;
}

*/
public static void excludeDestructive(TreeMap map, TreeSet set) {
// comparators must be equal
if (map == null) return;
Expand Down Expand Up @@ -411,7 +403,7 @@ public static TreeSet loadList(File file, Comparator c) {

public static void main(String[] args) {
TreeMap m = new TreeMap();
TreeSet s = new TreeSet();
TreeMap s = new TreeMap();
m.put("a", "a");
m.put("x", "x");
m.put("f", "f");
Expand All @@ -422,26 +414,26 @@ public static void main(String[] args) {
m.put("k", "k");
m.put("y", "y");
m.put("z", "z");
s.add("a");
s.add("b");
s.add("c");
s.add("k");
s.add("l");
s.add("m");
s.add("n");
s.add("o");
s.add("p");
s.add("q");
s.add("r");
s.add("s");
s.add("t");
s.add("x");
s.put("a", "a");
s.put("b", "b");
s.put("c", "c");
s.put("k", "k");
s.put("l", "l");
s.put("m", "m");
s.put("n", "n");
s.put("o", "o");
s.put("p", "p");
s.put("q", "q");
s.put("r", "r");
s.put("s", "s");
s.put("t", "t");
s.put("x", "x");
System.out.println("Compare " + m.toString() + " with " + s.toString());
System.out.println("Join=" + joinConstructiveByEnumeration(m, s));
System.out.println("Join=" + joinConstructiveByTestMapInSet(m, s));
System.out.println("Join=" + joinConstructiveByTestSetInMap(m, s));
System.out.println("Join=" + joinConstructive(m, s));
System.out.println("Exclude=" + excludeConstructiveByEnumeration(m, s));
System.out.println("Join=" + joinConstructiveByEnumeration(m, s, true));
System.out.println("Join=" + joinConstructiveByTest(m, s, true));
System.out.println("Join=" + joinConstructiveByTest(m, s, true));
System.out.println("Join=" + joinConstructive(m, s, true));
System.out.println("Exclude=" + excludeConstructiveByTestMapInSet(m, s.keySet()));

/*
for (int low = 0; low < 10; low++)
Expand Down
24 changes: 18 additions & 6 deletions source/de/anomic/plasma/plasmaGrafics.java
View file
Expand Up @@ -67,8 +67,9 @@ public class plasmaGrafics {

public static ymagePainter getSearchEventPicture() {
if (plasmaSearchEvent.lastEvent == null) return null;
yacySearch[] searches = plasmaSearchEvent.lastEvent.getSearchThreads();
if (searches == null) return null; // this was a local search and there are no threads
yacySearch[] primarySearches = plasmaSearchEvent.lastEvent.getPrimarySearchThreads();
yacySearch[] secondarySearches = plasmaSearchEvent.lastEvent.getSecondarySearchThreads();
if (primarySearches == null) return null; // this was a local search and there are no threads

// get a copy of a recent network picture
ymagePainter eventPicture = getNetworkPicture(120000);
Expand All @@ -82,14 +83,25 @@ public static ymagePainter getSearchEventPicture() {
String hash;
int angle;

// draw in the search peers
for (int j = 0; j < searches.length; j++) {
eventPicture.setColor((searches[j].isAlive()) ? ymageMatrix.ADDITIVE_RED : ymageMatrix.ADDITIVE_GREEN);
hash = searches[j].target().hash;
// draw in the primary search peers
for (int j = 0; j < primarySearches.length; j++) {
eventPicture.setColor((primarySearches[j].isAlive()) ? ymageMatrix.ADDITIVE_RED : ymageMatrix.ADDITIVE_GREEN);
hash = primarySearches[j].target().hash;
angle = (int) ((long) 360 * (yacySeed.dhtPosition(hash) / (yacySeed.maxDHTDistance / (long) 10000)) / (long) 10000);
eventPicture.arcLine(cx, cy, cr - 20, cr, angle);
}

// draw in the secondary search peers
if (secondarySearches != null) {
for (int j = 0; j < secondarySearches.length; j++) {
eventPicture.setColor((secondarySearches[j].isAlive()) ? ymageMatrix.ADDITIVE_RED : ymageMatrix.ADDITIVE_GREEN);
hash = secondarySearches[j].target().hash;
angle = (int) ((long) 360 * (yacySeed.dhtPosition(hash) / (yacySeed.maxDHTDistance / (long) 10000)) / (long) 10000);
eventPicture.arcLine(cx, cy, cr - 10, cr, angle - 1);
eventPicture.arcLine(cx, cy, cr - 10, cr, angle + 1);
}
}

// draw in the search target
plasmaSearchQuery query = plasmaSearchEvent.lastEvent.getQuery();
Iterator i = query.queryHashes.iterator();
Expand Down

0 comments on commit cf9884e

Please sign in to comment.