Skip to content

Commit

Permalink
- reduced default search time
Browse files Browse the repository at this point in the history 
- catched case when web structure cannot be painted because of too less data
- better logging when balance fails


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3892 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Jun 14, 2007
1 parent 506a5a4 commit 71fd972
Show file tree
Hide file tree
Showing 8 changed files with 45 additions and 35 deletions.
2 changes: 1 addition & 1 deletion build.properties
View file
Expand Up @@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4

# Release Configuration
releaseVersion=0.524
releaseVersion=0.525
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}
Expand Down
41 changes: 25 additions & 16 deletions htroot/WebStructurePicture_p.java
View file
Expand Up @@ -81,28 +81,37 @@ public static ymageMatrix respond(httpHeader header, serverObjects post, serverS
// find domain with most references
host = sb.webStructure.hostWithMaxReferences();
}
// find start hash
String hash = null;
try {
hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6);
} catch (MalformedURLException e) {e.printStackTrace();}
assert (sb.webStructure.references(hash) != null);

// recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print();

ymageMatrix graphPicture = graph.draw(width, height, 40, 40, 16, 16);

ymageMatrix graphPicture;
if (host == null) {
// probably no information available
graphPicture = new ymageMatrix(width, height, ymageGraph.color_back);
graphPicture.setMode(ymageMatrix.MODE_SUB);
ymageToolPrint.print(graphPicture, width / 2, height / 2, 0, "NO WEB STRUCTURE DATA AVAILABLE.", 0);
ymageToolPrint.print(graphPicture, width / 2, height / 2 + 16, 0, "START A WEB CRAWL TO OBTAIN STRUCTURE DATA.", 0);
} else {
// find start hash
String hash = null;
try {
hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6);
} catch (MalformedURLException e) {e.printStackTrace();}
assert (sb.webStructure.references(hash) != null);

// recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print();

graphPicture = graph.draw(width, height, 40, 40, 16, 16);
}
// print headline
graphPicture.setColor(ymageMatrix.SUBTRACTIVE_BLACK);
graphPicture.setColor(ymageGraph.color_text);
graphPicture.setMode(ymageMatrix.MODE_SUB);
ymageToolPrint.print(graphPicture, 2, 8, 0, "YACY WEB-STRUCTURE ANALYSIS", -1);
ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1);
if (host != null) ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1);
ymageToolPrint.print(graphPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1);

return graphPicture;

}

private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, long timeout, double x, double y, int nextlayer, int maxlayer) {
Expand Down
9 changes: 4 additions & 5 deletions htroot/index.html
View file
Expand Up @@ -33,7 +33,7 @@ <h2 class="yacy">
#(searchoptions)#<!-- default values are hard-coded
<input type="hidden" name="count" value="10" />
<input type="hidden" name="resource" value="global" />
<input type="hidden" name="time" value="6" />
<input type="hidden" name="time" value="4" />
<input type="hidden" name="urlmaskfilter" value=".*" />
<input type="hidden" name="prefermaskfilter" value="" />
<input type="hidden" name="indexof" value="off" />-->
Expand All @@ -50,7 +50,6 @@ <h2 class="yacy">
<option #(count-10)#::selected="selected"#(/count-10)#>10</option>
<option #(count-50)#::selected="selected"#(/count-50)#>50</option>
<option #(count-100)#::selected="selected"#(/count-100)#>100</option>
<option #(count-1000)#::selected="selected"#(/count-1000)#>1000</option>
</select>
</td>
</tr>
Expand All @@ -75,11 +74,11 @@ <h2 class="yacy">
<td>
<select id="time" name="time">
<option #(time-1)#::selected="selected"#(/time-1)#>1</option>
<option #(time-3)#::selected="selected"#(/time-3)#>3</option>
<option #(time-2)#::selected="selected"#(/time-2)#>2</option>
<option #(time-4)#::selected="selected"#(/time-4)#>4</option>
<option #(time-6)#::selected="selected"#(/time-6)#>6</option>
<option #(time-8)#::selected="selected"#(/time-8)#>8</option>
<option #(time-10)#::selected="selected"#(/time-10)#>10</option>
<option #(time-30)#::selected="selected"#(/time-30)#>30</option>
<option #(time-60)#::selected="selected"#(/time-60)#>60</option>
</select>
</td>
</tr>
Expand Down
11 changes: 5 additions & 6 deletions htroot/index.java
View file
Expand Up @@ -66,8 +66,8 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
final int searchoptions = (post == null) ? 0 : post.getInt("searchoptions", 0);
final String former = (post == null) ? "" : post.get("former", "");
final int count = (post == null) ? 10 : post.getInt("count", 10);
final int time = (post == null) ? 10 : post.getInt("time", 6);
final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10));
final int time = Math.min(60, (post == null) ? 4 : post.getInt("time", 4));
final String urlmaskfilter = (post == null) ? ".*" : post.get("urlmaskfilter", ".*");
final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", "");
final String constraint = (post == null) ? plasmaSearchQuery.catchall_constraint.exportB64() : post.get("constraint", "______");
Expand Down Expand Up @@ -127,17 +127,16 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
prop.put("searchoptions_count-10", (count == 10) ? 1 : 0);
prop.put("searchoptions_count-50", (count == 50) ? 1 : 0);
prop.put("searchoptions_count-100", (count == 100) ? 1 : 0);
prop.put("searchoptions_count-1000", (count == 1000) ? 1 : 0);
prop.put("searchoptions_resource-global", ((global) ? 1 : 0));
prop.put("searchoptions_resource-global-disabled", (global || clustersearch) ? 0 : 1);
prop.put("searchoptions_resource-global-disabled_reason", (indexReceiveGranted) ? 0 : (indexDistributeGranted) ? 1 : 2);
prop.put("searchoptions_resource-local", ((global) ? 0 : 1));
prop.put("searchoptions_time-1", (time == 1) ? 1 : 0);
prop.put("searchoptions_time-3", (time == 3) ? 1 : 0);
prop.put("searchoptions_time-2", (time == 2) ? 1 : 0);
prop.put("searchoptions_time-4", (time == 4) ? 1 : 0);
prop.put("searchoptions_time-6", (time == 6) ? 1 : 0);
prop.put("searchoptions_time-8", (time == 8) ? 1 : 0);
prop.put("searchoptions_time-10", (time == 10) ? 1 : 0);
prop.put("searchoptions_time-30", (time == 30) ? 1 : 0);
prop.put("searchoptions_time-60", (time == 60) ? 1 : 0);
prop.put("searchoptions_urlmaskoptions", 0);
prop.put("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter);
prop.put("searchoptions_prefermaskoptions", 0);
Expand Down
8 changes: 5 additions & 3 deletions source/de/anomic/plasma/plasmaCrawlLoader.java
View file
Expand Up @@ -277,7 +277,9 @@ public void loadAsync(
String referer,
String initiator,
int depth,
plasmaCrawlProfile.entry profile
plasmaCrawlProfile.entry profile,
int timeout,
boolean keepInMemory
) {

if (!this.crawlwerPool.isClosed) {
Expand All @@ -293,8 +295,8 @@ public void loadAsync(
profile, // crawling profile
crawlingPriority, // crawling priority
false, // only download documents whose mimetypes are enabled for the crawler
-1, // use default crawler timeout
false // resource should not be kept in memory
timeout, // -1 = use default crawler timeout
keepInMemory // kept in memory ?
);

// adding the message to the queue
Expand Down
3 changes: 2 additions & 1 deletion source/de/anomic/plasma/plasmaCrawlNURL.java
View file
Expand Up @@ -191,8 +191,9 @@ private plasmaCrawlEntry pop(plasmaCrawlBalancer balancer) throws IOException {
entry = balancer.pop(minimumDelta, maximumDomAge);
if (entry == null) {
if (s > balancer.size()) continue;
int aftersize = balancer.size();
balancer.clear(); // the balancer is broken and cannot shrink
throw new IOException("entry is null, balancer cannot shrink; reset of balancer");
throw new IOException("entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
}
return entry;
}
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/plasma/plasmaSwitchboard.java
View file
Expand Up @@ -2639,7 +2639,7 @@ private void processLocalCrawling(plasmaCrawlEntry urlEntry, plasmaCrawlProfile.
} catch (IOException e) {
refererURL = null;
}
cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile);
cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile, -1, false);
log.logInfo(stats + ": enqueued for load " + urlEntry.url() + " [" + urlEntry.urlhash() + "]");
return;
}
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/ymage/ymageGraph.java
View file
Expand Up @@ -124,11 +124,11 @@ public void print() {
}
}

private static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE;
public static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE;
public static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK;
private static final long color_dot = 0x6633AA;
private static final long color_line = 0x333333;
private static final long color_lineend = 0x555555;
private static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK;

public ymageMatrix draw(int width, int height, int leftborder, int rightborder, int topborder, int bottomborder) {
ymageMatrix image = new ymageMatrix(width, height, color_back);
Expand Down

0 comments on commit 71fd972

Please sign in to comment.