Skip to content

Commit

Permalink
more + changed log for better understanding of outOfMemory bug and ot…
Browse files Browse the repository at this point in the history 
…hers

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@846 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Oct 4, 2005
1 parent e5f8163 commit 0c3a20d
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 22 deletions.
26 changes: 13 additions & 13 deletions htroot/Network.html
View file
Expand Up @@ -40,11 +40,6 @@ <h2>Manually contacting Peer</h2>
<tr class="TableHeader" valign="bottom">
<td class="small">send&nbsp;<b>M</b>essage/<br>show&nbsp;<b>P</b>rofile/<br>edit&nbsp;<b>W</b>iki<br>&nbsp;</td>
<td class="small"><b>Name</b><br>&nbsp;</td>
#(complete)#::
<td class="small"><b>Address</b><br>&nbsp;</td>
<td class="small"><b>Hash</b><br>&nbsp;</td>
<td class="small"><b>Age</b><br>&nbsp;</td>
#(/complete)#
<td class="small"><b>Info</b><br>&nbsp;</td>
<td class="small"><b>Release/<br>SVN</b><br>&nbsp;</td>
<td class="small"><b>Last<br>Seen</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=down">&gt;</a></td>
Expand All @@ -58,21 +53,20 @@ <h2>Manually contacting Peer</h2>
<td class="small"><b>Received<br>URLs</b><br>&nbsp;</td>
<td class="small"><b>PPM</b><br>&nbsp;</td>
<td class="small"><b>#Seeds</b><br>&nbsp;</td>
<td class="small"><b>#Connects<br>per hour</b><br>&nbsp;</td></tr>
#(complete)#::
<td class="small"><b>Address</b><br>&nbsp;</td>
<td class="small"><b>Hash</b><br>&nbsp;</td>
<td class="small"><b>Age</b><br>&nbsp;</td>
<td class="small"><b>con/h<br></b><br>&nbsp;</td>
#(/complete)#
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td class="small"><a href="MessageSend_p.html?hash=#[hash]#" class="small" title="Send message to peer #[fullname]#">m</a>&nbsp;&nbsp;
<a href="ViewProfile.html?hash=#[hash]#" class="small" title="View profile of peer #[fullname]#">p</a>&nbsp;&nbsp;
<a href="http://#[fullname]#.yacy/Wiki.html#[updatedWikiPage]#" class="small" title="Read and edit wiki on peer #[fullname]#">w</a>&nbsp;&nbsp;#[alert]#
</td>
<td class="small"><a href="http://www.#[fullname]#.yacy" class="small">#[shortname]#</a></td>
#(complete)#
::
<td class="small"><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=" class="small">http://#[ip]#:#[port]#</a></td>
<td class=small>#[hash]#</td>
<td class=small>#[age]#</td>
#(/complete)#

<td class="small">#(info)##(direct)#<img border="0" src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive"></img>::<img border="0" src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct"></img>#(/direct)#::#(direct)#<img border="0" src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive"></img>::<img border="0" src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct"></img>#(/direct)#::<a href="#[url]#">#(direct)#<img border="0" src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible"></img>#(/direct)#</a>#(/info)##(acceptcrawl)#<img border="0" src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no"></img>::<img border="0" src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes"></img>#(/acceptcrawl)##(dhtreceive)#<img border="0" src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no"></img>::<img border="0" src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes"></img>#(/dhtreceive)#</td>
<td class="small" align="right">#[version]#</td>
<td class="small" align="right">#[lastSeen]#</td>
Expand All @@ -86,7 +80,13 @@ <h2>Manually contacting Peer</h2>
<td class="small" align="right">#[rU]#</td>
<td class="small" align="right">#[ppm]#</td>
<td class="small" align="right">#[seeds]#</td>
#(complete)#
::
<td class="small"><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=" class="small">http://#[ip]#:#[port]#</a></td>
<td class=small>#[hash]#</td>
<td class=small>#[age]#</td>
<td class="small" align="right">#[connects]#</td>
#(/complete)#
</tr>
#{/list}#
</table>
Expand Down
2 changes: 1 addition & 1 deletion htroot/Network.java
View file
Expand Up @@ -302,6 +302,7 @@ record = yacyCore.newsPool.get(yacyNewsPool.INCOMING_DB, c);
prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get("Port", "-") );
prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge());
prop.put(STR_TABLE_LIST + conCount + "_complete_connects", groupDigits(seed.get("CCount", "0")));
} else {
prop.put(STR_TABLE_LIST + conCount + "_complete", 0);
}
Expand Down Expand Up @@ -338,7 +339,6 @@ record = yacyCore.newsPool.get(yacyNewsPool.INCOMING_DB, c);
prop.put(STR_TABLE_LIST + conCount + "_rU", groupDigits(seed.get(yacySeed.URL_IN, "0")));
prop.put(STR_TABLE_LIST + conCount + "_ppm", PPM);
prop.put(STR_TABLE_LIST + conCount + "_seeds", seed.get("SCount", "-"));
prop.put(STR_TABLE_LIST + conCount + "_connects", groupDigits(seed.get("CCount", "0")));
conCount++;
} // seed != null
} // while
Expand Down
10 changes: 7 additions & 3 deletions htroot/yacy/transferRWI.java
View file
Expand Up @@ -84,6 +84,10 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

if (granted) {
// log value status (currently added to find outOfMemory error
switchboard.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
long startProcess = System.currentTimeMillis();

// decode request
Vector v = new Vector();
int s = 0;
Expand All @@ -94,7 +98,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
s = e; while (s < indexes.length) if (indexes[s++] >= 32) {s--; break;}
}
// the value-vector should now have the same length as entryc
if (v.size() != entryc) System.out.println("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc);
if (v.size() != entryc) switchboard.getLog().logSevere("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc);

// now parse the Strings in the value-vector and write index entries
String estring;
Expand Down Expand Up @@ -128,10 +132,10 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
while (it.hasNext()) unknownURLs += "," + (String) it.next();
if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1);
if (wordhashes.length == 0)
switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs");
else {
double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[wordhashes.length - 1])) / 2.0;
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs");
}
result = "ok";
} else {
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/plasma/plasmaCrawlWorker.java
View file
Expand Up @@ -410,8 +410,8 @@ private static void load(
remote = null;

// restart crawling with new url
log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString() +
"\nRedirecting request to: " + redirectionUrl);
log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString());
log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl);

// if we are already doing a shutdown we don't need to retry crawling
if (Thread.currentThread().isInterrupted()) {
Expand Down
10 changes: 7 additions & 3 deletions source/de/anomic/plasma/plasmaSwitchboard.java
View file
Expand Up @@ -145,6 +145,7 @@ this class is also the core of the http crawling.
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyNewsPool;

public final class plasmaSwitchboard extends serverAbstractSwitch implements serverSwitch {

Expand Down Expand Up @@ -173,7 +174,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public String remoteProxyHost;
public int remoteProxyPort;
public boolean remoteProxyUse;
public static plasmaCrawlRobotsTxt robots;
public static plasmaCrawlRobotsTxt robots;
public plasmaCrawlProfile profiles;
public plasmaCrawlProfile.entry defaultProxyProfile;
public plasmaCrawlProfile.entry defaultRemoteProfile;
Expand All @@ -185,7 +186,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public long proxyLastAccess;
public yacyCore yc;
public HashMap indexingTasksInProcess;
public userDB userDB;
public userDB userDB;

private static final String STR_PROXYPROFILE = "defaultProxyProfile";
private static final String STR_REMOTEPROFILE = "defaultRemoteProfile";
Expand Down Expand Up @@ -700,13 +701,15 @@ public boolean cleanupJob() {

// clean up error stack
if ((urlPool.errorURL.stackSize() > 1000)) {
log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack");
urlPool.errorURL.clearStack();
hasDoneSomething = true;
}
// clean up loadedURL stack
for (int i = 1; i <= 6; i++) {
if (urlPool.loadedURL.getStackSize(i) > 1000) {
urlPool.loadedURL.clearStack(i);
log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i);
urlPool.loadedURL.clearStack(i);
hasDoneSomething = true;
}
}
Expand All @@ -715,6 +718,7 @@ public boolean cleanupJob() {

// clean up news
try {
log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
} catch (IOException e) {}

Expand Down

0 comments on commit 0c3a20d

Please sign in to comment.