Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
redesigned IndexCreate menu -- introduced submenues to enable more cr…
…awl queue control pages git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@370 6c8d7289-2bf4-0310-a012-ef5d649a1542
- Loading branch information
orbiter
committed
Jul 4, 2005
1 parent
57c30f1
commit 9a3f804
Showing
10 changed files
with
634 additions
and
294 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd"> | ||
<html> | ||
<head> | ||
<title>YaCy: Index Creation/Indexing Queue</title> | ||
#[metas]# | ||
</head> | ||
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0"> | ||
#[header]# | ||
#[submenuIndexCreate]# | ||
<br> | ||
<h2>Index Creation: Indexing Queue</h2> | ||
|
||
<p> | ||
#(indexing-queue)# | ||
The indexing queue is empty<br> | ||
:: | ||
There are #[num]# entries in the indexing queue:<br> | ||
<table border="0" cellpadding="2" cellspacing="1"> | ||
<tr class="TableHeader"> | ||
<th class="small">Initiator</th> | ||
<th class="small">Depth</th> | ||
<th class="small">Modified Date</th> | ||
<th class="small">#HREF</th> | ||
<td class="small">Anchor Name</th> | ||
<th class="small">URL</th> | ||
</tr> | ||
#{list}# | ||
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" class="small"> | ||
<td width="60" class="small">#[initiator]#</td> | ||
<td width="10" class="small">#[depth]#</td> | ||
<td width="80" class="small">#[modified]#</td> | ||
<td width="10" class="small">#[href]#</td> | ||
<td width="180" class="small">#[anchor]#</td> | ||
<td class="small"><a class="small" href="#[url]#">#[url]#</a></td> | ||
</tr> | ||
#{/list}# | ||
</table> | ||
#(/indexing-queue)# | ||
</p><br> | ||
|
||
<p> | ||
#(rejected)# | ||
:: | ||
<form action="IndexCreateIndexingQueue_p.html" method="post" enctype="multipart/form-data"> | ||
<b>Rejected URL List:</b> There are #[num]# entries in the rejected-urls list. | ||
#(only-latest)# | ||
:: | ||
Showing latest #[num]# entries. | ||
<input type="hidden" name="showRejected" value="#[newnum]#"> | ||
<input type="submit" name="moreRejected" value="show more"> | ||
#(/only-latest)# | ||
<input type="submit" name="clearRejected" value="clear list"> | ||
</form> | ||
There are #[num]# entries in the rejected-queue:<br> | ||
<table border="0" cellpadding="2" cellspacing="1" width="100%"> | ||
<tr class="TableHeader"> | ||
<th class="small">Initiator</th> | ||
<th class="small">Executor</th> | ||
<th class="small">URL</th> | ||
<th class="small">Fail-Reason</th> | ||
</tr> | ||
#{list}# | ||
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" class="small"> | ||
<td width="60" class="small">#[initiator]#</td> | ||
<td width="60" class="small">#[executor]#</td> | ||
<td class="small"><a class="small" href="#[url]#">#[url]#</a></td> | ||
<td class="small">#[failreason]#</td> | ||
</tr> | ||
#{/list}# | ||
</table> | ||
#(/rejected)# | ||
</p> | ||
|
||
#[footer]# | ||
</body> | ||
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,161 @@ | ||
// IndexCreateIndexingQueue_p.java | ||
// ------------------------------- | ||
// part of the AnomicHTTPD caching proxy | ||
// (C) by Michael Peter Christen; mc@anomic.de | ||
// first published on http://www.anomic.de | ||
// Frankfurt, Germany, 2004, 2005 | ||
// last major change: 04.07.2005 | ||
// | ||
// This program is free software; you can redistribute it and/or modify | ||
// it under the terms of the GNU General Public License as published by | ||
// the Free Software Foundation; either version 2 of the License, or | ||
// (at your option) any later version. | ||
// | ||
// This program is distributed in the hope that it will be useful, | ||
// but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
// GNU General Public License for more details. | ||
// | ||
// You should have received a copy of the GNU General Public License | ||
// along with this program; if not, write to the Free Software | ||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
// | ||
// Using this software in any meaning (reading, learning, copying, compiling, | ||
// running) means that you agree that the Author(s) is (are) not responsible | ||
// for cost, loss of data or any harm that may be caused directly or indirectly | ||
// by usage of this softare or this documentation. The usage of this software | ||
// is on your own risk. The installation and usage (starting/running) of this | ||
// software may allow other people or application to access your computer and | ||
// any attached devices and is highly dependent on the configuration of the | ||
// software which must be done by the user of the software; the author(s) is | ||
// (are) also not responsible for proper configuration and usage of the | ||
// software, even if provoked by documentation provided together with | ||
// the software. | ||
// | ||
// Any changes to this file according to the GPL as documented in the file | ||
// gpl.txt aside this file in the shipment you received can be done to the | ||
// lines that follows this copyright notice here, but changes must not be | ||
// done inside the copyright notive above. A re-distribution must contain | ||
// the intact and unchanged copyright notice. | ||
// Contributions and changes to the program code must be marked as such. | ||
|
||
// You must compile this file with | ||
// javac -classpath .:../classes IndexCreate_p.java | ||
// if the shell's current path is HTROOT | ||
|
||
import java.net.MalformedURLException; | ||
import java.net.URL; | ||
import java.text.SimpleDateFormat; | ||
import java.util.Date; | ||
import java.util.Enumeration; | ||
import java.util.Iterator; | ||
import java.util.Locale; | ||
|
||
import de.anomic.http.httpHeader; | ||
import de.anomic.plasma.plasmaCrawlEURL; | ||
import de.anomic.plasma.plasmaCrawlLoaderMessage; | ||
import de.anomic.plasma.plasmaCrawlNURL; | ||
import de.anomic.plasma.plasmaCrawlProfile; | ||
import de.anomic.plasma.plasmaCrawlWorker; | ||
import de.anomic.plasma.plasmaHTCache; | ||
import de.anomic.plasma.plasmaSwitchboard; | ||
import de.anomic.plasma.plasmaURL; | ||
import de.anomic.server.serverObjects; | ||
import de.anomic.server.serverSwitch; | ||
import de.anomic.yacy.yacyCore; | ||
import de.anomic.yacy.yacySeed; | ||
|
||
public class IndexCreateIndexingQueue_p { | ||
|
||
private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US); | ||
private static String daydate(Date date) { | ||
if (date == null) return ""; else return dayFormatter.format(date); | ||
} | ||
|
||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { | ||
// return variable that accumulates replacements | ||
plasmaSwitchboard switchboard = (plasmaSwitchboard) env; | ||
serverObjects prop = new serverObjects(); | ||
prop.put("rejected", 0); | ||
int showRejectedCount = 10; | ||
|
||
if (post != null) { | ||
if (post.containsKey("clearRejected")) { | ||
switchboard.urlPool.errorURL.clearStack(); | ||
} | ||
if (post.containsKey("moreRejected")) { | ||
showRejectedCount = Integer.parseInt(post.get("showRejected", "10")); | ||
} | ||
} | ||
|
||
yacySeed initiator; | ||
boolean dark; | ||
int i; | ||
|
||
if (switchboard.queueStack.size() == 0) { | ||
prop.put("indexing-queue", 0); //is empty | ||
} else { | ||
prop.put("indexing-queue", 1); | ||
prop.put("indexing-queue_num", switchboard.queueStack.size());//num entries in queue | ||
dark = true; | ||
plasmaHTCache.Entry pcentry; | ||
for (i = 0; i < switchboard.queueStack.size(); i++) { | ||
pcentry = (plasmaHTCache.Entry) switchboard.queueStack.get(i); | ||
if (pcentry != null) { | ||
initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); | ||
prop.put("indexing-queue_list_"+i+"_dark", ((dark) ? 1 : 0)); | ||
prop.put("indexing-queue_list_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName())); | ||
prop.put("indexing-queue_list_"+i+"_depth", pcentry.depth); | ||
prop.put("indexing-queue_list_"+i+"_modified", daydate(pcentry.lastModified)); | ||
prop.put("indexing-queue_list_"+i+"_href",((pcentry.scraper == null) ? "0" : ("" + pcentry.scraper.getAnchors().size()))); | ||
prop.put("indexing-queue_list_"+i+"_anchor", ((pcentry.scraper == null) ? "-" : pcentry.scraper.getHeadline()) ); | ||
prop.put("indexing-queue_list_"+i+"_url", pcentry.nomalizedURLString); | ||
dark = !dark; | ||
} | ||
} | ||
prop.put("indexing-queue_list", i); | ||
} | ||
|
||
// failure cases | ||
if (switchboard.urlPool.errorURL.stackSize() != 0) { | ||
if (showRejectedCount > switchboard.urlPool.errorURL.stackSize()) showRejectedCount = switchboard.urlPool.errorURL.stackSize(); | ||
prop.put("rejected", 1); | ||
prop.put("rejected_num", switchboard.urlPool.errorURL.stackSize()); | ||
if (showRejectedCount != switchboard.urlPool.errorURL.stackSize()) { | ||
prop.put("rejected_only-latest", 1); | ||
prop.put("rejected_only-latest_num", showRejectedCount); | ||
prop.put("rejected_only-latest_newnum", ((int) (showRejectedCount * 1.5))); | ||
}else{ | ||
prop.put("rejected_only-latest", 0); | ||
} | ||
dark = true; | ||
String url, initiatorHash, executorHash; | ||
plasmaCrawlEURL.entry entry; | ||
yacySeed initiatorSeed, executorSeed; | ||
int j=0; | ||
for (i = switchboard.urlPool.errorURL.stackSize() - 1; i >= (switchboard.urlPool.errorURL.stackSize() - showRejectedCount); i--) { | ||
entry = (plasmaCrawlEURL.entry) switchboard.urlPool.errorURL.getStack(i); | ||
initiatorHash = entry.initiator(); | ||
executorHash = entry.executor(); | ||
url = entry.url().toString(); | ||
initiatorSeed = yacyCore.seedDB.getConnected(initiatorHash); | ||
executorSeed = yacyCore.seedDB.getConnected(executorHash); | ||
prop.put("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName())); | ||
prop.put("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName())); | ||
prop.put("rejected_list_"+j+"_url", url); | ||
prop.put("rejected_list_"+j+"_failreason", entry.failreason()); | ||
prop.put("rejected_list_"+j+"_dark", ((dark) ? 1 : 0)); | ||
dark = !dark; | ||
j++; | ||
} | ||
prop.put("rejected_list", j); | ||
} | ||
|
||
// return rewrite properties | ||
return prop; | ||
} | ||
|
||
} | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd"> | ||
<html> | ||
<head> | ||
<title>YaCy: Index Creation / Loader Queue</title> | ||
#[metas]# | ||
</head> | ||
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0"> | ||
#[header]# | ||
#[submenuIndexCreate]# | ||
<br> | ||
<h2>Index Creation: Loader Queue</h2> | ||
|
||
<p> | ||
#(loader-set)# | ||
The loader set is empty<br> | ||
:: | ||
There are #[num]# entries in the loader set:<br> | ||
<table border="0" cellpadding="2" cellspacing="1"> | ||
<tr class="TableHeader"> | ||
<th class="small">Initiator</th> | ||
<th class="small">Depth</td> | ||
<th class="small">URL</th> | ||
</tr> | ||
#{list}# | ||
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" class="small"> | ||
<td width="60" class="small">#[initiator]#</td> | ||
<td width="10" class="small">#[depth]#</td> | ||
<td class="small"><a class="small" href="#[url]#">#[url]#</a></td> | ||
</tr> | ||
#{/list}# | ||
</table> | ||
#(/loader-set)# | ||
</p> | ||
|
||
#[footer]# | ||
</body> | ||
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
// IndexCreateLoaderQueue_p.java | ||
// ----------------------------- | ||
// part of the AnomicHTTPD caching proxy | ||
// (C) by Michael Peter Christen; mc@anomic.de | ||
// first published on http://www.anomic.de | ||
// Frankfurt, Germany, 2004, 2005 | ||
// last major change: 04.07.2005 | ||
// | ||
// This program is free software; you can redistribute it and/or modify | ||
// it under the terms of the GNU General Public License as published by | ||
// the Free Software Foundation; either version 2 of the License, or | ||
// (at your option) any later version. | ||
// | ||
// This program is distributed in the hope that it will be useful, | ||
// but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
// GNU General Public License for more details. | ||
// | ||
// You should have received a copy of the GNU General Public License | ||
// along with this program; if not, write to the Free Software | ||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
// | ||
// Using this software in any meaning (reading, learning, copying, compiling, | ||
// running) means that you agree that the Author(s) is (are) not responsible | ||
// for cost, loss of data or any harm that may be caused directly or indirectly | ||
// by usage of this softare or this documentation. The usage of this software | ||
// is on your own risk. The installation and usage (starting/running) of this | ||
// software may allow other people or application to access your computer and | ||
// any attached devices and is highly dependent on the configuration of the | ||
// software which must be done by the user of the software; the author(s) is | ||
// (are) also not responsible for proper configuration and usage of the | ||
// software, even if provoked by documentation provided together with | ||
// the software. | ||
// | ||
// Any changes to this file according to the GPL as documented in the file | ||
// gpl.txt aside this file in the shipment you received can be done to the | ||
// lines that follows this copyright notice here, but changes must not be | ||
// done inside the copyright notive above. A re-distribution must contain | ||
// the intact and unchanged copyright notice. | ||
// Contributions and changes to the program code must be marked as such. | ||
|
||
// You must compile this file with | ||
// javac -classpath .:../classes IndexCreate_p.java | ||
// if the shell's current path is HTROOT | ||
|
||
import java.net.MalformedURLException; | ||
import java.net.URL; | ||
import java.text.SimpleDateFormat; | ||
import java.util.Date; | ||
import java.util.Enumeration; | ||
import java.util.Iterator; | ||
import java.util.Locale; | ||
|
||
import de.anomic.http.httpHeader; | ||
import de.anomic.plasma.plasmaCrawlEURL; | ||
import de.anomic.plasma.plasmaCrawlLoaderMessage; | ||
import de.anomic.plasma.plasmaCrawlNURL; | ||
import de.anomic.plasma.plasmaCrawlProfile; | ||
import de.anomic.plasma.plasmaCrawlWorker; | ||
import de.anomic.plasma.plasmaHTCache; | ||
import de.anomic.plasma.plasmaSwitchboard; | ||
import de.anomic.plasma.plasmaURL; | ||
import de.anomic.server.serverObjects; | ||
import de.anomic.server.serverSwitch; | ||
import de.anomic.yacy.yacyCore; | ||
import de.anomic.yacy.yacySeed; | ||
|
||
public class IndexCreateLoaderQueue_p { | ||
|
||
private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US); | ||
private static String daydate(Date date) { | ||
if (date == null) return ""; else return dayFormatter.format(date); | ||
} | ||
|
||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { | ||
// return variable that accumulates replacements | ||
plasmaSwitchboard switchboard = (plasmaSwitchboard) env; | ||
serverObjects prop = new serverObjects(); | ||
|
||
|
||
if (switchboard.cacheLoader.size() == 0) { | ||
prop.put("loader-set", 0); | ||
} else { | ||
prop.put("loader-set", 1); | ||
prop.put("loader-set_num", switchboard.cacheLoader.size()); | ||
boolean dark = true; | ||
|
||
ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus(); | ||
|
||
int threadCount = loaderThreads.activeCount(); | ||
Thread[] threadList = new Thread[threadCount*2]; | ||
threadCount = loaderThreads.enumerate(threadList); | ||
yacySeed initiator; | ||
int i; | ||
for (i = 0; i < threadCount; i++) { | ||
plasmaCrawlWorker theWorker = (plasmaCrawlWorker)threadList[i]; | ||
plasmaCrawlLoaderMessage theMsg = theWorker.theMsg; | ||
if (theMsg == null) continue; | ||
|
||
initiator = yacyCore.seedDB.getConnected(theMsg.initiator); | ||
prop.put("loader-set_list_"+i+"_dark", ((dark) ? 1 : 0) ); | ||
prop.put("loader-set_list_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) ); | ||
prop.put("loader-set_list_"+i+"_depth", theMsg.depth ); | ||
prop.put("loader-set_list_"+i+"_url", theMsg.url ); // null pointer exception here !!! maybe url = null; check reason. | ||
dark = !dark; | ||
} | ||
prop.put("loader-set_list", i ); | ||
} | ||
|
||
// return rewrite properties | ||
return prop; | ||
} | ||
|
||
} | ||
|
||
|
||
|
Oops, something went wrong.