Skip to content

Commit

Permalink
*) Bugfix for Loader Queue:
Browse files Browse the repository at this point in the history 
   Job count was not displayed correctly
*) IndexingQueue:
- now it's possible to delete single entries from the queue
- now it's possible to clear the whole queue
  See: http://www.yacy-forum.de/viewtopic.php?t=995

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@641 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
theli committed Sep 2, 2005
1 parent 732a107 commit dc0a2d4
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 14 deletions.
10 changes: 8 additions & 2 deletions htroot/IndexCreateIndexingQueue_p.html
View file
Expand Up @@ -14,6 +14,10 @@ <h2>Index Creation: Indexing Queue</h2>
#(indexing-queue)#
The indexing queue is empty<br>
::
<form action="IndexCreateIndexingQueue_p.html" method="post" enctype="multipart/form-data">
<input type="submit" name="clearIndexingQueue" value="clear indexing queue">
</form>
<br>
There are #[num]# entries in the indexing queue:<br>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
Expand All @@ -23,15 +27,17 @@ <h2>Index Creation: Indexing Queue</h2>
<td class="small">Anchor Name</th>
<th class="small">URL</th>
<th class="small">Size</th>
<th class="small">Delete</th>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" class="small">
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#" class="small">
<td width="60" class="small">#[initiator]#</td>
<td width="10" class="small">#[depth]#</td>
<td width="80" class="small">#[modified]#</td>
<td width="180" class="small">#[anchor]#</td>
<td class="small"><a class="small" href="#[url]#">#[url]#</a></td>
<td width="80" class="small" align="right">#[size]#</td>
<td width="80" class="small" align="right"><nobr>#[size]#</nobr></td>
<td width="10" class="small">#(inProcess)#<a class="small" href="IndexCreateIndexingQueue_p.html?deleteEntry=#[hash]#">[Delete]</a>::#(/inProcess)#</td>
</tr>
#{/list}#
</table>
Expand Down
43 changes: 34 additions & 9 deletions htroot/IndexCreateIndexingQueue_p.java
View file
Expand Up @@ -77,18 +77,37 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
if (post != null) {
if (post.containsKey("clearRejected")) {
switchboard.urlPool.errorURL.clearStack();
}
}
if (post.containsKey("moreRejected")) {
showRejectedCount = Integer.parseInt(post.get("showRejected", "10"));
}
if (post.containsKey("clearsbqueue")) {
//switchboard.sbQueue.
if (post.containsKey("clearIndexingQueue")) {
try {
synchronized (switchboard.sbQueue) {
switchboard.sbQueue.clear();
}
} catch (Exception e) {}
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
try {
synchronized (switchboard.sbQueue) {
ArrayList entries = switchboard.sbQueue.list(0);
for (int i=entries.size()-1; i >= 0; i--) {
plasmaSwitchboardQueue.Entry pcentry = (plasmaSwitchboardQueue.Entry) entries.get(i);
if (pcentry.urlHash().equals(urlHash)) {
switchboard.sbQueue.remove(i);
break;
}
}
}
} catch (Exception e) {}
prop.put("LOCATION","");
return prop;
}
}

yacySeed initiator;
boolean dark;
int i=0;

if ((switchboard.sbQueue.size() == 0) && (switchboard.indexingTasksInProcess.size() == 0)) {
prop.put("indexing-queue", 0); //is empty
Expand All @@ -97,29 +116,35 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve

dark = true;
plasmaSwitchboardQueue.Entry pcentry;
int entryCount = 0;
int inProcessCount = 0, entryCount = 0;
try {
ArrayList entryList = new ArrayList();

// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
inProcessCount = switchboard.indexingTasksInProcess.size();
entryList.addAll(switchboard.indexingTasksInProcess.values());
}

// getting all enqueued entries
entryList.addAll(switchboard.sbQueue.list(0));
if ((switchboard.sbQueue.size() > 0)) {
entryList.addAll(switchboard.sbQueue.list(0));
}

for (i = 0; i < entryList.size(); i++) {
for (int i = 0; i < entryList.size(); i++) {
boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", ((dark) ? 1 : 0));
prop.put("indexing-queue_list_"+entryCount+"_dark", (inProcess)? 2: ((dark) ? 1 : 0));
prop.put("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
prop.put("indexing-queue_list_"+entryCount+"_modified", (pcentry.responseHeader() == null) ? "" : daydate(pcentry.responseHeader().lastModified()));
prop.put("indexing-queue_list_"+entryCount+"_anchor", (pcentry.anchorName()==null)?"":pcentry.anchorName());
prop.put("indexing-queue_list_"+entryCount+"_url", pcentry.normalizedURLString());
prop.put("indexing-queue_list_"+entryCount+"_size", Status.bytesToString(pcentry.size()));
prop.put("indexing-queue_list_"+entryCount+"_inProcess", (inProcess)?1:0);
prop.put("indexing-queue_list_"+entryCount+"_0_hash", pcentry.urlHash());
dark = !dark;
entryCount++;
}
Expand Down Expand Up @@ -147,7 +172,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
plasmaCrawlEURL.Entry entry;
yacySeed initiatorSeed, executorSeed;
int j=0;
for (i = switchboard.urlPool.errorURL.stackSize() - 1; i >= (switchboard.urlPool.errorURL.stackSize() - showRejectedCount); i--) {
for ( int i = switchboard.urlPool.errorURL.stackSize() - 1; i >= (switchboard.urlPool.errorURL.stackSize() - showRejectedCount); i--) {
entry = (plasmaCrawlEURL.Entry) switchboard.urlPool.errorURL.getStack(i);
initiatorHash = entry.initiator();
executorHash = entry.executor();
Expand Down
5 changes: 2 additions & 3 deletions htroot/IndexCreateLoaderQueue_p.java
View file
Expand Up @@ -73,11 +73,9 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
prop.put("loader-set", 0);
} else {
prop.put("loader-set", 1);
prop.put("loader-set_num", switchboard.cacheLoader.size());
boolean dark = true;

ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus();

ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus();
int threadCount = loaderThreads.activeCount();
Thread[] threadList = new Thread[threadCount*2];
threadCount = loaderThreads.enumerate(threadList);
Expand All @@ -96,6 +94,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
dark = !dark;
}
prop.put("loader-set_list", i );
prop.put("loader-set_num", i);
}

// return rewrite properties
Expand Down
9 changes: 9 additions & 0 deletions source/de/anomic/plasma/plasmaSwitchboardQueue.java
View file
Expand Up @@ -107,6 +107,11 @@ public Entry pop() throws IOException {
return new Entry(sbQueueStack.pot());
}

public Entry remove(int index) throws IOException {
if (sbQueueStack.size() == 0) return null;
return new Entry(sbQueueStack.pot(index));
}

public Entry get(int index) throws IOException {
if ((index < 0) || (index >= sbQueueStack.size())) throw new ArrayIndexOutOfBoundsException();
return new Entry(sbQueueStack.bot(index));
Expand All @@ -125,6 +130,10 @@ public ArrayList list(int index) throws IOException {
return list;
}

public void clear() throws IOException {
sbQueueStack.clear();
}

public void close() {
if (sbQueueStack != null) try {
sbQueueStack.close();
Expand Down

0 comments on commit dc0a2d4

Please sign in to comment.