Skip to content

Commit

Permalink
enhancements to news and switchboard queue processing
Browse files Browse the repository at this point in the history 
removed direct access and replaced by iteration

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3961 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Jul 13, 2007
1 parent a45216b commit a4e8ad9
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 223 deletions.
86 changes: 37 additions & 49 deletions htroot/IndexCreateIndexingQueue_p.java
View file
Expand Up @@ -46,6 +46,7 @@
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Iterator;

import de.anomic.data.htmlTools;
import de.anomic.http.httpHeader;
Expand Down Expand Up @@ -95,19 +96,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
try {
synchronized (switchboard.sbQueue) {
ArrayList entries = switchboard.sbQueue.list(0);
for (int i=entries.size()-1; i >= 0; i--) {
plasmaSwitchboardQueue.Entry pcentry = (plasmaSwitchboardQueue.Entry) entries.get(i);
if (pcentry.urlHash().equals(urlHash)) {
plasmaSwitchboardQueue.Entry entry = switchboard.sbQueue.remove(i);
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
switchboard.cacheManager.deleteFile(entry.url());
}
break;
}
}
}
switchboard.sbQueue.remove(urlHash);
} catch (Exception e) {}
prop.put("LOCATION","");
return prop;
Expand All @@ -126,44 +115,43 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
plasmaSwitchboardQueue.Entry pcentry;
int inProcessCount = 0, entryCount = 0, totalCount = 0;
long totalSize = 0;
try {
ArrayList entryList = new ArrayList();
// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
inProcessCount = switchboard.indexingTasksInProcess.size();
entryList.addAll(switchboard.indexingTasksInProcess.values());
}
// getting all enqueued entries
if ((switchboard.sbQueue.size() > 0)) {
entryList.addAll(switchboard.sbQueue.list(0));
}
int count=entryList.size();
totalCount = count;
for (int i = 0; (i < count) && (entryCount < showLimit); i++) {
ArrayList entryList = new ArrayList();

// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
inProcessCount = switchboard.indexingTasksInProcess.size();
entryList.addAll(switchboard.indexingTasksInProcess.values());
}

// getting all enqueued entries
if ((switchboard.sbQueue.size() > 0)) {
Iterator i = switchboard.sbQueue.entryIterator(false);
while (i.hasNext()) entryList.add((plasmaSwitchboardQueue.Entry) i.next());
}

int count=entryList.size();
totalCount = count;
for (int i = 0; (i < count) && (entryCount < showLimit); i++) {

boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", (inProcess)? 2: ((dark) ? 1 : 0));
prop.put("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName())));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
prop.put("indexing-queue_list_"+entryCount+"_modified", pcentry.getModificationDate());
prop.put("indexing-queue_list_"+entryCount+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName()));
prop.put("indexing-queue_list_"+entryCount+"_url", htmlTools.replaceHTML(pcentry.normalizedURLString()));
prop.put("indexing-queue_list_"+entryCount+"_size", bytesToString(entrySize));
prop.put("indexing-queue_list_"+entryCount+"_inProcess", (inProcess)?1:0);
prop.put("indexing-queue_list_"+entryCount+"_inProcess_hash", pcentry.urlHash());
dark = !dark;
entryCount++;
}
boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", (inProcess)? 2: ((dark) ? 1 : 0));
prop.put("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName())));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
prop.put("indexing-queue_list_"+entryCount+"_modified", pcentry.getModificationDate());
prop.put("indexing-queue_list_"+entryCount+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName()));
prop.put("indexing-queue_list_"+entryCount+"_url", htmlTools.replaceHTML(pcentry.normalizedURLString()));
prop.put("indexing-queue_list_"+entryCount+"_size", bytesToString(entrySize));
prop.put("indexing-queue_list_"+entryCount+"_inProcess", (inProcess)?1:0);
prop.put("indexing-queue_list_"+entryCount+"_inProcess_hash", pcentry.urlHash());
dark = !dark;
entryCount++;
}
} catch (IOException e) {}
}

prop.put("indexing-queue_show", entryCount);//show shown entries
prop.put("indexing-queue_num", totalCount);//num entries in queue
Expand Down
77 changes: 39 additions & 38 deletions htroot/xml/queues_p.java
View file
Expand Up @@ -46,10 +46,11 @@

//package xml.queues;
package xml;
import java.io.IOException;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;

import de.anomic.data.htmlTools;
Expand Down Expand Up @@ -98,44 +99,44 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
int inProcessCount = 0;
long totalSize = 0;
int i=0; //counter
try {
ArrayList entryList = new ArrayList();
// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
inProcessCount = switchboard.indexingTasksInProcess.size();
entryList.addAll(switchboard.indexingTasksInProcess.values());
}
// getting all enqueued entries
if ((switchboard.sbQueue.size() > 0)) {
entryList.addAll(switchboard.sbQueue.list(0));
}
int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size());
if (size > entryList.size()) size = entryList.size();
int ok = 0;
for (i = 0; i < size; i++) {
boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName())));
prop.put("list-indexing_"+i+"_depth", pcentry.depth());
prop.put("list-indexing_"+i+"_modified", pcentry.getModificationDate());
prop.putSafeXML("list-indexing_"+i+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName()));
prop.putSafeXML("list-indexing_"+i+"_url", pcentry.normalizedURLString());
prop.put("list-indexing_"+i+"_size", entrySize);
prop.put("list-indexing_"+i+"_inProcess", (inProcess)?1:0);
prop.put("list-indexing_"+i+"_hash", pcentry.urlHash());
ok++;
}
ArrayList entryList = new ArrayList();

// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
inProcessCount = switchboard.indexingTasksInProcess.size();
entryList.addAll(switchboard.indexingTasksInProcess.values());
}

// getting all enqueued entries
if ((switchboard.sbQueue.size() > 0)) {
Iterator i1 = switchboard.sbQueue.entryIterator(false);
while (i1.hasNext()) entryList.add((plasmaSwitchboardQueue.Entry) i1.next());
}

int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size());
if (size > entryList.size()) size = entryList.size();

int ok = 0;
for (i = 0; i < size; i++) {
boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName())));
prop.put("list-indexing_"+i+"_depth", pcentry.depth());
prop.put("list-indexing_"+i+"_modified", pcentry.getModificationDate());
prop.putSafeXML("list-indexing_"+i+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName()));
prop.putSafeXML("list-indexing_"+i+"_url", pcentry.normalizedURLString());
prop.put("list-indexing_"+i+"_size", entrySize);
prop.put("list-indexing_"+i+"_inProcess", (inProcess)?1:0);
prop.put("list-indexing_"+i+"_hash", pcentry.urlHash());
ok++;
}
prop.put("list-indexing", ok);
} catch (IOException e) {}
}
prop.put("list-indexing", ok);
}

//loader queue
Expand Down

0 comments on commit a4e8ad9

Please sign in to comment.