Skip to content

Commit

Permalink
*) Displaying content size of each entry in indexing queue
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@639 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
theli committed Sep 2, 2005
1 parent 022c1ab commit 33aaffb
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 13 deletions.
2 changes: 2 additions & 0 deletions htroot/IndexCreateIndexingQueue_p.html
View file
Expand Up @@ -22,6 +22,7 @@ <h2>Index Creation: Indexing Queue</h2>
<th class="small">Modified Date</th>
<td class="small">Anchor Name</th>
<th class="small">URL</th>
<th class="small">Size</th>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" class="small">
Expand All @@ -30,6 +31,7 @@ <h2>Index Creation: Indexing Queue</h2>
<td width="80" class="small">#[modified]#</td>
<td width="180" class="small">#[anchor]#</td>
<td class="small"><a class="small" href="#[url]#">#[url]#</a></td>
<td width="80" class="small" align="right">#[size]#</td>
</tr>
#{/list}#
</table>
Expand Down
23 changes: 17 additions & 6 deletions htroot/IndexCreateIndexingQueue_p.java
View file
Expand Up @@ -81,6 +81,9 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
if (post.containsKey("moreRejected")) {
showRejectedCount = Integer.parseInt(post.get("showRejected", "10"));
}
if (post.containsKey("clearsbqueue")) {
//switchboard.sbQueue.
}
}

yacySeed initiator;
Expand All @@ -90,33 +93,41 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
if ((switchboard.sbQueue.size() == 0) && (switchboard.indexingTasksInProcess.size() == 0)) {
prop.put("indexing-queue", 0); //is empty
} else {
prop.put("indexing-queue", 1);
prop.put("indexing-queue_num", switchboard.sbQueue.size() + switchboard.indexingTasksInProcess.size());//num entries in queue
prop.put("indexing-queue", 1); // there are entries in the queue or in process

dark = true;
plasmaSwitchboardQueue.Entry pcentry;
int entryCount = 0;
try {
ArrayList entryList = new ArrayList();

// getting all entries that are currently in process
synchronized (switchboard.indexingTasksInProcess) {
entryList.addAll(switchboard.indexingTasksInProcess.values());
}

entryList.addAll(switchboard.sbQueue.list(0));
// getting all enqueued entries
entryList.addAll(switchboard.sbQueue.list(0));

for (i = 0; i < entryList.size(); i++) {
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
if (pcentry != null) {
if ((pcentry != null)&&(pcentry.url() != null)) {
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+i+"_dark", ((dark) ? 1 : 0));
prop.put("indexing-queue_list_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("indexing-queue_list_"+i+"_depth", pcentry.depth());
prop.put("indexing-queue_list_"+i+"_modified", (pcentry.responseHeader() == null) ? "null" : daydate(pcentry.responseHeader().lastModified()));
prop.put("indexing-queue_list_"+i+"_modified", (pcentry.responseHeader() == null) ? "" : daydate(pcentry.responseHeader().lastModified()));
prop.put("indexing-queue_list_"+i+"_anchor", (pcentry.anchorName()==null)?"":pcentry.anchorName());
prop.put("indexing-queue_list_"+i+"_url", pcentry.normalizedURLString());
prop.put("indexing-queue_list_"+i+"_size", Status.bytesToString(pcentry.size()));
dark = !dark;
entryCount++;
}
}
} catch (IOException e) {}
prop.put("indexing-queue_list", i);

prop.put("indexing-queue_num", entryCount);//num entries in queue
prop.put("indexing-queue_list", entryCount);
}

// failure cases
Expand Down
8 changes: 4 additions & 4 deletions source/de/anomic/plasma/plasmaCrawlLURL.java
View file
Expand Up @@ -342,14 +342,14 @@ public Entry(URL url, String descr, Date moddate, Date loaddate,
// create new entry and store it into database
this.urlHash = urlHash(url);
this.url = url;
this.descr = descr;
this.descr = (descr==null)?this.url.toString():descr;
this.moddate = moddate;
this.loaddate = loaddate;
this.referrerHash = (referrerHash == null) ? dummyHash : referrerHash;
this.copyCount = copyCount; // the number of remote (global) copies of this object without this one
this.flags = (localNeed) ? "L " : " ";
this.quality = quality;
this.language = language;
this.language = (language==null)?"uk":language;
this.doctype = doctype;
this.size = size;
this.wordCount = wordCount;
Expand All @@ -370,10 +370,10 @@ public Entry(String urlHash) {
byte[][] entry = urlHashCache.get(urlHash.getBytes());
if (entry != null) {
this.url = new URL(new String(entry[1]).trim());
this.descr = (entry[2] == null) ? "" : new String(entry[2]).trim();
this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2]).trim();
this.moddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[3])));
this.loaddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[4])));
this.referrerHash = new String(entry[5]);
this.referrerHash = (entry[5]==null)?dummyHash:new String(entry[5]);
this.copyCount = (int) serverCodings.enhancedCoder.decodeBase64Long(new String(entry[6]));
this.flags = new String(entry[7]);
this.quality = (int) serverCodings.enhancedCoder.decodeBase64Long(new String(entry[8]));
Expand Down
6 changes: 3 additions & 3 deletions source/de/anomic/plasma/plasmaCrawlNURL.java
View file
Expand Up @@ -334,7 +334,7 @@ public Entry(String initiator,
this.hash = urlHash(url);
this.initiator = initiator;
this.url = url;
this.referrer = (referrer == null) ? "------------" : referrer;
this.referrer = (referrer == null) ? dummyHash : referrer;
this.name = (name == null) ? "" : name;
this.loaddate = (loaddate == null) ? new Date() : loaddate;
this.profileHandle = profileHandle;
Expand All @@ -352,7 +352,7 @@ public String toString() {
str.append("hash: ").append(url==null ? "null" : urlHash(url)).append(" | ")
.append("initiator: ").append(initiator==null?"null":initiator).append(" | ")
.append("url: ").append(url==null?"null":url.toString()).append(" | ")
.append("referrer: ").append((referrer == null) ? "------------" : referrer).append(" | ")
.append("referrer: ").append((referrer == null) ? dummyHash : referrer).append(" | ")
.append("name: ").append((name == null) ? "null" : name).append(" | ")
.append("loaddate: ").append((loaddate == null) ? new Date() : loaddate).append(" | ")
.append("profile: ").append(profileHandle==null?"null":profileHandle).append(" | ")
Expand All @@ -378,7 +378,7 @@ public Entry(String hash) {
if (entry != null) {
this.initiator = new String(entry[1]);
this.url = new URL(new String(entry[2]).trim());
this.referrer = new String(entry[3]);
this.referrer = (entry[3]==null) ? dummyHash : new String(entry[3]);
this.name = (entry[4] == null) ? "" : new String(entry[4]).trim();
this.loaddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[5])));
this.profileHandle = new String(entry[6]).trim();
Expand Down

0 comments on commit 33aaffb

Please sign in to comment.