Skip to content

Commit

Permalink
many YaCyNews fixes
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@461 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Jul 31, 2005
1 parent 13abd8b commit 1022fbe
Show file tree
Hide file tree
Showing 12 changed files with 66 additions and 29 deletions.
2 changes: 1 addition & 1 deletion htroot/IndexControl_p.java
View file
Expand Up @@ -91,7 +91,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
String urlstring = ((String) post.get("urlstring")).trim();
String urlhash = ((String) post.get("urlhash")).trim();

if (!(urlstring.startsWith("http://"))) urlstring = "http://" + urlstring;
if ((!(urlstring.startsWith("http://"))) && (!(urlstring.startsWith("https://")))) urlstring = "http://" + urlstring;

prop.put("keystring", keystring);
prop.put("keyhash", keyhash);
Expand Down
14 changes: 13 additions & 1 deletion htroot/IndexCreate_p.java
View file
Expand Up @@ -144,6 +144,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
String urlhash = plasmaURL.urlHash(crawlingStart);
switchboard.urlPool.loadedURL.remove(urlhash);
switchboard.urlPool.noticeURL.remove(urlhash);
switchboard.urlPool.errorURL.remove(urlhash);

// stack url
plasmaCrawlProfile.entry pe = switchboard.profiles.newEntry(crawlingStartURL.getHost(), crawlingStart, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingQ, storeHTCache, true, localIndexing, crawlOrder, xsstopw, xdstopw, xpstopw);
Expand All @@ -156,7 +157,18 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve

// generate a YaCyNews if the global flag was set
if (crawlOrder) {
yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", pe.map()));
Map m = new HashMap(pe.map()); // must be cloned
m.remove("specificDepth");
m.remove("localIndexing");
m.remove("remoteIndexing");
m.remove("xsstopw");
m.remove("xpstopw");
m.remove("xdstopw");
m.remove("storeTXCache");
m.remove("storeHTCache");
m.remove("generalFilter");
m.remove("specificFilter");
yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", m));
}

} else {
Expand Down
4 changes: 3 additions & 1 deletion htroot/News.html
View file
Expand Up @@ -42,7 +42,9 @@ <h2>Published&nbsp;News</h2>
<p>
<form action="News.html" method="post" enctype="multipart/form-data">
<input type="hidden" name="page" value="#[page]#">
<input type="submit" name="delete" value="#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"><br><br>
<input type="submit" name="deletespecific" value="#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#">
<input type="submit" name="deleteall" value="#(page)#::Process All News::Delete All News::Abort Publication of All News::Delete All News#(/page)#">
<br><br>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td class="small"></td>
Expand Down
36 changes: 24 additions & 12 deletions htroot/News.java
View file
Expand Up @@ -72,19 +72,31 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
return prop;
}

if ((post.containsKey("delete")) && (tableID >= 0)) {
Enumeration e = post.keys();
String check;
String id;
while (e.hasMoreElements()) {
check = (String) e.nextElement();
if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
yacyCore.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
if ((post.containsKey("deletespecific")) && (tableID >= 0)) {
Enumeration e = post.keys();
String check;
String id;
while (e.hasMoreElements()) {
check = (String) e.nextElement();
if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
yacyCore.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
}
}
}

if ((post.containsKey("deleteall")) && (tableID >= 0)) {
yacyNewsRecord record;
try {
while (yacyCore.newsPool.size(tableID) > 0) {
record = yacyCore.newsPool.get(tableID, 0);
yacyCore.newsPool.moveOff(tableID, record.id());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}

Expand Down Expand Up @@ -115,7 +127,7 @@ record = yacyCore.newsPool.get(tableID, i);
prop.put("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
prop.put("table_list_" + i + "_cre", yacyCore.universalDateShortString(record.created()));
prop.put("table_list_" + i + "_cat", record.category());
prop.put("table_list_" + i + "_rec", yacyCore.universalDateShortString(record.received()));
prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : yacyCore.universalDateShortString(record.received()));
prop.put("table_list_" + i + "_dis", record.distributed());
prop.put("table_list_" + i + "_att", record.attributes().toString());
} catch (IOException e) {e.printStackTrace();}
Expand Down
9 changes: 6 additions & 3 deletions source/de/anomic/kelondro/kelondroRecords.java
View file
Expand Up @@ -588,7 +588,7 @@ public byte[][] setValues(byte[][] row) throws IOException {
for (int j = values[i].length; j < COLWIDTHS[i]; j++)
entryFile.writeByte(0);
}
seek = seek + COLWIDTHS[i];
seek += COLWIDTHS[i];
}
}
updateNodeCache();
Expand Down Expand Up @@ -630,7 +630,8 @@ public byte[][] getValues() throws IOException {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
if (values[i][0] == 0) values[i] = null;
seek += COLWIDTHS[i];
}
}
return values;
Expand All @@ -643,7 +644,8 @@ public byte[][] getValues() throws IOException {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
if (values[i][0] == 0) values[i] = null;
seek += COLWIDTHS[i];
}
}
return values;
Expand Down Expand Up @@ -865,6 +867,7 @@ public static byte[] long2bytes(long x, int length) {
}

public static long bytes2long(byte[] b) {
if (b == null) return 0;
long x = 0;
for (int i = 0; i < b.length; i++) x = (x << 8) | (0xff & (int) b[i]);
return x;
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/plasma/plasmaCrawlLURL.java
View file
Expand Up @@ -370,7 +370,7 @@ public Entry(String urlHash) {
byte[][] entry = urlHashCache.get(urlHash.getBytes());
if (entry != null) {
this.url = new URL(new String(entry[1]).trim());
this.descr = new String(entry[2]).trim();
this.descr = (entry[2] == null) ? "" : new String(entry[2]).trim();
this.moddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[3])));
this.loaddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[4])));
this.referrerHash = new String(entry[5]);
Expand Down
4 changes: 4 additions & 0 deletions source/de/anomic/plasma/plasmaSwitchboard.java
View file
Expand Up @@ -701,6 +701,10 @@ public boolean coreCrawlJob() {
}
String profileHandle = urlEntry.profileHandle();
//System.out.println("DEBUG plasmaSwitchboard.processCrawling: profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
if (profileHandle == null) {
log.logError(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' (must be internal error) for URL " + urlEntry.url());
return true;
}
plasmaCrawlProfile.entry profile = profiles.getEntry(profileHandle);
if (profile == null) {
log.logError(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' (must be internal error) for URL " + urlEntry.url());
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/plasma/plasmaSwitchboardQueue.java
View file
Expand Up @@ -159,7 +159,7 @@ public Entry(URL url, String referrer, Date ifModifiedSince, boolean requestWith

public Entry(byte[][] row) {
long ims = serverCodings.enhancedCoder.decodeBase64Long(new String(row[2]));
byte flags = row[3][0];
byte flags = (row[3] == null) ? 0 : row[3][0];
try {
this.url = new URL(new String(row[0]));
} catch (MalformedURLException e) {
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/yacy/yacyNewsDB.java
View file
Expand Up @@ -159,7 +159,7 @@ private static yacyNewsRecord b2r(byte[][] b) throws IOException {
return new yacyNewsRecord(
new String(b[0]),
new String(b[1]),
yacyCore.parseUniversalDate(new String(b[2])),
(b[2] == null) ? null : yacyCore.parseUniversalDate(new String(b[2])),
(int) serverCodings.enhancedCoder.decodeBase64Long(new String(b[3])),
serverCodings.string2map(new String(b[4]))
);
Expand All @@ -172,7 +172,7 @@ private static byte[][] r2b(yacyNewsRecord r) throws IOException {
byte[][] b = new byte[5][];
b[0] = r.id().getBytes();
b[1] = r.category().getBytes();
b[2] = yacyCore.universalDateShortString(r.created()).getBytes();
b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes();
b[3] = serverCodings.enhancedCoder.encodeBase64Long(r.distributed(), 2).getBytes();
b[4] = attributes.getBytes();
return b;
Expand Down
6 changes: 4 additions & 2 deletions source/de/anomic/yacy/yacyNewsPool.java
View file
Expand Up @@ -111,7 +111,9 @@ public yacyNewsRecord myPublication() throws IOException {

public void enqueueIncomingNews(yacyNewsRecord record) throws IOException {
// called if a news is attached to a seed
if (newsDB.get(record.id()) == null) incomingNews.push(record);
if (record.created().getTime() == 0) return;
if (newsDB.get(record.id()) != null) return;
incomingNews.push(record);
}

public synchronized boolean commitIncomingNews(String id) throws IOException {
Expand Down Expand Up @@ -202,5 +204,5 @@ public void moveOff(int dbKey, String id) throws IOException {
case PUBLISHED_DB: deletePublishedNews(id); break;
}
}

}
9 changes: 5 additions & 4 deletions source/de/anomic/yacy/yacyNewsRecord.java
View file
Expand Up @@ -61,6 +61,7 @@ public class yacyNewsRecord {
public yacyNewsRecord(String newsString) {
this.attributes = serverCodings.string2map(newsString);
this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date();
//this.received = new Date();
this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date();
this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0;
Expand Down Expand Up @@ -101,10 +102,10 @@ public String toString() {
// this creates the string that shall be distributed
// attention: this has no additional encoding
if (this.originator != null) attributes.put("ori", this.originator);
if (this.category != null) attributes.put("cat", this.category);
attributes.put("cre", yacyCore.universalDateShortString(this.created));
attributes.put("rec", yacyCore.universalDateShortString(this.received));
attributes.put("dis", "" +this. distributed);
if (this.category != null) attributes.put("cat", this.category);
if (this.created != null) attributes.put("cre", yacyCore.universalDateShortString(this.created));
if (this.received != null) attributes.put("rec", yacyCore.universalDateShortString(this.received));
attributes.put("dis", "" + this.distributed);
String theString = attributes.toString();
removeStandards();
return theString;
Expand Down
3 changes: 2 additions & 1 deletion source/de/anomic/yacy/yacyPeerActions.java
View file
Expand Up @@ -94,7 +94,8 @@ public void updateMySeed() {
if (sb.getConfig("peerName", "nameless").equals("nameless")) {
// generate new peer name
String newPeerName = serverCore.publicIP().getHostName() + yacyCore.speedKey + serverSystem.infoKey() + (System.currentTimeMillis() & 99);
newPeerName = newPeerName.replace('.', '_');
newPeerName = newPeerName.replace('.', '-');
newPeerName = newPeerName.replace('_', '-');
sb.setConfig("peerName", newPeerName);
}
seedDB.mySeed.put("Name", sb.getConfig("peerName", "nameless"));
Expand Down

0 comments on commit 1022fbe

Please sign in to comment.