Skip to content

Commit

Permalink
added peer tags for peers in robinson mode
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2741 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Oct 10, 2006
1 parent f744789 commit 2bb529c
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 24 deletions.
8 changes: 8 additions & 0 deletions htroot/IndexControl_p.html
View file
Expand Up @@ -99,6 +99,14 @@ <h2>Index Administration</h2>
<td><input type="checkbox" name="indexReceiveBlockBlacklist" #[indexReceiveBlockBlacklistChecked]# /></td>
<td>If checked, your peer silently ignores transmitted URLs that match your blacklist</td>
</tr>
<tr valign="top" class="TableCellDark">
<td>Peer Tags:</td>
<td colspan="2"><input type="text" name="peertags" value="#[peertags]#" size="40" maxlength="80" /></td>
<td>If your peer runs in 'Robinson Mode' (Distribution and Receive off), you probably run YaCy as a search engine
for your own search portal. Please describe your search portal with some keywords (comma-separated).
This will help to use your peer as search target even if you do not distribute your web index by
DHT distribution.</td>
</tr>
<tr valign="top" class="TableCellLight">
<td></td>
<td><input type="submit" name="setIndexTransmission" value="set" /></td>
Expand Down
7 changes: 7 additions & 0 deletions htroot/IndexControl_p.java
View file
Expand Up @@ -65,6 +65,7 @@
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaWordIndex;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverCodings;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
Expand Down Expand Up @@ -92,6 +93,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
prop.put("indexDistributeWhileCrawling", (switchboard.getConfig("allowDistributeIndexWhileCrawling", "true").equals("true")) ? "checked" : "");
prop.put("indexReceiveChecked", (switchboard.getConfig("allowReceiveIndex", "true").equals("true")) ? "checked" : "");
prop.put("indexReceiveBlockBlacklistChecked", (switchboard.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "checked" : "");
prop.put("peertags", serverCodings.set2string(yacyCore.seedDB.mySeed.getPeerTags(), ",", false));
return prop; // be save
}

Expand Down Expand Up @@ -143,6 +145,10 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
} else {
switchboard.setConfig("indexReceiveBlockBlacklist", "false");
}

if (post.containsKey("peertags")) {
yacyCore.seedDB.mySeed.setPeerTags(serverCodings.string2set((String) post.get("peertags"), ","));
}
}

// delete word
Expand Down Expand Up @@ -392,6 +398,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
prop.put("indexDistributeWhileCrawling", (switchboard.getConfig("allowDistributeIndexWhileCrawling", "true").equals("true")) ? "checked" : "");
prop.put("indexReceiveChecked", (switchboard.getConfig("allowReceiveIndex", "true").equals("true")) ? "checked" : "");
prop.put("indexReceiveBlockBlacklistChecked", (switchboard.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "checked" : "");
prop.put("peertags", serverCodings.set2string(yacyCore.seedDB.mySeed.getPeerTags(), ",", false));
// return rewrite properties
return prop;
}
Expand Down
2 changes: 1 addition & 1 deletion htroot/Network.html
View file
Expand Up @@ -70,7 +70,7 @@ <h2>Manually contacting Peer</h2>
#(isCrawling)#::<a href="#[page]#"><img src="/env/grafics/crawl.gif" alt="Crawl" /></a>#(/isCrawling)#
</td>
<td><a href="http://www.#[fullname]#.yacy">#[shortname]#</a></td>
<td>#(type)##(direct)#<img src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive" alt="Junior passive" />::<img src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct" alt="Junior direct" /></img>::<img src="/env/grafics/JuniorOffline.gif" title="Type: Junior | Contact: offline" alt="Junior offline" />#(/direct)#::#(direct)#<img src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive" alt="senior passive" />::<img src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct" alt="Senior direct" />::<img src="/env/grafics/SeniorOffline.gif" title="Type: Senior | Contact: offline" alt="Senior offline" />#(/direct)#::<a href="#[url]#">#(direct)#<img src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible" alt="Principal passive" />::<img src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible" alt="Principal active" />::<img src="/env/grafics/PrincipalOffline.gif" title="Type: Principal | Contact: offline | Seed download: ?" alt="Principal offline" />#(/direct)#</a>#(/type)##(acceptcrawl)#<img src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no" alt="no crawl" />::<img src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes" alt="crawl possible" />::<img src="/env/grafics/CrawlYesOffline.gif" title="Accept Crawl: yes" alt="crawl possible" />#(/acceptcrawl)##(dhtreceive)#<img src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no" alt="no DHT receive" />::<img src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes" alt="DHT receive enabled" />::<img src="/env/grafics/DHTReceiveYesOffline.gif" title="DHT Receive: yes" alt="DHT receive enabled" />#(/dhtreceive)##(rankingreceive)#<img src="/env/grafics/RankingReceiveNo.gif" title="Ranking Receive: no" alt="no ranking receive" />::<img src="/env/grafics/RankingReceiveYes.gif" title="Ranking Receive: yes" alt="Ranking receive enabled" />::<img src="/env/grafics/RankingCollects.gif" title="Ranking Receive: collection" alt="ranking collects" />#(/rankingreceive)#</td>
<td>#(type)##(direct)#<img src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive" alt="Junior passive" />::<img src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct" alt="Junior direct" /></img>::<img src="/env/grafics/JuniorOffline.gif" title="Type: Junior | Contact: offline" alt="Junior offline" />#(/direct)#::#(direct)#<img src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive" alt="senior passive" />::<img src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct" alt="Senior direct" />::<img src="/env/grafics/SeniorOffline.gif" title="Type: Senior | Contact: offline" alt="Senior offline" />#(/direct)#::<a href="#[url]#">#(direct)#<img src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible" alt="Principal passive" />::<img src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible" alt="Principal active" />::<img src="/env/grafics/PrincipalOffline.gif" title="Type: Principal | Contact: offline | Seed download: ?" alt="Principal offline" />#(/direct)#</a>#(/type)##(acceptcrawl)#<img src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no" alt="no crawl" />::<img src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes" alt="crawl possible" />::<img src="/env/grafics/CrawlYesOffline.gif" title="Accept Crawl: yes" alt="crawl possible" />#(/acceptcrawl)##(dhtreceive)#<img src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no; #[peertags]#" alt="no DHT receive" />::<img src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes" alt="DHT receive enabled" />::<img src="/env/grafics/DHTReceiveYesOffline.gif" title="DHT Receive: yes" alt="DHT receive enabled" />#(/dhtreceive)##(rankingreceive)#<img src="/env/grafics/RankingReceiveNo.gif" title="Ranking Receive: no" alt="no ranking receive" />::<img src="/env/grafics/RankingReceiveYes.gif" title="Ranking Receive: yes" alt="Ranking receive enabled" />::<img src="/env/grafics/RankingCollects.gif" title="Ranking Receive: collection" alt="ranking collects" />#(/rankingreceive)#</td>
<td align="right">#[version]#</td>
<td align="right">#[ppm]#</td>
<td align="right">#[lastSeen]#</td>
Expand Down
8 changes: 8 additions & 0 deletions htroot/Network.java
View file
Expand Up @@ -54,6 +54,7 @@

import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
Expand Down Expand Up @@ -409,13 +410,20 @@ record = yacyCore.newsPool.get(yacyNewsPool.INCOMING_DB, c);
} else {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", 0); // red/red; offline was off
}

if (seed.getVersion() >= yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION &&
seed.getFlagAcceptCitationReference()) {
prop.put(STR_TABLE_LIST + conCount + "_rankingreceive", 1);
} else {
prop.put(STR_TABLE_LIST + conCount + "_rankingreceive", 0);
}
}
if (seed.getFlagAcceptRemoteIndex()) {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", "");
} else {
String peertags = serverCodings.set2string(seed.getPeerTags(), ",", false);
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags));
}
prop.put(STR_TABLE_LIST + conCount + "_version", yacy.combinedVersionString2PrettyString(seed.get(yacySeed.VERSION, "0.1")));
prop.put(STR_TABLE_LIST + conCount + "_lastSeen", lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
Expand Down
83 changes: 68 additions & 15 deletions source/de/anomic/server/serverCodings.java
View file
Expand Up @@ -47,8 +47,13 @@
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.Map.Entry;

public final class serverCodings {

Expand Down Expand Up @@ -154,21 +159,69 @@ public static Properties s2p(String s) {
return p;
}

public static HashMap string2map(String string) {
public static HashMap string2map(String string, String separator) {
// this can be used to parse a Map.toString() into a Map again
if (string == null) return null;
HashMap map = new HashMap();
int pos;
pos = string.indexOf("{"); if (pos >= 0) string = string.substring(pos + 1).trim();
pos = string.lastIndexOf("}"); if (pos >= 0) string = string.substring(0, pos).trim();
StringTokenizer st = new StringTokenizer(string, ",");
String token;
while (st.hasMoreTokens()) {
token = st.nextToken().trim();
pos = token.indexOf("=");
if (pos > 0) map.put(token.substring(0, pos).trim(), token.substring(pos + 1).trim());
}
return map;
if (string == null) return null;
HashMap map = new HashMap();
int pos;
if ((pos = string.indexOf("{")) >= 0) string = string.substring(pos + 1).trim();
if ((pos = string.lastIndexOf("}")) >= 0) string = string.substring(0, pos).trim();
StringTokenizer st = new StringTokenizer(string, separator);
String token;
while (st.hasMoreTokens()) {
token = st.nextToken().trim();
pos = token.indexOf("=");
if (pos > 0) map.put(token.substring(0, pos).trim(), token.substring(pos + 1).trim());
}
return map;
}

public static String map2string(Map m, String separator, boolean braces) {
StringBuffer buf = new StringBuffer();
if (braces) buf.append("{");
Iterator i = m.entrySet().iterator();
boolean hasNext = i.hasNext();
while (hasNext) {
Entry e = (Entry) (i.next());
Object key = e.getKey();
Object value = e.getValue();
buf.append(key.toString());
buf.append('=');
buf.append(value.toString());

hasNext = i.hasNext();
if (hasNext) buf.append(separator);
}
if (braces) buf.append("}");
return buf.toString();
}

public static Set string2set(String string, String separator) {
// this can be used to parse a Map.toString() into a Map again
if (string == null) return null;
HashSet set = new HashSet();
int pos;
if ((pos = string.indexOf("{")) >= 0) string = string.substring(pos + 1).trim();
if ((pos = string.lastIndexOf("}")) >= 0) string = string.substring(0, pos).trim();
StringTokenizer st = new StringTokenizer(string, separator);
while (st.hasMoreTokens()) {
set.add(st.nextToken().trim());
}
return set;
}

public static String set2string(Set s, String separator, boolean braces) {
StringBuffer buf = new StringBuffer();
if (braces) buf.append("{");
Iterator i = s.iterator();
boolean hasNext = i.hasNext();
while (hasNext) {
buf.append(i.next().toString());
hasNext = i.hasNext();
if (hasNext) buf.append(separator);
}
if (braces) buf.append("}");
return buf.toString();
}

public static void main(String[] s) {
Expand All @@ -179,7 +232,7 @@ public static void main(String[] s) {

if (s[0].equals("-s2m")) {
// generate a b64 decoding from a given string
System.out.println(string2map(s[1]).toString());
System.out.println(string2map(s[1], ",").toString());
}
}

Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/yacy/yacyNewsAction.java
View file
Expand Up @@ -64,8 +64,8 @@ public void processPeerArrival(yacySeed peer, boolean direct) {
String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, "");
yacyNewsRecord record = new yacyNewsRecord(decodedString);
//System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString());
String cre1 = (String) serverCodings.string2map(decodedString).get("cre");
String cre2 = (String) serverCodings.string2map(record.toString()).get("cre");
String cre1 = (String) serverCodings.string2map(decodedString, ",").get("cre");
String cre2 = (String) serverCodings.string2map(record.toString(), ",").get("cre");
if ((cre1 == null) || (cre2 == null) || (!(cre1.equals(cre2)))) {
System.out.println("### ERROR - cre are not equal: cre1=" + cre1 + ", cre2=" + cre2);
return;
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/yacy/yacyNewsDB.java
View file
Expand Up @@ -160,7 +160,7 @@ protected final static yacyNewsRecord b2r(kelondroRow.Entry b) {
b.getColString(1, "UTF-8"),
(b.empty(2)) ? null : yacyCore.parseUniversalDate(b.getColString(2, null), serverDate.UTCDiffString()),
(int) b.getColLong(3),
serverCodings.string2map(b.getColString(4, "UTF-8"))
serverCodings.string2map(b.getColString(4, "UTF-8"), ",")
);
}

Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/yacy/yacyNewsRecord.java
View file
Expand Up @@ -79,7 +79,7 @@ public class yacyNewsRecord {
);

public yacyNewsRecord(String newsString) {
this.attributes = serverCodings.string2map(newsString);
this.attributes = serverCodings.string2map(newsString, ",");
this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec"), serverDate.UTCDiffString()) : new Date();
this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre"), serverDate.UTCDiffString()) : new Date();
this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
Expand Down
18 changes: 14 additions & 4 deletions source/de/anomic/yacy/yacySeed.java
View file
Expand Up @@ -67,6 +67,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.net.natLib;
Expand Down Expand Up @@ -128,6 +129,7 @@ public class yacySeed {
public static final String NAME = "Name";
public static final String BDATE = "BDate";
public static final String UTC = "UTC";
public static final String PEERTAGS = "Tags";

public static final String ISPEED = "ISpeed";
public static final String UPTIME = "Uptime";
Expand Down Expand Up @@ -392,6 +394,14 @@ public void setLastSeenTime() {
put(LASTSEEN, yacyCore.shortFormatter.format(new Date(System.currentTimeMillis() - serverDate.UTCDiff() + getUTCDiff())));
}

public void setPeerTags(Set keys) {
put(PEERTAGS, serverCodings.set2string(keys, "|", false));
}

public Set getPeerTags() {
return serverCodings.string2set(get(PEERTAGS, ""), "|");
}

public int getPPM() {
try {
return Integer.parseInt(get(ISPEED, "0"));
Expand Down Expand Up @@ -609,7 +619,7 @@ public static yacySeed genRemoteSeed(String seedStr, String key, boolean properT
if (seedStr == null) { return null; }
final String seed = crypt.simpleDecode(seedStr, key);
if (seed == null) { return null; }
final HashMap dna = serverCodings.string2map(seed);
final HashMap dna = serverCodings.string2map(seed, ",");
final String hash = (String) dna.remove("Hash");
yacySeed resultSeed = new yacySeed(hash, dna);
if (properTest) {
Expand All @@ -624,9 +634,9 @@ public static yacySeed genRemoteSeed(String seedStr, String key, boolean properT

public String toString() {
synchronized (this.dna) {
this.dna.put("Hash", this.hash); // set hash into seed code structure
final String s = this.dna.toString(); // generate string representation
this.dna.remove("Hash"); // reconstruct original: hash is stored external
this.dna.put("Hash", this.hash); // set hash into seed code structure
final String s = serverCodings.map2string(this.dna, ",", true); // generate string representation
this.dna.remove("Hash"); // reconstruct original: hash is stored external
return s;
}
}
Expand Down

0 comments on commit 2bb529c

Please sign in to comment.