From f4172cbb3da45f9268ac32fa4408955c3eafe06a Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 17 Nov 2013 00:17:25 +0100 Subject: [PATCH 01/10] fix for another XSS bug --- htroot/portalsearch/yacy-portalsearch.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/htroot/portalsearch/yacy-portalsearch.js b/htroot/portalsearch/yacy-portalsearch.js index 8121347e56..ed77cd46a2 100644 --- a/htroot/portalsearch/yacy-portalsearch.js +++ b/htroot/portalsearch/yacy-portalsearch.js @@ -242,6 +242,7 @@ function yacysearch(clear) { param[i] = item; }); param[param.length] = { name : 'startRecord', value : startRecord }; + ycurr = ycurr.replace("<"," ").replace(">"," "); $.ajaxSetup({ timeout: 10000, @@ -274,7 +275,7 @@ function yacysearch(clear) { else data = json; var searchTerms = ""; - searchTerms = data.channels[0].searchTerms; + searchTerms = data.channels[0].searchTerms.replace("<"," ").replace(">"," ");; if($.trim(ycurr.replace(/ /g,"+")) != searchTerms) { return false; From e4f49fb17596eb02fdf2f31b79225f3b840d3d95 Mon Sep 17 00:00:00 2001 From: reger Date: Mon, 18 Nov 2013 19:41:31 +0100 Subject: [PATCH 02/10] for searchresults with empty title use filename as title - to not store a title in index which isn't extracted from source the title is empty check only added to ResultEntry class --- source/net/yacy/search/snippet/ResultEntry.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/net/yacy/search/snippet/ResultEntry.java b/source/net/yacy/search/snippet/ResultEntry.java index de5e4d95f7..7da5c08589 100644 --- a/source/net/yacy/search/snippet/ResultEntry.java +++ b/source/net/yacy/search/snippet/ResultEntry.java @@ -141,7 +141,12 @@ public String urlname() { return (this.alternative_urlname == null) ? MultiProtocolURL.unescape(this.urlentry.url().toNormalform(true)) : this.alternative_urlname; } public String title() { - return this.urlentry.dc_title(); + String titlestr = this.urlentry.dc_title(); + // if title is empty use filename as title + if (titlestr.isEmpty()) { // if url has no filename, title is still empty (e.g. "www.host.com/" ) + titlestr = this.urlentry.url() != null ? this.urlentry.url().getFileName() : ""; + } + return titlestr; } public String publisher() { // dc:publisher From c2d720cdaf8f572c3b1481c4f922f56866d0a03d Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 18 Nov 2013 22:47:35 +0100 Subject: [PATCH 03/10] purge a lucene cache - possible memory leak fix --- htroot/IndexControlURLs_p.java | 3 +++ source/net/yacy/search/Switchboard.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 48da0982ce..6141c9557f 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.Set; +import org.apache.lucene.search.FieldCache; + import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.id.DigestURL; @@ -292,6 +294,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("rebootsolr")) { segment.fulltext().rebootSolr(); + FieldCache.DEFAULT.purgeAllCaches(); sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr reboot"); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index f618481efc..279a3076bd 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -82,6 +82,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import org.apache.lucene.search.FieldCache; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; @@ -2027,7 +2028,8 @@ public boolean cleanupJob() { // clear caches if (WordCache.sizeCommonWords() > 1000) WordCache.clearCommonWords(); Word.clearCache(); - // Domains.clear(); + // Domains.clear(); + FieldCache.DEFAULT.purgeAllCaches(); // clean up image stack ResultImages.clearQueues(); From b7f1e5af51666e7360746a8ee0f1c306488e0638 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 19 Nov 2013 15:56:10 +0100 Subject: [PATCH 04/10] added new servlet which generates the same file as the principal peers upload to a bootstrap position you can call it either with http://localhost:8090/yacy/seedlist.html or to generate json (or jsonp) with http://localhost:8090/yacy/seedlist.json http://localhost:8090/yacy/seedlist.json?callback=seedlist --- htroot/yacy/seedlist.html | 1 + htroot/yacy/seedlist.java | 89 +++++++++++++++++++++++++++++++ htroot/yacy/seedlist.json | 7 +++ source/net/yacy/peers/Seed.java | 4 +- source/net/yacy/peers/SeedDB.java | 65 +++++++++++----------- 5 files changed, 133 insertions(+), 33 deletions(-) create mode 100644 htroot/yacy/seedlist.html create mode 100644 htroot/yacy/seedlist.java create mode 100644 htroot/yacy/seedlist.json diff --git a/htroot/yacy/seedlist.html b/htroot/yacy/seedlist.html new file mode 100644 index 0000000000..19613da51f --- /dev/null +++ b/htroot/yacy/seedlist.html @@ -0,0 +1 @@ +#[encoded]# \ No newline at end of file diff --git a/htroot/yacy/seedlist.java b/htroot/yacy/seedlist.java new file mode 100644 index 0000000000..fa6d26c7d4 --- /dev/null +++ b/htroot/yacy/seedlist.java @@ -0,0 +1,89 @@ +// seedlist.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://yacy.net +// Frankfurt, Germany, 2013 +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.util.ArrayList; +import java.util.Map; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.peers.Seed; +import net.yacy.search.Switchboard; +import net.yacy.server.serverCore; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +/** + * this servlet generates the same file as the principal peers upload to a bootstrap position + * you can call it either with + * http://localhost:8090/yacy/seedlist.html + * or to generate json (or jsonp) with + * http://localhost:8090/yacy/seedlist.json + * http://localhost:8090/yacy/seedlist.json?callback=seedlist + */ +public final class seedlist { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + // return variable that accumulates replacements + final Switchboard sb = (Switchboard) env; + int maxcount = post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE); + final ArrayList v = sb.peers.getSeedlist(maxcount, true); + final serverObjects prop = new serverObjects(); + + // write simple-encoded seed lines or json + String EXT = header.get("EXT"); + boolean json = EXT != null && EXT.equals("json"); + + if (json) { + // check for JSONP + if ( post != null && post.containsKey("callback") ) { + prop.put("jsonp-start", post.get("callback") + "(["); + prop.put("jsonp-end", "]);"); + } else { + prop.put("jsonp-start", ""); + prop.put("jsonp-end", ""); + } + // construct json property lists + for (int i = 0; i < v.size(); i++) { + Map map = v.get(i).getMap(); + int c = 0; + for (Map.Entry m: map.entrySet()) { + prop.putJSON("peers_" + i + "_map_" + c + "_k", m.getKey()); + prop.putJSON("peers_" + i + "_map_" + c + "_v", m.getValue()); + prop.put("peers_" + i + "_map_" + c + "_c", 1); + c++; + } + prop.put("peers_" + i + "_map_" + (c - 1) + "_c", 0); + prop.put("peers_" + i + "_map", c); + prop.put("peers_" + i + "_c", i < v.size() - 1 ? 1 : 0); + } + prop.put("peers", v.size()); + } else { + final StringBuilder encoded = new StringBuilder(1024); + for (Seed seed: v) { + encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING); + } + prop.put("encoded", encoded.toString()); + } + + // return rewrite properties + return prop; + } + +} diff --git a/htroot/yacy/seedlist.json b/htroot/yacy/seedlist.json new file mode 100644 index 0000000000..1fac594ca9 --- /dev/null +++ b/htroot/yacy/seedlist.json @@ -0,0 +1,7 @@ +#[jsonp-start]#{ +"peers":[#{peers}#{ +#{map}#"#[k]#"="#[v]#"#(c)#::,#(/c)# +#{/map}#}#(c)#::,#(/c)##{/peers}# +] +}#[jsonp-end]# + diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 82e45db1e9..da2b847f1b 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -1102,8 +1102,8 @@ public final String genSeedStr(final String key) { final String r = toString(); final String z = crypt.simpleEncode(r, key, 'z'); final String b = crypt.simpleEncode(r, key, 'b'); - // the compressed string may be longer that the uncompressed if there is too much overhead for compression meta-info - // take simply that string that is shorter + // the compressed string may be longer than the uncompressed if there is too much overhead for compression meta-info + // take simply that string which is shorter return ( b.length() < z.length() ) ? b : z; } diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 85cd90f68f..1bcaa13338 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -403,7 +404,7 @@ public Iterator seedsConnected(final boolean up, final boolean rot, final return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion); } - private Iterator seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) { + public Iterator seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) { // enumerates seed-type objects: all seeds sequentially without order return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion); } @@ -716,40 +717,13 @@ private ArrayList storeSeedList(final File seedFile, final boolean addMy try { pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile))); - - // store own peer seed + List seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed); String line; - if (this.mySeed == null) initMySeed(); - if (addMySeed) { - line = this.mySeed.genSeedStr(null); + for (Seed seed: seedlist) { + line = seed.genSeedStr(null); v.add(line); pw.print(line + serverCore.CRLF_STRING); } - - // store active peer seeds - Seed ys; - Iterator se = seedsConnected(true, false, null, (float) 0.0); - while (se.hasNext()) { - ys = se.next(); - if (ys != null) { - line = ys.genSeedStr(null); - v.add(line); - pw.print(line + serverCore.CRLF_STRING); - } - } - - // store some of the not-so-old passive peer seeds (limit: 1 day) - se = seedsDisconnected(true, false, null, (float) 0.0); - final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L); - while (se.hasNext()) { - ys = se.next(); - if (ys != null) { - if (ys.getLastSeenUTC() < timeout) continue; - line = ys.genSeedStr(null); - v.add(line); - pw.print(line + serverCore.CRLF_STRING); - } - } pw.flush(); } finally { if (pw != null) try { pw.close(); } catch (final Exception e) {} @@ -757,6 +731,35 @@ private ArrayList storeSeedList(final File seedFile, final boolean addMy return v; } + public ArrayList getSeedlist(int maxcount, boolean addMySeed) { + final ArrayList v = new ArrayList(this.seedActiveDB.size() + 1000); + + // store own peer seed + if (addMySeed) v.add(this.mySeed); + + // store active peer seeds + Seed ys; + Iterator se = this.seedsConnected(true, false, null, (float) 0.0); + while (se.hasNext() && v.size() < maxcount) { + ys = se.next(); + if (ys != null) v.add(ys); + } + + // store some of the not-so-old passive peer seeds (limit: 1 day) + se = this.seedsDisconnected(true, false, null, (float) 0.0); + final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L); + while (se.hasNext() && v.size() < maxcount) { + ys = se.next(); + if (ys != null && ys.getLastSeenUTC() >= timeout) v.add(ys); + } + + final StringBuilder encoded = new StringBuilder(1024); + for (Seed seed: v) { + encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING); + } + return v; + } + protected String uploadSeedList(final yacySeedUploader uploader, final serverSwitch sb, final SeedDB seedDB, From 64048ff217356fb363ffc33b93abbdb002fa6a5b Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 22 Nov 2013 09:53:32 +0100 Subject: [PATCH 05/10] fir for XSS --- htroot/HostBrowser.java | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/htroot/HostBrowser.java b/htroot/HostBrowser.java index 8d53f191f8..242b33ada6 100644 --- a/htroot/HostBrowser.java +++ b/htroot/HostBrowser.java @@ -137,13 +137,13 @@ url, null, load, new Date(), sb.crawler.defaultProxyProfile.handle(), 0, 0, 0, 0 )); - prop.put("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString)); + prop.putHTML("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString)); if (wait) for (int i = 0; i < 30; i++) { if (sb.index.exists(ASCII.String(url.hash()))) break; try {Thread.sleep(100);} catch (final InterruptedException e) {} } } catch (final MalformedURLException e) { - prop.put("result", "bad url '" + load + "'"); + prop.putHTML("result", "bad url '" + load + "'"); } } @@ -192,7 +192,7 @@ url, null, load, new Date(), String host; while (i.hasNext() && c < maxcount) { host = i.next(); - prop.put("hosts_list_" + c + "_host", host); + prop.putHTML("hosts_list_" + c + "_host", host); boolean inCrawler = crawler.containsKey(host); int exclcount = exclscore.get(host); int failcount = failscore.get(host); @@ -234,13 +234,13 @@ url, null, load, new Date(), if (p > 0) path = path.substring(0, p + 1); } prop.put("files_complete", complete ? 1 : 0); - prop.put("files_complete_path", path); + prop.putHTML("files_complete_path", path); p = path.substring(0, path.length() - 1).lastIndexOf('/'); if (p < 8) { prop.put("files_root", 1); } else { prop.put("files_root", 0); - prop.put("files_root_path", path.substring(0, p + 1)); + prop.putHTML("files_root_path", path.substring(0, p + 1)); } try { // generate file list from path @@ -420,7 +420,7 @@ url, null, load, new Date(), if (entry.getValue() instanceof StoreType) { // this is a file prop.put("files_list_" + c + "_type", 0); - prop.put("files_list_" + c + "_type_url", entry.getKey()); + prop.putHTML("files_list_" + c + "_type_url", entry.getKey()); StoreType type = (StoreType) entry.getValue(); try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;} HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); @@ -439,7 +439,7 @@ url, null, load, new Date(), FailType failType = errorDocs.get(entry.getKey()); if (failType == null) { // maybe this is only in the errorURL - prop.put("files_list_" + c + "_type_stored_error", process == HarvestProcess.ERRORS ? sb.crawlQueues.errorURL.get(ASCII.String(uri.hash())).getFailReason() : "unknown error"); + prop.putHTML("files_list_" + c + "_type_stored_error", process == HarvestProcess.ERRORS ? sb.crawlQueues.errorURL.get(ASCII.String(uri.hash())).getFailReason() : "unknown error"); } else { String ids = ASCII.String(uri.hash()); InfoCacheEntry ice = infoCache.get(ids); @@ -447,8 +447,8 @@ url, null, load, new Date(), } } if (loadRight) { - prop.put("files_list_" + c + "_type_stored_load_url", entry.getKey()); - prop.put("files_list_" + c + "_type_stored_load_path", path); + prop.putHTML("files_list_" + c + "_type_stored_load_url", entry.getKey()); + prop.putHTML("files_list_" + c + "_type_stored_load_path", path); } if (++c >= maxcount) break; } @@ -471,7 +471,7 @@ url, null, load, new Date(), Iterator i = score.keys(false); while (i.hasNext() && c < maxcount) { host = i.next(); - prop.put("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host)); + prop.putHTML("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host)); prop.put("inbound_list_" + c + "_count", score.get(host)); c++; } @@ -490,7 +490,7 @@ url, null, load, new Date(), Iterator i = score.keys(false); while (i.hasNext() && c < maxcount) { host = i.next(); - prop.put("outbound_list_" + c + "_host", host); + prop.putHTML("outbound_list_" + c + "_host", host); prop.put("outbound_list_" + c + "_count", score.get(host)); prop.put("outbound_list_" + c + "_link", outboundHosts.get(host).getMinKey()); c++; From c927b428d3b8bc4f205d9b19df987db4a2e9ca98 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 22 Nov 2013 10:07:08 +0100 Subject: [PATCH 06/10] fixed json --- htroot/yacy/seedlist.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/htroot/yacy/seedlist.json b/htroot/yacy/seedlist.json index 1fac594ca9..b86b49ce03 100644 --- a/htroot/yacy/seedlist.json +++ b/htroot/yacy/seedlist.json @@ -1,7 +1,6 @@ #[jsonp-start]#{ "peers":[#{peers}#{ -#{map}#"#[k]#"="#[v]#"#(c)#::,#(/c)# +#{map}#"#[k]#":"#[v]#"#(c)#::,#(/c)# #{/map}#}#(c)#::,#(/c)##{/peers}# ] -}#[jsonp-end]# - +}#[jsonp-end]# \ No newline at end of file From 1f0bfa8fec6223b1f8c73d19a62f7bc8457f03e6 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 22 Nov 2013 10:38:42 +0100 Subject: [PATCH 07/10] added test to Base64Order (runs successfully!) --- source/net/yacy/cora/order/Base64Order.java | 61 ++++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/source/net/yacy/cora/order/Base64Order.java b/source/net/yacy/cora/order/Base64Order.java index 64d5eeec40..435aa4a401 100644 --- a/source/net/yacy/cora/order/Base64Order.java +++ b/source/net/yacy/cora/order/Base64Order.java @@ -20,8 +20,10 @@ package net.yacy.cora.order; +import java.io.IOException; import java.io.Serializable; import java.util.Comparator; +import java.util.Random; import net.yacy.cora.document.encoding.UTF8; @@ -51,18 +53,18 @@ public class Base64Order extends AbstractOrder implements ByteOrder, Com public static final Base64Order standardCoder = new Base64Order(true, true); public static final Base64Order enhancedCoder = new Base64Order(true, false); - private final boolean rfc1113compliant; + private final boolean rfc1521compliant; private final byte[] alpha; private final byte[] ahpla; private final byte[] ab; // decision table for comparisments - public Base64Order(final boolean up, final boolean rfc1113compliant) { - // if we choose not to be rfc1113compliant, + public Base64Order(final boolean up, final boolean rfc1521compliant) { + // if we choose not to be rfc1521compliant, // then we get shorter base64 results which are also filename-compatible - this.rfc1113compliant = rfc1113compliant; + this.rfc1521compliant = rfc1521compliant; this.asc = up; - this.alpha = (rfc1113compliant) ? alpha_standard : alpha_enhanced; - this.ahpla = (rfc1113compliant) ? ahpla_standard : ahpla_enhanced; + this.alpha = (rfc1521compliant) ? alpha_standard : alpha_enhanced; + this.ahpla = (rfc1521compliant) ? ahpla_standard : ahpla_enhanced; this.ab = new byte[1 << 14]; byte acc, bcc; byte c; @@ -90,7 +92,7 @@ public static byte[] zero(int length) { @Override public Order clone() { - final Base64Order o = new Base64Order(this.asc, this.rfc1113compliant); + final Base64Order o = new Base64Order(this.asc, this.rfc1521compliant); o.rotate(this.zero); return o; } @@ -121,10 +123,10 @@ public final static ByteOrder bySignature(final String signature) { @Override public final String signature() { - if ((!this.asc) && (!this.rfc1113compliant)) return "Bd"; - if ((!this.asc) && ( this.rfc1113compliant)) return "bd"; - if (( this.asc) && (!this.rfc1113compliant)) return "Bu"; - if (( this.asc) && ( this.rfc1113compliant)) return "bu"; + if ((!this.asc) && (!this.rfc1521compliant)) return "Bd"; + if ((!this.asc) && ( this.rfc1521compliant)) return "bd"; + if (( this.asc) && (!this.rfc1521compliant)) return "Bu"; + if (( this.asc) && ( this.rfc1521compliant)) return "bu"; return null; } @@ -218,7 +220,7 @@ public final String encode(final byte[] in) { } // now there may be remaining bytes if (in.length % 3 != 0) out = out.append((in.length % 3 == 2) ? encodeLongSB((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4).substring(0, 3) : encodeLongSB((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2)); - if (this.rfc1113compliant) while (out.length() % 4 > 0) out.append("="); + if (this.rfc1521compliant) while (out.length() % 4 > 0) out.append("="); // return result //assert lene == out.length() : "lene = " + lene + ", out.len = " + out.length(); return out.toString(); @@ -247,7 +249,7 @@ public final byte[] encodeSubstring(final byte[] in, final int sublen) { } } - if (this.rfc1113compliant) while (writepos % 4 > 0 && writepos < sublen) out[writepos] = '='; + if (this.rfc1521compliant) while (writepos % 4 > 0 && writepos < sublen) out[writepos] = '='; //assert encode(in).substring(0, sublen).equals(ASCII.String(out)); return out; } @@ -261,7 +263,7 @@ public final byte[] decode(String in) { try { int posIn = 0; int posOut = 0; - if (this.rfc1113compliant) while (in.charAt(in.length() - 1) == '=') in = in.substring(0, in.length() - 1); + if (this.rfc1521compliant) while (in.charAt(in.length() - 1) == '=') in = in.substring(0, in.length() - 1); final byte[] out = new byte[in.length() / 4 * 3 + (((in.length() % 4) == 0) ? 0 : in.length() % 4 - 1)]; long l; while (posIn + 3 < in.length()) { @@ -595,5 +597,36 @@ public static void main(final String[] s) { // return the cardinal of a given string as normalized float 0 .. 1 with the enhanced encoder System.out.println(((double) Base64Order.enhancedCoder.cardinal(s[1].getBytes())) / ((double) Long.MAX_VALUE)); } + if ("-test".equals(s[0])) { + // do some checks + Random r = new Random(System.currentTimeMillis()); + + // encode with enhancedCoder, decode with standard RFC 1521 base64 + sun.misc.BASE64Decoder rfc1521Decoder = new sun.misc.BASE64Decoder(); + for (int i = 0; i < 100000; i++) { + String challenge = Long.toString(r.nextLong()); + String eb64 = enhancedCoder.encode(UTF8.getBytes(challenge)); + String rfc1521 = new String(eb64); + while (rfc1521.length() % 4 != 0) rfc1521 += "="; + rfc1521 = rfc1521.replace('-', '+').replace('_', '/'); + System.out.println("Encode enhancedB64 + Decode RFC1521: Challenge=" + challenge + ", eb64=" + eb64 + ", rfc1521=" + rfc1521); + try { + if (!UTF8.String(rfc1521Decoder.decodeBuffer(rfc1521)).equals(challenge)) System.out.println("Encode Fail for " + challenge); + } catch (IOException e) {} + } + + // encode with enhancedCoder, decode with standard RFC 1521 base64 + sun.misc.BASE64Encoder rfc1521Encoder = new sun.misc.BASE64Encoder(); + for (int i = 0; i < 100000; i++) { + // encode with enhancedCoder, decode with standard RFC 1521 base64 + String challenge = Long.toString(r.nextLong()); + String rfc1521 = rfc1521Encoder.encode(UTF8.getBytes(challenge)); + String eb64 = new String(rfc1521); + while (eb64.endsWith("=")) eb64 = eb64.substring(0, eb64.length() - 1); + eb64 = eb64.replace('+', '-').replace('/', '_'); + System.out.println("Encode RFC1521 + Decode enhancedB64: Challenge=" + challenge + ", rfc1521=" + rfc1521 + ", eb64=" + eb64); + if (!UTF8.String(enhancedCoder.decode(eb64)).equals(challenge)) System.out.println("Encode Fail for " + challenge); + } + } } } From ccf2f4e43bdf084f89c8e94f68f3a8fc53d01565 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 22 Nov 2013 14:15:31 +0100 Subject: [PATCH 08/10] refactoring of seed attributes (introduced more constants) --- htroot/Blog.java | 3 ++- htroot/BlogComments.java | 3 ++- htroot/Wiki.java | 3 ++- source/net/yacy/peers/Network.java | 4 ++-- source/net/yacy/peers/PeerActions.java | 4 ++-- source/net/yacy/peers/Seed.java | 5 ++++- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/htroot/Blog.java b/htroot/Blog.java index 9ddbef9eed..1a2ab6eea6 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -45,6 +45,7 @@ import net.yacy.data.BlogBoard; import net.yacy.data.UserDB; import net.yacy.peers.NewsPool; +import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -112,7 +113,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (sb.peers.mySeed() == null) { strAuthor = "anonymous"; } else { - strAuthor = sb.peers.mySeed().get("Name", "anonymous"); + strAuthor = sb.peers.mySeed().get(Seed.NAME, "anonymous"); } } } diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index d51bb6ff07..7e21a15289 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -46,6 +46,7 @@ import net.yacy.data.UserDB; import net.yacy.data.BlogBoard.BlogEntry; import net.yacy.peers.Network; +import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -95,7 +96,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po if (sb.peers.mySeed() == null) { strAuthor = "anonymous"; } else { - strAuthor = sb.peers.mySeed().get("Name", "anonymous"); + strAuthor = sb.peers.mySeed().get(Seed.NAME, "anonymous"); } } } diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 25a6ea2259..b74c76140d 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -46,6 +46,7 @@ import net.yacy.data.Diff; import net.yacy.data.wiki.WikiBoard; import net.yacy.peers.NewsPool; +import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -79,7 +80,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po if (author.equals(ANONYMOUS)) { author = WikiBoard.guessAuthor(ip); if (author == null) { - author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get("Name", ANONYMOUS); + author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get(Seed.NAME, ANONYMOUS); } } diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 2b4935205e..502851e511 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -427,9 +427,9 @@ private int publishMySeed(final boolean force) { try { final NewsDB.Record record = this.sb.peers.newsPool.myPublication(); if ( record == null ) { - this.sb.peers.mySeed().put("news", ""); + this.sb.peers.mySeed().put(Seed.NEWS, ""); } else { - this.sb.peers.mySeed().put("news", net.yacy.utils.crypt.simpleEncode(record.toString())); + this.sb.peers.mySeed().put(Seed.NEWS, net.yacy.utils.crypt.simpleEncode(record.toString())); } } catch (final Exception e ) { log.severe("publishMySeed: problem with news encoding", e); diff --git a/source/net/yacy/peers/PeerActions.java b/source/net/yacy/peers/PeerActions.java index c7bc831b4d..845b109469 100644 --- a/source/net/yacy/peers/PeerActions.java +++ b/source/net/yacy/peers/PeerActions.java @@ -207,7 +207,7 @@ public void peerDeparture(final Seed peer, final String cause) { if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause); synchronized (this.seedDB) { if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; } - peer.put("dct", Long.toString(System.currentTimeMillis())); + peer.put(Seed.DCT, Long.toString(System.currentTimeMillis())); this.seedDB.addDisconnected(peer); // update info } EventChannel.channels(EventChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", "")); @@ -223,7 +223,7 @@ public void peerPing(final Seed peer) { } private void processPeerArrival(final Seed peer) { - final String recordString = peer.get("news", null); + final String recordString = peer.get(Seed.NEWS, null); //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); if ((recordString == null) || (recordString.isEmpty())) return; final String decodedString = net.yacy.utils.crypt.simpleDecode(recordString); diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index da2b847f1b..2061e1ebd6 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -141,7 +141,7 @@ public class Seed implements Cloneable, Comparable, Comparator /** the name of the peer (user-set) */ public static final String NAME = "Name"; - private static final String HASH = "Hash"; + public static final String HASH = "Hash"; /** Birthday - first startup */ private static final String BDATE = "BDate"; /** UTC-Offset */ @@ -169,6 +169,9 @@ public class Seed implements Cloneable, Comparable, Comparator public static final String IP = "IP"; public static final String PORT = "Port"; public static final String SEEDLISTURL = "seedURL"; + public static final String NEWS = "news"; // news attachment + public static final String DCT = "dct"; // disconnect time + /** zero-value */ private static final String ZERO = "0"; From caa20d63d99f90fed6ec406cf8563599622a5358 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 22 Nov 2013 14:15:52 +0100 Subject: [PATCH 09/10] fixed seedlist (hash was missing) --- htroot/yacy/seedlist.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/htroot/yacy/seedlist.java b/htroot/yacy/seedlist.java index fa6d26c7d4..02315a1404 100644 --- a/htroot/yacy/seedlist.java +++ b/htroot/yacy/seedlist.java @@ -39,10 +39,12 @@ */ public final class seedlist { + private static final int LISTMAX = 1000; + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; - int maxcount = post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE); + int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE)); final ArrayList v = sb.peers.getSeedlist(maxcount, true); final serverObjects prop = new serverObjects(); @@ -61,8 +63,11 @@ public static serverObjects respond(final RequestHeader header, final serverObje } // construct json property lists for (int i = 0; i < v.size(); i++) { + prop.putJSON("peers_" + i + "_map_0_k", Seed.HASH); + prop.putJSON("peers_" + i + "_map_0_v", v.get(i).hash); + prop.put("peers_" + i + "_map_0_c", 1); Map map = v.get(i).getMap(); - int c = 0; + int c = 1; for (Map.Entry m: map.entrySet()) { prop.putJSON("peers_" + i + "_map_" + c + "_k", m.getKey()); prop.putJSON("peers_" + i + "_map_" + c + "_v", m.getValue()); From 8da75a4b0cde4f79a7fd431f14199a10ad94f923 Mon Sep 17 00:00:00 2001 From: reger Date: Sun, 24 Nov 2013 04:31:08 +0100 Subject: [PATCH 10/10] fix contentType definition for Solr html responswriter from xml to html (hint: value is currently not used, but is in SolrServlet) --- .../federate/solr/responsewriter/GrepHTMLResponseWriter.java | 2 +- .../cora/federate/solr/responsewriter/HTMLResponseWriter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java index 2c83ff81fe..2f349cb1aa 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java @@ -68,7 +68,7 @@ public GrepHTMLResponseWriter() { @Override public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) { - return CONTENT_TYPE_XML_UTF8; + return "text/html"; } @Override diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java index b7132a308f..4575534b55 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java @@ -60,7 +60,7 @@ public HTMLResponseWriter() { @Override public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) { - return CONTENT_TYPE_XML_UTF8; + return "text/html"; } @Override