Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge origin/master into jetty
  • Loading branch information
reger committed Nov 24, 2013
2 parents 444a9ae + 8da75a4 commit 5c4a3d1
Show file tree
Hide file tree
Showing 17 changed files with 224 additions and 70 deletions.
3 changes: 2 additions & 1 deletion htroot/Blog.java
Expand Up @@ -45,6 +45,7 @@
import net.yacy.data.BlogBoard;
import net.yacy.data.UserDB;
import net.yacy.peers.NewsPool;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
Expand Down Expand Up @@ -112,7 +113,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
if (sb.peers.mySeed() == null) {
strAuthor = "anonymous";
} else {
strAuthor = sb.peers.mySeed().get("Name", "anonymous");
strAuthor = sb.peers.mySeed().get(Seed.NAME, "anonymous");
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion htroot/BlogComments.java
Expand Up @@ -46,6 +46,7 @@
import net.yacy.data.UserDB;
import net.yacy.data.BlogBoard.BlogEntry;
import net.yacy.peers.Network;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
Expand Down Expand Up @@ -95,7 +96,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po
if (sb.peers.mySeed() == null) {
strAuthor = "anonymous";
} else {
strAuthor = sb.peers.mySeed().get("Name", "anonymous");
strAuthor = sb.peers.mySeed().get(Seed.NAME, "anonymous");
}
}
}
Expand Down
22 changes: 11 additions & 11 deletions htroot/HostBrowser.java
Expand Up @@ -137,13 +137,13 @@ url, null, load, new Date(),
sb.crawler.defaultProxyProfile.handle(),
0, 0, 0, 0
));
prop.put("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString));
prop.putHTML("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString));
if (wait) for (int i = 0; i < 30; i++) {
if (sb.index.exists(ASCII.String(url.hash()))) break;
try {Thread.sleep(100);} catch (final InterruptedException e) {}
}
} catch (final MalformedURLException e) {
prop.put("result", "bad url '" + load + "'");
prop.putHTML("result", "bad url '" + load + "'");
}
}

Expand Down Expand Up @@ -192,7 +192,7 @@ url, null, load, new Date(),
String host;
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.put("hosts_list_" + c + "_host", host);
prop.putHTML("hosts_list_" + c + "_host", host);
boolean inCrawler = crawler.containsKey(host);
int exclcount = exclscore.get(host);
int failcount = failscore.get(host);
Expand Down Expand Up @@ -234,13 +234,13 @@ url, null, load, new Date(),
if (p > 0) path = path.substring(0, p + 1);
}
prop.put("files_complete", complete ? 1 : 0);
prop.put("files_complete_path", path);
prop.putHTML("files_complete_path", path);
p = path.substring(0, path.length() - 1).lastIndexOf('/');
if (p < 8) {
prop.put("files_root", 1);
} else {
prop.put("files_root", 0);
prop.put("files_root_path", path.substring(0, p + 1));
prop.putHTML("files_root_path", path.substring(0, p + 1));
}
try {
// generate file list from path
Expand Down Expand Up @@ -420,7 +420,7 @@ url, null, load, new Date(),
if (entry.getValue() instanceof StoreType) {
// this is a file
prop.put("files_list_" + c + "_type", 0);
prop.put("files_list_" + c + "_type_url", entry.getKey());
prop.putHTML("files_list_" + c + "_type_url", entry.getKey());
StoreType type = (StoreType) entry.getValue();
try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;}
HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash());
Expand All @@ -439,16 +439,16 @@ url, null, load, new Date(),
FailType failType = errorDocs.get(entry.getKey());
if (failType == null) {
// maybe this is only in the errorURL
prop.put("files_list_" + c + "_type_stored_error", process == HarvestProcess.ERRORS ? sb.crawlQueues.errorURL.get(ASCII.String(uri.hash())).getFailReason() : "unknown error");
prop.putHTML("files_list_" + c + "_type_stored_error", process == HarvestProcess.ERRORS ? sb.crawlQueues.errorURL.get(ASCII.String(uri.hash())).getFailReason() : "unknown error");
} else {
String ids = ASCII.String(uri.hash());
InfoCacheEntry ice = infoCache.get(ids);
prop.put("files_list_" + c + "_type_stored_error", failType == FailType.excl ? "excluded from indexing" : "load fail; " + ice.toString());
}
}
if (loadRight) {
prop.put("files_list_" + c + "_type_stored_load_url", entry.getKey());
prop.put("files_list_" + c + "_type_stored_load_path", path);
prop.putHTML("files_list_" + c + "_type_stored_load_url", entry.getKey());
prop.putHTML("files_list_" + c + "_type_stored_load_path", path);
}
if (++c >= maxcount) break;
}
Expand All @@ -471,7 +471,7 @@ url, null, load, new Date(),
Iterator<String> i = score.keys(false);
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.put("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host));
prop.putHTML("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host));
prop.put("inbound_list_" + c + "_count", score.get(host));
c++;
}
Expand All @@ -490,7 +490,7 @@ url, null, load, new Date(),
Iterator<String> i = score.keys(false);
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.put("outbound_list_" + c + "_host", host);
prop.putHTML("outbound_list_" + c + "_host", host);
prop.put("outbound_list_" + c + "_count", score.get(host));
prop.put("outbound_list_" + c + "_link", outboundHosts.get(host).getMinKey());
c++;
Expand Down
3 changes: 3 additions & 0 deletions htroot/IndexControlURLs_p.java
Expand Up @@ -34,6 +34,8 @@
import java.util.Map;
import java.util.Set;

import org.apache.lucene.search.FieldCache;

import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
Expand Down Expand Up @@ -292,6 +294,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea

if (post.containsKey("rebootsolr")) {
segment.fulltext().rebootSolr();
FieldCache.DEFAULT.purgeAllCaches();
sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr reboot");
}

Expand Down
3 changes: 2 additions & 1 deletion htroot/Wiki.java
Expand Up @@ -46,6 +46,7 @@
import net.yacy.data.Diff;
import net.yacy.data.wiki.WikiBoard;
import net.yacy.peers.NewsPool;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
Expand Down Expand Up @@ -79,7 +80,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po
if (author.equals(ANONYMOUS)) {
author = WikiBoard.guessAuthor(ip);
if (author == null) {
author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get("Name", ANONYMOUS);
author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get(Seed.NAME, ANONYMOUS);
}
}

Expand Down
3 changes: 2 additions & 1 deletion htroot/portalsearch/yacy-portalsearch.js
Expand Up @@ -242,6 +242,7 @@ function yacysearch(clear) {
param[i] = item;
});
param[param.length] = { name : 'startRecord', value : startRecord };
ycurr = ycurr.replace("<"," ").replace(">"," ");

$.ajaxSetup({
timeout: 10000,
Expand Down Expand Up @@ -274,7 +275,7 @@ function yacysearch(clear) {
else data = json;

var searchTerms = "";
searchTerms = data.channels[0].searchTerms;
searchTerms = data.channels[0].searchTerms.replace("<"," ").replace(">"," ");;

if($.trim(ycurr.replace(/ /g,"+")) != searchTerms) {
return false;
Expand Down
1 change: 1 addition & 0 deletions htroot/yacy/seedlist.html
@@ -0,0 +1 @@
#[encoded]#
94 changes: 94 additions & 0 deletions htroot/yacy/seedlist.java
@@ -0,0 +1,94 @@
// seedlist.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
// Frankfurt, Germany, 2013
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import java.util.ArrayList;
import java.util.Map;

import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;

/**
* this servlet generates the same file as the principal peers upload to a bootstrap position
* you can call it either with
* http://localhost:8090/yacy/seedlist.html
* or to generate json (or jsonp) with
* http://localhost:8090/yacy/seedlist.json
* http://localhost:8090/yacy/seedlist.json?callback=seedlist
*/
public final class seedlist {

private static final int LISTMAX = 1000;

public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
final ArrayList<Seed> v = sb.peers.getSeedlist(maxcount, true);
final serverObjects prop = new serverObjects();

// write simple-encoded seed lines or json
String EXT = header.get("EXT");
boolean json = EXT != null && EXT.equals("json");

if (json) {
// check for JSONP
if ( post != null && post.containsKey("callback") ) {
prop.put("jsonp-start", post.get("callback") + "([");
prop.put("jsonp-end", "]);");
} else {
prop.put("jsonp-start", "");
prop.put("jsonp-end", "");
}
// construct json property lists
for (int i = 0; i < v.size(); i++) {
prop.putJSON("peers_" + i + "_map_0_k", Seed.HASH);
prop.putJSON("peers_" + i + "_map_0_v", v.get(i).hash);
prop.put("peers_" + i + "_map_0_c", 1);
Map<String, String> map = v.get(i).getMap();
int c = 1;
for (Map.Entry<String, String> m: map.entrySet()) {
prop.putJSON("peers_" + i + "_map_" + c + "_k", m.getKey());
prop.putJSON("peers_" + i + "_map_" + c + "_v", m.getValue());
prop.put("peers_" + i + "_map_" + c + "_c", 1);
c++;
}
prop.put("peers_" + i + "_map_" + (c - 1) + "_c", 0);
prop.put("peers_" + i + "_map", c);
prop.put("peers_" + i + "_c", i < v.size() - 1 ? 1 : 0);
}
prop.put("peers", v.size());
} else {
final StringBuilder encoded = new StringBuilder(1024);
for (Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
}
prop.put("encoded", encoded.toString());
}

// return rewrite properties
return prop;
}

}
6 changes: 6 additions & 0 deletions htroot/yacy/seedlist.json
@@ -0,0 +1,6 @@
#[jsonp-start]#{
"peers":[#{peers}#{
#{map}#"#[k]#":"#[v]#"#(c)#::,#(/c)#
#{/map}#}#(c)#::,#(/c)##{/peers}#
]
}#[jsonp-end]#
Expand Up @@ -60,7 +60,7 @@ public HTMLResponseWriter() {

@Override
public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) {
return CONTENT_TYPE_XML_UTF8;
return "text/html";
}

@Override
Expand Down

0 comments on commit 5c4a3d1

Please sign in to comment.