Skip to content

Commit

Permalink
... migrating to HttpComponents-Client-4.x ...
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6966 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
sixcooler committed Jul 13, 2010
1 parent c5c67f0 commit e1316d1
Show file tree
Hide file tree
Showing 4 changed files with 222 additions and 81 deletions.
161 changes: 107 additions & 54 deletions source/de/anomic/yacy/yacyClient.java
View file
Expand Up @@ -60,6 +60,7 @@
import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.RSSReader;
import net.yacy.cora.protocol.ByteArrayBody;
import net.yacy.cora.protocol.HttpConnector;
import net.yacy.cora.services.Search;
import net.yacy.kelondro.data.meta.URIMetadataRow;
Expand All @@ -77,15 +78,14 @@
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;

import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
//import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;

import de.anomic.crawler.ResultURLs;
import de.anomic.crawler.retrieval.EventOrigin;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.http.client.DefaultCharsetFilePart;
import de.anomic.http.client.DefaultCharsetStringPart;
import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework;
Expand Down Expand Up @@ -147,10 +147,10 @@ public static int publishMySeed(final yacySeed mySeed, final yacyPeerActions pee
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt);
try {
// generate request
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt);
// post.add(new DefaultCharsetStringPart("count", "20"));
parts.put("count", new StringBody("20"));
// post.add(new DefaultCharsetStringPart("seed", mySeed.genSeedStr(salt)));
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt);
parts.put("count", new StringBody("20"));
parts.put("seed", new StringBody(mySeed.genSeedStr(salt)));
// send request
final long start = System.currentTimeMillis();
Expand Down Expand Up @@ -336,15 +336,20 @@ public static RSSFeed queryRemoteCrawlURLs(final yacySeedDB seedDB, final yacySe

// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
post.add(new DefaultCharsetStringPart("call", "remotecrawl"));
post.add(new DefaultCharsetStringPart("count", Integer.toString(maxCount)));
post.add(new DefaultCharsetStringPart("time", Long.toString(maxTime)));
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
// post.add(new DefaultCharsetStringPart("call", "remotecrawl"));
// post.add(new DefaultCharsetStringPart("count", Integer.toString(maxCount)));
// post.add(new DefaultCharsetStringPart("time", Long.toString(maxTime)));

// send request
try {
/* a long time-out is needed */
final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", post, (int) maxTime);
// final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", post, (int) maxTime);
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("call", new StringBody("remotecrawl"));
parts.put("count", new StringBody(Integer.toString(maxCount)));
parts.put("time", new StringBody(Long.toString(maxTime)));
final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", parts, (int) maxTime);
final RSSReader reader = RSSReader.parse(result);
if (reader == null) {
yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");
Expand Down Expand Up @@ -417,33 +422,54 @@ public static int search(

// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
post.add(new DefaultCharsetStringPart("myseed", mySeed.genSeedStr(salt)));
post.add(new DefaultCharsetStringPart("count", Integer.toString(Math.max(10, count))));
post.add(new DefaultCharsetStringPart("resource", ((global) ? "global" : "local")));
post.add(new DefaultCharsetStringPart("partitions", Integer.toString(partitions)));
post.add(new DefaultCharsetStringPart("query", wordhashes));
post.add(new DefaultCharsetStringPart("exclude", excludehashes));
post.add(new DefaultCharsetStringPart("duetime", "1000"));
post.add(new DefaultCharsetStringPart("urls", urlhashes));
post.add(new DefaultCharsetStringPart("prefer", prefer.toString()));
post.add(new DefaultCharsetStringPart("filter", filter.toString()));
post.add(new DefaultCharsetStringPart("language", language));
post.add(new DefaultCharsetStringPart("sitehash", sitehash));
post.add(new DefaultCharsetStringPart("authorhash", authorhash));
post.add(new DefaultCharsetStringPart("ttl", "0"));
post.add(new DefaultCharsetStringPart("maxdist", Integer.toString(maxDistance)));
post.add(new DefaultCharsetStringPart("profile", crypt.simpleEncode(rankingProfile.toExternalString())));
post.add(new DefaultCharsetStringPart("constraint", (constraint == null) ? "" : constraint.exportB64()));
if (secondarySearchSuperviser != null) post.add(new DefaultCharsetStringPart("abstracts", "auto"));
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
// post.add(new DefaultCharsetStringPart("myseed", mySeed.genSeedStr(salt)));
// post.add(new DefaultCharsetStringPart("count", Integer.toString(Math.max(10, count))));
// post.add(new DefaultCharsetStringPart("resource", ((global) ? "global" : "local")));
// post.add(new DefaultCharsetStringPart("partitions", Integer.toString(partitions)));
// post.add(new DefaultCharsetStringPart("query", wordhashes));
// post.add(new DefaultCharsetStringPart("exclude", excludehashes));
// post.add(new DefaultCharsetStringPart("duetime", "1000"));
// post.add(new DefaultCharsetStringPart("urls", urlhashes));
// post.add(new DefaultCharsetStringPart("prefer", prefer.toString()));
// post.add(new DefaultCharsetStringPart("filter", filter.toString()));
// post.add(new DefaultCharsetStringPart("language", language));
// post.add(new DefaultCharsetStringPart("sitehash", sitehash));
// post.add(new DefaultCharsetStringPart("authorhash", authorhash));
// post.add(new DefaultCharsetStringPart("ttl", "0"));
// post.add(new DefaultCharsetStringPart("maxdist", Integer.toString(maxDistance)));
// post.add(new DefaultCharsetStringPart("profile", crypt.simpleEncode(rankingProfile.toExternalString())));
// post.add(new DefaultCharsetStringPart("constraint", (constraint == null) ? "" : constraint.exportB64()));
// if (secondarySearchSuperviser != null) post.add(new DefaultCharsetStringPart("abstracts", "auto"));
final long timestamp = System.currentTimeMillis();
boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;

// send request
Map<String, String> result = null;
try {
result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", post, 60000));
// result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", post, 60000));
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("myseed", new StringBody(mySeed.genSeedStr(salt)));
parts.put("count", new StringBody(Integer.toString(Math.max(10, count))));
parts.put("resource", new StringBody(((global) ? "global" : "local")));
parts.put("partitions", new StringBody(Integer.toString(partitions)));
parts.put("query", new StringBody(wordhashes));
parts.put("exclude", new StringBody(excludehashes));
parts.put("duetime", new StringBody("1000"));
parts.put("urls", new StringBody(urlhashes));
parts.put("prefer", new StringBody(prefer.toString()));
parts.put("filter", new StringBody(filter.toString()));
parts.put("language", new StringBody(language));
parts.put("sitehash", new StringBody(sitehash));
parts.put("authorhash", new StringBody(authorhash));
parts.put("ttl", new StringBody("0"));
parts.put("maxdist", new StringBody(Integer.toString(maxDistance)));
parts.put("profile", new StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
parts.put("constraint", new StringBody((constraint == null) ? "" : constraint.exportB64()));
if (secondarySearchSuperviser != null)
parts.put("abstracts", new StringBody("auto"));
result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", parts, 60000));
} catch (final IOException e) {
yacyCore.log.logInfo("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + "), score=" + target.selectscore);
//yacyCore.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage());
Expand Down Expand Up @@ -706,16 +732,23 @@ public static Map<String, String> transferPermission(final String targetAddress,

// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt);
post.add(new DefaultCharsetStringPart("process", "permission"));
post.add(new DefaultCharsetStringPart("purpose", "crcon"));
post.add(new DefaultCharsetStringPart("filename", filename));
post.add(new DefaultCharsetStringPart("filesize", Long.toString(filesize)));
post.add(new DefaultCharsetStringPart("can-send-protocol", "http"));
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt);
// post.add(new DefaultCharsetStringPart("process", "permission"));
// post.add(new DefaultCharsetStringPart("purpose", "crcon"));
// post.add(new DefaultCharsetStringPart("filename", filename));
// post.add(new DefaultCharsetStringPart("filesize", Long.toString(filesize)));
// post.add(new DefaultCharsetStringPart("can-send-protocol", "http"));

// send request
try {
final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 10000);
// final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 10000);
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt);
parts.put("process", new StringBody("permission"));
parts.put("purpose", new StringBody("crcon"));
parts.put("filename", new StringBody(filename));
parts.put("filesize", new StringBody(Long.toString(filesize)));
parts.put("can-send-protocol", new StringBody("http"));
final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, parts, 10000);
final Map<String, String> result = FileUtils.table(content);
return result;
} catch (final Exception e) {
Expand All @@ -729,17 +762,25 @@ public static Map<String, String> transferStore(final String targetAddress, fina

// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt);
post.add(new DefaultCharsetStringPart("process", "store"));
post.add(new DefaultCharsetStringPart("purpose", "crcon"));
post.add(new DefaultCharsetStringPart("filesize", Long.toString(file.length)));
post.add(new DefaultCharsetStringPart("md5", Digest.encodeMD5Hex(file)));
post.add(new DefaultCharsetStringPart("access", access));
post.add(new DefaultCharsetFilePart("filename", new ByteArrayPartSource(filename, file)));
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt);
// post.add(new DefaultCharsetStringPart("process", "store"));
// post.add(new DefaultCharsetStringPart("purpose", "crcon"));
// post.add(new DefaultCharsetStringPart("filesize", Long.toString(file.length)));
// post.add(new DefaultCharsetStringPart("md5", Digest.encodeMD5Hex(file)));
// post.add(new DefaultCharsetStringPart("access", access));
// post.add(new DefaultCharsetFilePart("filename", new ByteArrayPartSource(filename, file)));

// send request
try {
final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 20000);
// final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 20000);
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt);
parts.put("process", new StringBody("store"));
parts.put("purpose", new StringBody("crcon"));
parts.put("filesize", new StringBody(Long.toString(file.length)));
parts.put("md5", new StringBody(Digest.encodeMD5Hex(file)));
parts.put("access", new StringBody(access));
parts.put("filename", new ByteArrayBody(file, filename));
final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, parts, 20000);
final Map<String, String> result = FileUtils.table(content);
return result;
} catch (final Exception e) {
Expand Down Expand Up @@ -800,21 +841,31 @@ public static Map<String, String> crawlReceipt(final yacySeed mySeed, final yacy

// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
post.add(new DefaultCharsetStringPart("process", process));
post.add(new DefaultCharsetStringPart("urlhash", ((entry == null) ? "" : new String(entry.hash()))));
post.add(new DefaultCharsetStringPart("result", result));
post.add(new DefaultCharsetStringPart("reason", reason));
post.add(new DefaultCharsetStringPart("wordh", wordhashes));
post.add(new DefaultCharsetStringPart("lurlEntry", ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt))));
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt);
// post.add(new DefaultCharsetStringPart("process", process));
// post.add(new DefaultCharsetStringPart("urlhash", ((entry == null) ? "" : new String(entry.hash()))));
// post.add(new DefaultCharsetStringPart("result", result));
// post.add(new DefaultCharsetStringPart("reason", reason));
// post.add(new DefaultCharsetStringPart("wordh", wordhashes));
// post.add(new DefaultCharsetStringPart("lurlEntry", ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt))));

// determining target address
final String address = target.getClusterAddress();
if (address == null) { return null; }

// send request
try {
final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", post, 10000);
// final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", post, 10000);
// prepare request
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("process", new StringBody(process));
parts.put("urlhash", new StringBody(((entry == null) ? "" : new String(entry.hash()))));
parts.put("result", new StringBody(result));
parts.put("reason", new StringBody(reason));
parts.put("wordh", new StringBody(wordhashes));
parts.put("lurlEntry", new StringBody(((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt))));
// send request
final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", parts, 10000);
return FileUtils.table(content);
} catch (final Exception e) {
// most probably a network time-out exception
Expand Down Expand Up @@ -1028,12 +1079,14 @@ public static Map<String, String> getProfile(final yacySeed targetSeed) {

// this post a message to the remote message board
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), targetSeed.hash, salt);
// final List<Part> post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), targetSeed.hash, salt);

String address = targetSeed.getClusterAddress();
if (address == null) { address = "localhost:8080"; }
try {
final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", post, 5000);
// final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", post, 5000);
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", parts, 5000);
return FileUtils.table(content);
} catch (final Exception e) {
yacyCore.log.logSevere("yacyClient.getProfile error:" + e.getMessage());
Expand Down

0 comments on commit e1316d1

Please sign in to comment.