Skip to content

Commit

Permalink
... migrating to HttpComponents-Client-4.x ...
Browse files Browse the repository at this point in the history 
monitoring and first try to use remoteProxy

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6979 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
sixcooler committed Jul 20, 2010
1 parent dec1419 commit 5fa8038
Show file tree
Hide file tree
Showing 6 changed files with 362 additions and 67 deletions.
7 changes: 4 additions & 3 deletions htroot/Connections_p.java
View file
Expand Up @@ -32,12 +32,13 @@
import java.net.URLEncoder;
import java.util.Set;

import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.workflow.WorkflowThread;

import de.anomic.http.client.ConnectionInfo;
import de.anomic.http.client.Client;
//import de.anomic.http.client.ConnectionInfo;
//import de.anomic.http.client.Client;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
Expand Down Expand Up @@ -167,7 +168,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
}
}
prop.put("clientList", c);
prop.put("clientActive", Client.connectionCount());
prop.put("clientActive", ConnectionInfo.getCount());

// return rewrite values for templates
return prop;
Expand Down
10 changes: 6 additions & 4 deletions source/de/anomic/search/Switchboard.java
View file
Expand Up @@ -72,6 +72,7 @@

import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.ProxySettings;
import net.yacy.document.Condenser;
import net.yacy.document.Document;
Expand Down Expand Up @@ -1485,6 +1486,7 @@ public boolean cleanupJob() {

// close unused connections
Client.cleanup();
ConnectionInfo.cleanUp();

// do transmission of CR-files
checkInterruption();
Expand Down Expand Up @@ -2125,8 +2127,8 @@ public boolean dhtTransferJob(final String segment) {
log.logInfo("dhtTransferJob: no selection, too many entries in transmission cloud: " + this.dhtDispatcher.cloudSize());
} else if (MemoryControl.available() < 1024*1024*25) {
log.logInfo("dhtTransferJob: no selection, too less memory available : " + (MemoryControl.available() / 1024 / 1024) + " MB");
} else if (net.yacy.cora.protocol.Client.connectionCount() > 5) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount());
} else if (ConnectionInfo.getLoadPercent() > 50) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else {
Expand Down Expand Up @@ -2157,8 +2159,8 @@ public boolean dhtTransferJob(final String segment) {
// check if we can deliver entries to other peers
if (this.dhtDispatcher.transmissionSize() >= 10) {
log.logInfo("dhtTransferJob: no dequeueing from cloud to transmission: too many concurrent sessions: " + this.dhtDispatcher.transmissionSize());
} else if (net.yacy.cora.protocol.Client.connectionCount() > 5) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount());
} else if (ConnectionInfo.getLoadPercent() > 75) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else {
Expand Down
70 changes: 35 additions & 35 deletions source/de/anomic/yacy/yacyNetwork.java
View file
Expand Up @@ -28,19 +28,19 @@
package de.anomic.yacy;

import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
//import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
//import java.util.List;

import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.util.DateFormatter;

import org.apache.commons.httpclient.methods.multipart.Part;
//import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;

import de.anomic.http.client.DefaultCharsetStringPart;
//import de.anomic.http.client.DefaultCharsetStringPart;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
Expand Down Expand Up @@ -77,37 +77,37 @@ public static final boolean authentifyRequest(final serverObjects post, final se
return false;
}

public static final List<Part> basicRequestPost(final Switchboard sb, final String targetHash, final String salt) {
// put in all the essentials for routing and network authentication
// generate a session key
final ArrayList<Part> post = new ArrayList<Part>();
post.add(new DefaultCharsetStringPart("key", salt));

// just standard identification essentials
post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash));
if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash));

// time information for synchronization
post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date())));
post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));

// network identification
post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));

// authentication essentials
final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) {
if (authenticationMethod.equals("salted-magic-sim")) {
// generate an authentication essential using the salt, the iam-hash and the network magic
final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
post.add(new DefaultCharsetStringPart("magicmd5", md5));
}
}

return post;
}
// public static final List<Part> basicRequestPost(final Switchboard sb, final String targetHash, final String salt) {
// // put in all the essentials for routing and network authentication
// // generate a session key
// final ArrayList<Part> post = new ArrayList<Part>();
// post.add(new DefaultCharsetStringPart("key", salt));
//
// // just standard identification essentials
// post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash));
// if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash));
//
// // time information for synchronization
// post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date())));
// post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));
//
// // network identification
// post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
//
// // authentication essentials
// final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
// final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
// if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) {
// if (authenticationMethod.equals("salted-magic-sim")) {
// // generate an authentication essential using the salt, the iam-hash and the network magic
// final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
// final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
// post.add(new DefaultCharsetStringPart("magicmd5", md5));
// }
// }
//
// return post;
// }

public static final LinkedHashMap<String,ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) throws UnsupportedEncodingException {
// put in all the essentials for routing and network authentication
Expand Down
67 changes: 42 additions & 25 deletions source/net/yacy/cora/protocol/Client.java
View file
Expand Up @@ -17,6 +17,7 @@
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
Expand All @@ -25,11 +26,10 @@
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
Expand All @@ -46,9 +46,9 @@
*/
public class Client {

private final static int maxcon = 20;
private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = null;
private static int count = 0;
private int timeout = 10000;
private String userAgent = null;
private String host = null;
Expand All @@ -67,12 +67,14 @@ private static void initConnectionManager() {
* ConnectionManager settings
*/
// TODO: how much connections do we need? - default: 20
// ConnManagerParams.setMaxTotalConnections(httpParams, 100);
ConnManagerParams.setMaxTotalConnections(httpParams, maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// perhaps we need more than 2(default) connections per host?
ConnPerRouteBean connPerRoute = new ConnPerRouteBean(2);
// Increase max connections for localhost to 100
HttpHost localhost = new HttpHost("locahost");
connPerRoute.setMaxForRoute(new HttpRoute(localhost), 100);
connPerRoute.setMaxForRoute(new HttpRoute(localhost), maxcon);
ConnManagerParams.setMaxConnectionsPerRoute(httpParams, connPerRoute);
// how long to wait for getting a connection from manager in milliseconds
ConnManagerParams.setTimeout(httpParams, 3000L);
Expand All @@ -88,7 +90,7 @@ private static void initConnectionManager() {
// timeout in milliseconds until a connection is established in milliseconds
HttpConnectionParams.setConnectionTimeout(httpParams, 10000);
// SO_LINGER affects the socket close operation in seconds
HttpConnectionParams.setLinger(httpParams, 6);
// HttpConnectionParams.setLinger(httpParams, 6);
// TODO: is default ok?
// HttpConnectionParams.setSocketBufferSize(httpParams, 8192);
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
Expand All @@ -97,6 +99,7 @@ private static void initConnectionManager() {
HttpConnectionParams.setStaleCheckingEnabled(httpParams, true);
// conserve bandwidth by minimizing the number of segments that are sent
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// TODO: testing noreuse - there will be HttpConnectionParams.setSoReuseaddr(HttpParams params, boolean reuseaddr) in core-4.1

// Create and initialize scheme registry
SchemeRegistry schemeRegistry = new SchemeRegistry();
Expand Down Expand Up @@ -151,15 +154,6 @@ public void setHost(final String host) {
this.host = host;
}

/**
* number of active connections
*
* @return number of active connections
*/
public static int connectionCount() {
return count;
}

/**
* This method GETs a page from the server.
*
Expand Down Expand Up @@ -204,16 +198,12 @@ public byte[] POSTbytes(final String uri, LinkedHashMap<String,ContentBody> part
}

private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException {
count++;
byte[] content = null;
final HttpContext httpContext = new BasicHttpContext();
httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout);
httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout);
if (userAgent != null)
httpUriRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
if (host != null)
httpUriRequest.getParams().setParameter(HTTP.TARGET_HOST, host);

setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
HttpResponse httpResponse = httpClient.execute(httpUriRequest, httpContext);
Expand All @@ -229,13 +219,40 @@ private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) thr
}
} catch (final IOException e) {
httpUriRequest.abort();
count--;
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
throw e;
}
count--;
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
return content;
}

private void setParams(HttpParams httpParams) {
HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null)
HttpProtocolParams.setUserAgent(httpParams, userAgent);
if (host != null)
httpParams.setParameter(HTTP.TARGET_HOST, host);
}

private void setProxy(HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
}

private void storeConnectionInfo(HttpUriRequest httpUriRequest) {
int port = httpUriRequest.getURI().getPort();
String thost = httpUriRequest.getURI().getHost();
ConnectionInfo.addConnection(new ConnectionInfo(
httpUriRequest.getURI().getScheme(),
port == 80 ? thost : thost + ":" + port,
httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
httpUriRequest.hashCode(),
System.currentTimeMillis()));
}

/**
* provide system information for client identification
*/
Expand Down

0 comments on commit 5fa8038

Please sign in to comment.