- fixed "yacy2yacy no proxy"-problem

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5058 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/HTTPLoader.java

@@ -151,7 +151,7 @@ private plasmaHTCache.Entry load(final CrawlEntry entry, final String parserMode
             requestHeader.put(httpHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING));
 
             // HTTP-Client
-            final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null);
+            final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader);
 
             JakartaCommonsHttpResponse res = null;
             try {

source/de/anomic/crawler/RobotsTxt.java

@@ -514,7 +514,7 @@ private static Object[] downloadRobotsTxt(final yacyURL robotsURL, int redirecti
 
         // setup http-client
         //TODO: adding Traffic statistic for robots download?
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders);
         JakartaCommonsHttpResponse res = null;
         try {
             // sending the get request

source/de/anomic/data/SitemapParser.java

@@ -154,7 +154,7 @@ public void parse() {
         // download document
         final httpHeader header = new httpHeader();
         header.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, header, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, header);
         JakartaCommonsHttpResponse res = null;
         try {
             res = client.GET(siteMapURL.toString());

source/de/anomic/http/HttpClient.java

@@ -94,7 +94,7 @@ public static byte[] wget(final String uri, final httpHeader header, final int t
     public static byte[] wget(final String uri, final httpHeader header, final int timeout, final String vhost) {
         assert uri != null : "precondition violated: uri != null";
         addHostHeader(header, vhost);
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header);
 
         // do the request
         try {
@@ -141,7 +141,7 @@ public static httpHeader whead(final String uri) {
      * @return null on error
      */
     public static httpHeader whead(final String uri, final httpHeader header) {
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header);
         JakartaCommonsHttpResponse response = null;
         try {
             response = client.HEAD(uri);

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -32,6 +32,7 @@
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.commons.httpclient.ConnectMethod;
@@ -139,14 +140,39 @@ public class JakartaCommonsHttpClient {
 
     private Header[] headers = new Header[0];
     private httpRemoteProxyConfig proxyConfig = null;
+    private boolean useGlobalProxyConfig = true;
     private boolean followRedirects = true;
     private boolean ignoreCookies = false;
 
+
+    /**
+     * creates a new JakartaCommonsHttpClient with given timeout using global remoteProxyConfig
+     *
+     * @param timeout in milliseconds
+     */
+    public JakartaCommonsHttpClient(final int timeout) {
+        this(timeout, null);
+    }
+
     /**
-     * constructs a new Client with given parameters
+     * creates a new JakartaCommonsHttpClient with given timeout and requestHeader using global remoteProxyConfig
+     *
+     * @param timeout in milliseconds
+     * @param header header options to send
+     */
+    public JakartaCommonsHttpClient(final int timeout, final httpHeader header) {
+        super();
+        setTimeout(timeout);
+        setHeader(header);
+    }
+
+    /**
+     * creates a new JakartaCommonsHttpClient with given timeout and requestHeader using given remoteProxyConfig
+     * 
+     * if proxyConfig is null, then no proxy is used
      * 
      * @param timeout in milliseconds
-     * @param header
+     * @param header header options to send
      * @param proxyConfig
      */
     public JakartaCommonsHttpClient(final int timeout, final httpHeader header, final httpRemoteProxyConfig proxyConfig) {
@@ -161,9 +187,8 @@ public JakartaCommonsHttpClient(final int timeout, final httpHeader header, fina
      * @see de.anomic.http.HttpClient#setProxy(de.anomic.http.httpRemoteProxyConfig)
      */
     public void setProxy(final httpRemoteProxyConfig proxyConfig) {
-        if (proxyConfig != null && proxyConfig.useProxy()) {
-            this.proxyConfig = proxyConfig;
-        }
+        this.useGlobalProxyConfig = false;
+        this.proxyConfig = proxyConfig;
     }
 
     /*
@@ -381,8 +406,8 @@ private static Header[] convertHeaders(final httpHeader requestHeader) {
         } else {
             headers = new Header[requestHeader.size()];
             int i = 0;
-            for (final String name : requestHeader.keySet()) {
-                headers[i] = new Header(name, requestHeader.get(name));
+            for (final Entry<String, String> header : requestHeader.entrySet()) {
+                headers[i] = new Header(header.getKey(), header.getValue());
                 i++;
             }
         }
@@ -520,13 +545,17 @@ private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig h
     /**
      * 
      * @param hostname
-     * @return
+     * @return null if no proxy should be used
      */
     private httpRemoteProxyConfig getProxyConfig(final String hostname) {
         final httpRemoteProxyConfig hostProxyConfig;
-        if (proxyConfig != null) {
+        if (!useGlobalProxyConfig) {
             // client specific
-            hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null;
+            if(proxyConfig == null) {
+                hostProxyConfig = null;
+            } else {
+                hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null;
+            }
         } else {
             // default settings
             hostProxyConfig = httpRemoteProxyConfig.getProxyConfigForHost(hostname);
@@ -603,7 +632,7 @@ public static void main(final String[] args) {
                 files.add(new FilePart("anotherfile.raw", new ByteArrayPartSource("anotherfile.raw",
                         "this is not a binary file ;)".getBytes())));
                 System.out.println("POST " + files.size() + " elements to " + url);
-                final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null);
+                final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000);
                 resp = client.POST(url, files);
                 System.out.println("----- Header: -----");
                 System.out.println(resp.getResponseHeader().toString());

source/de/anomic/http/httpRemoteProxyConfig.java

@@ -70,7 +70,7 @@ public static synchronized void setRemoteProxyConfig(final httpRemoteProxyConfig
     /**
      * @return the remoteProxyConfig
      */
-    public static httpRemoteProxyConfig getRemoteProxyConfig() {
+    public static synchronized httpRemoteProxyConfig getRemoteProxyConfig() {
         return remoteProxyConfig;
     }
 

source/de/anomic/http/httpd.java

@@ -45,6 +45,7 @@
 import java.util.List;
 import java.util.Properties;
 import java.util.StringTokenizer;
+import java.util.Map.Entry;
 
 import org.apache.commons.fileupload.FileItem;
 import org.apache.commons.fileupload.FileItemFactory;
@@ -89,7 +90,7 @@ public final class httpd implements serverHandler, Cloneable {
 
     public static final int ERRORCASE_MESSAGE = 4;
     public static final int ERRORCASE_FILE = 5;
-    public static httpdAlternativeDomainNames alternativeResolver = null;
+    private static httpdAlternativeDomainNames alternativeResolver = null;
 
     /**
      * A hashset containing extensions that indicate content that should not be transported
@@ -349,8 +350,8 @@ private boolean handleYaCyHopAuthentication(final httpHeader header) {
         // user = addressed peer name
         // pw = addressed peer hash (b64-hash)
         final String auth = (String) header.get(httpHeader.PROXY_AUTHORIZATION,"xxxxxx");
-        if (alternativeResolver != null) {
-            final String test = kelondroBase64Order.standardCoder.encodeString(alternativeResolver.myName() + ":" + alternativeResolver.myID());
+        if (getAlternativeResolver() != null) {
+            final String test = kelondroBase64Order.standardCoder.encodeString(getAlternativeResolver().myName() + ":" + getAlternativeResolver().myID());
             if (!test.equals(auth.trim().substring(6))) return false;
         }
 
@@ -1311,15 +1312,15 @@ else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Intege
             }
 
             // if peer has public address it will be used
-            if (alternativeResolver != null) {
-                tp.put("extAddress", alternativeResolver.myIP() + ":" + alternativeResolver.myPort());
+            if (getAlternativeResolver() != null) {
+                tp.put("extAddress", getAlternativeResolver().myIP() + ":" + getAlternativeResolver().myPort());
             }
             // otherwise the local ip address will be used
             else {
                 tp.put("extAddress", tp.get("host", "127.0.0.1") + ":" + tp.get("port", "8080"));
             }
 
-            tp.put("peerName", (alternativeResolver == null) ? "" : alternativeResolver.myName());
+            tp.put("peerName", (getAlternativeResolver() == null) ? "" : getAlternativeResolver().myName());
             tp.put("errorMessageType", errorcase);
             tp.put("httpStatus",       Integer.toString(httpStatusCode) + " " + httpStatusText);
             tp.put("requestMethod",    conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD));
@@ -1330,10 +1331,8 @@ else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Intege
                     tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString());
                     if ((detailedErrorMsgValues != null) && (detailedErrorMsgValues.size() > 0)) {
                         // rewriting the value-names and add the proper name prefix:
-                        final Iterator<String> nameIter = detailedErrorMsgValues.keySet().iterator();
-                        while (nameIter.hasNext()) {
-                            final String name = nameIter.next();
-                            tp.put("errorMessageType_" + name, detailedErrorMsgValues.get(name));
+                        for(Entry<String, String> entry: detailedErrorMsgValues.entrySet()) {
+                            tp.put("errorMessageType_" + entry.getKey(), entry.getValue());
                         }                        
                     }                    
                     break;
@@ -1385,11 +1384,9 @@ else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Intege
                 serverFileUtils.copy(result, respond);
             }
             respond.flush();
-        } catch (final Exception e) { 
-            throw new IOException(e.getMessage());
         } finally {
-            if (fis != null) try { fis.close(); } catch (final Exception e) {}
-            if (o != null)   try { o.close();   } catch (final Exception e) {}
+            if (fis != null) try { fis.close(); } catch (final Exception e) { e.printStackTrace(); }
+            if (o != null)   try { o.close();   } catch (final Exception e) { e.printStackTrace(); }
         }     
     }
 
@@ -1622,10 +1619,10 @@ public static boolean isThisSeedIP(final String hostName) {
         if ((hostName == null) || (hostName.length() == 0)) return false;
 
         // getting ip address and port of this seed
-        if (alternativeResolver == null) return false;
+        if (getAlternativeResolver() == null) return false;
 
         // resolve ip addresses
-        final InetAddress seedInetAddress = serverDomains.dnsResolve(alternativeResolver.myIP());
+        final InetAddress seedInetAddress = serverDomains.dnsResolve(getAlternativeResolver().myIP());
         final InetAddress hostInetAddress = serverDomains.dnsResolve(hostName);
         if (seedInetAddress == null || hostInetAddress == null) return false;
 
@@ -1682,7 +1679,7 @@ public static boolean isThisHostName(final String hostName) {
             final Integer dstPort = (idx != -1) ? Integer.valueOf(hostName.substring(idx+1).trim()) : Integer.valueOf(80);
 
             // if the hostname endswith thisPeerName.yacy ...
-            final String alternativeAddress = (alternativeResolver == null) ? null : alternativeResolver.myAlternativeAddress();
+            final String alternativeAddress = (getAlternativeResolver() == null) ? null : getAlternativeResolver().myAlternativeAddress();
             if ((alternativeAddress != null) && (dstHost.endsWith(alternativeAddress))) {
                 return true;
             /* 
@@ -1703,5 +1700,19 @@ public static boolean isThisHostName(final String hostName) {
             }
         } catch (final Exception e) {}    
         return false;
+    }
+
+    /**
+     * @param alternativeResolver the alternativeResolver to set
+     */
+    public static void setAlternativeResolver(httpdAlternativeDomainNames alternativeResolver) {
+        httpd.alternativeResolver = alternativeResolver;
+    }
+
+    /**
+     * @return the alternativeResolver
+     */
+    static httpdAlternativeDomainNames getAlternativeResolver() {
+        return alternativeResolver;
     }       
 }

source/de/anomic/http/httpdProxyHandler.java

@@ -1068,7 +1068,7 @@ public static void doPost(final Properties conProp, final httpHeader requestHead
      * @return
      */
     private static String resolveYacyDomains(final String host) {
-        return (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
+        return (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().resolve(host);
     }
 
     /**
@@ -1121,7 +1121,7 @@ private static String domain(final String host) {
      */
     private static JakartaCommonsHttpClient setupHttpClient(final httpHeader requestHeader, final String connectHost) {
         // setup HTTP-client
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader);
         client.setFollowRedirects(false);
         // cookies are handled by the user's browser
         client.setIgnoreCookies(true);
@@ -1244,7 +1244,7 @@ private static void removeHopByHopHeaders(final httpHeader headers) {
 
     private static void setViaHeader(final httpHeader header, final String httpVer) {
         if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return;
-        final String myAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.myAlternativeAddress();
+        final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress();
         if (myAddress != null) {
 
             // getting header set by other proxies in the chain

source/de/anomic/index/indexRepositoryReference.java

@@ -248,7 +248,7 @@ public void deadlinkCleaner(final httpRemoteProxyConfig proxyConfig) {
                         final yacyURL newUrl = new yacyURL(newUrlStr, null);
 
                         // doing a http head request to test if the url is correct
-                        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, null, null);
+                        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000);
                         client.setProxy(proxyConfig);
                         JakartaCommonsHttpResponse res = null;
                         try {

source/de/anomic/plasma/plasmaParser.java

@@ -888,7 +888,7 @@ public static void main(final String[] args) {
                 contentURL = new yacyURL(args[1], null);
 
                 // downloading the document content
-                final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null);
+                final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000);
 
                 res = client.GET(args[1]);
                 if (res.getStatusCode() != 200) {

source/de/anomic/server/serverSystem.java

@@ -46,11 +46,11 @@ public final class serverSystem {
 	public static final String blankTypeString = "____";
 
 	// system-identification statics
-	public static int     systemOS = systemUnknown;
-	public static boolean isMacArchitecture = false;
-	public static boolean isUnixFS = false;
-	public static boolean canExecUnix = false;
-	public static boolean isWindows = false;
+	public static final int     systemOS;
+	public static final boolean isMacArchitecture;
+	public static final boolean isUnixFS;
+	public static final boolean canExecUnix;
+	public static final boolean isWindows;
 
 	// calculated system constants
 	public static int maxPathLength = 65535;
@@ -317,6 +317,7 @@ public static void openBrowser(final String url, final String app) {
 				}
 			}
 		} catch (final Exception e) {
+		    System.err.println("ERROR "+ e.getClass() +" in openBrowser(): "+ e.getMessage());
 			System.out.println("please start your browser and open the following location: " + url);
 		}
 	}

source/de/anomic/yacy/yacyClient.java

@@ -270,7 +270,7 @@ private static byte[] wput(final String url, final String vhost, final List<Part
         final httpHeader header = new httpHeader();
         header.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
         header.put(httpHeader.HOST, vhost);
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header);
         client.setProxy(proxyConfig());
 
         JakartaCommonsHttpResponse res = null;

source/de/anomic/yacy/yacySeedDB.java

@@ -127,7 +127,7 @@ public yacySeedDB(
         lastSeedUpload_seedDBSize = sizeConnected();
 
         // tell the httpdProxy how to find this table as address resolver
-        httpd.alternativeResolver = this;
+        httpd.setAlternativeResolver(this);
     }
 
     private synchronized void initMySeed() {
@@ -850,7 +850,7 @@ private ArrayList<String> downloadSeedFile(final yacyURL seedURL) throws IOExcep
         reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
 
         // init http-client
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader);
         byte[] content = null;
         JakartaCommonsHttpResponse res = null;
         try {

source/de/anomic/yacy/yacyVersion.java

@@ -340,7 +340,7 @@ public static File downloadRelease(final yacyVersion release) {
         File download = null;
         final httpHeader header = new httpHeader();
         header.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
-        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(120000, header, null);
+        final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(120000, header);
         JakartaCommonsHttpResponse res = null;
         final String name = release.url.getFileName();
         try {

source/yacy.java

@@ -568,7 +568,7 @@ static void shutdown(final File homePath) {
         // send 'wget' to web interface
         final httpHeader requestHeader = new httpHeader();
         requestHeader.put(httpHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
-        final JakartaCommonsHttpClient con = new JakartaCommonsHttpClient(10000, requestHeader, null);
+        final JakartaCommonsHttpClient con = new JakartaCommonsHttpClient(10000, requestHeader);
         JakartaCommonsHttpResponse res = null;
         try {
             res = con.GET("http://localhost:"+ port +"/Steering.html?shutdown=");