- faster search: using different data structures that avoid multiplr …

…calculations - no more table copy for error-eco table - optional table copy for lurl-entries - more abstractions (less single constant strings) - better logging (using host names instead of ips) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4459 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Feb 7, 2008 · bd63999 · bd63999
1 parent 8358652
commit bd63999
Show file tree

Hide file tree

Showing 28 changed files with 113 additions and 89 deletions.
diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java
@@ -102,7 +102,7 @@ else if(post.containsKey("login")){
         }
 
         String pagename = post.get("page", "blog_default");
-        String ip = post.get("CLIENTIP", "127.0.0.1");
+        String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
 
         String StrAuthor = post.get("author", "anonymous");
 

diff --git a/htroot/CrawlProfileEditor_p.java b/htroot/CrawlProfileEditor_p.java
@@ -105,9 +105,9 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
         while (it.hasNext()) {
             selentry = (entry)it.next();
             if (selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_PROXY) ||
-                    selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_REMOTE) ||
+                    selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_REMOTE) /*||
                     selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_TEXT) ||
-                    selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_MEDIA))
+                    selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_MEDIA)*/)
                 continue;
             prop.put("profiles_" + count + "_name", selentry.name());
             prop.put("profiles_" + count + "_handle", selentry.handle());

diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java
@@ -212,7 +212,7 @@ else if (page.equals("crawler")) {
         }
 
         // clientIP
-        prop.putHTML("clientIP", (String) header.get("CLIENTIP", "<unknown>"), true); // read an artificial header addendum
+        prop.putHTML("clientIP", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"), true); // read an artificial header addendum
 
         /* 
          * seed upload settings

diff --git a/htroot/TestApplet.java b/htroot/TestApplet.java
@@ -50,7 +50,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
     //File templatefile=filehandler.getOverlayedFile((String)post.get("url"));
     File classfile = httpdFileHandler.getOverlayedClass((String)post.get("url"));
     httpHeader header2=new httpHeader();
-    header2.put("CLIENTIP", "127.0.0.1");
+    header2.put(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
     header2.put("PATH", post.get("url"));
     serverObjects tp=null;
     try {

diff --git a/htroot/User.java b/htroot/User.java
@@ -79,7 +79,7 @@ public static servletProperties respond(httpHeader header, serverObjects post, s
             prop.put("logged-in_identified-by", "2");
             //try via ip
             if(entry == null){
-                entry=sb.userDB.ipAuth(((String)header.get("CLIENTIP", "xxxxxx")));
+                entry=sb.userDB.ipAuth(((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")));
                 if(entry != null){
                     prop.put("logged-in_identified-by", "0");
                 }
@@ -108,7 +108,7 @@ public static servletProperties respond(httpHeader header, serverObjects post, s
         //identified via form-login
         //TODO: this does not work for a static admin, yet.
         }else if(post != null && post.containsKey("username") && post.containsKey("password")){
-            //entry=sb.userDB.passwordAuth((String)post.get("username"), (String)post.get("password"), (String)header.get("CLIENTIP", "xxxxxx"));
+            //entry=sb.userDB.passwordAuth((String)post.get("username"), (String)post.get("password"), (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx"));
             String username=(String)post.get("username");
             String password=(String)post.get("password");
 
@@ -163,7 +163,7 @@ else if(staticAdmin)
         if(post!=null && post.containsKey("logout")){
             prop.put("logged-in", "0");
             if(entry != null){
-                entry.logout(((String)header.get("CLIENTIP", "xxxxxx")), userDB.getLoginToken(header.getHeaderCookies())); //todo: logout cookie
+                entry.logout(((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")), userDB.getLoginToken(header.getHeaderCookies())); //todo: logout cookie
             }else{
                 sb.userDB.adminLogout(userDB.getLoginToken(header.getHeaderCookies()));
             }

diff --git a/htroot/ViewImage.java b/htroot/ViewImage.java
@@ -72,7 +72,7 @@ public static Image respond(httpHeader header, serverObjects post, serverSwitch
 
         String urlString = post.get("url", "");
         String urlLicense = post.get("code", "");
-        boolean auth = ((String) header.get("CLIENTIP", "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
+        boolean auth = ((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
 
         yacyURL url = null;
         if ((urlString.length() > 0) && (auth)) try {

diff --git a/htroot/Wiki.java b/htroot/Wiki.java
@@ -88,7 +88,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 
         String access = switchboard.getConfig("WikiAccess", "admin");
         String pagename = post.get("page", "start");
-        String ip = post.get("CLIENTIP", "127.0.0.1");
+        String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
         String author = post.get("author", "anonymous");
         if (author.equals("anonymous")) {
             author = wikiBoard.guessAuthor(ip);

diff --git a/htroot/www/welcome.java b/htroot/www/welcome.java
@@ -78,7 +78,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
             prop.put("hostip", "Unknown Host Exception");
         }       
         prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
-        prop.put("clientip", (String) header.get("CLIENTIP", ""));
+        prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
 
         final String peertype = (yacyCore.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
         final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));

diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java
@@ -103,7 +103,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 //      if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
 
         // we easily know the caller's IP:
-        final String clientip = (String) header.get("CLIENTIP", "<unknown>"); // read an artificial header addendum
+        final String clientip = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
         InetAddress ias = serverDomains.dnsResolve(clientip);
         if (ias == null) {
             prop.put("message", "cannot resolve your IP from your reported location " + clientip);

diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java
@@ -282,7 +282,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
         // prepare search statistics
         Long trackerHandle = new Long(System.currentTimeMillis());
         HashMap<String, Object> searchProfile = theQuery.resultProfile(joincount, System.currentTimeMillis() - timestamp, urlRetrievalAllTime, snippetComputationAllTime);
-        String client = (String) header.get("CLIENTIP");
+        String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
         searchProfile.put("host", client);
         yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
         searchProfile.put("peername", (remotepeer == null) ? "unknown" : remotepeer.getName());

diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java
@@ -89,14 +89,14 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
         final yacySeed opeer = yacyCore.seedDB.get(ohash);
         if (opeer == null) {
             // reject unknown peers: this does not appear fair, but anonymous senders are dangerous
-            sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get("CLIENTIP", "unknown"));
+            sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
             return prop;
         }
         opeer.setLastSeenUTC();
 
         if (filename.indexOf("..") >= 0) {
             // reject paths that contain '..' because they are dangerous
-            sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + opeer.getName() + "/" + opeer.getPublicAddress()+ ", current IP " + header.get("CLIENTIP", "unknown"));
+            sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + opeer.getName() + "/" + opeer.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
             return prop;
         }
 

diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java
@@ -257,7 +257,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
                     constraint,
                     true);
 
-            String client = (String) header.get("CLIENTIP"); // the search client who initiated the search
+            String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
 
             // tell all threads to do nothing for a specific time
             sb.intermissionAllThreads(10000);

diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java
@@ -155,7 +155,7 @@ public Entry proxyAuth(String auth) {
 		return null;
 	}
         public Entry getUser(httpHeader header){
-            return getUser((String) header.get(httpHeader.AUTHORIZATION), (String)header.get("CLIENTIP"), header.getHeaderCookies());
+            return getUser((String) header.get(httpHeader.AUTHORIZATION), (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP), header.getHeaderCookies());
         }
         public Entry getUser(String auth, String ip, String cookies){
         Entry entry=null;

diff --git a/source/de/anomic/http/httpSSI.java b/source/de/anomic/http/httpSSI.java
@@ -85,7 +85,7 @@ private static void writeContent(String path, OutputStream out, String authoriza
         conProp.setProperty(httpHeader.CONNECTION_PROP_PATH, path);
         conProp.setProperty(httpHeader.CONNECTION_PROP_ARGS, args);
         conProp.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
-        conProp.setProperty("CLIENTIP", "127.0.0.1");
+        conProp.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
         header.put(httpHeader.AUTHORIZATION, authorization);
         httpdFileHandler.doGet(conProp, header, out);
     }

diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java
@@ -193,7 +193,7 @@ public void reset()  {
     public void initSession(serverCore.Session newsession) throws IOException {
         this.session = newsession;
         this.userAddress = session.userAddress; // client InetAddress
-        this.clientIP = this.userAddress.getHostAddress();
+        this.clientIP = this.userAddress.getHostName();
         if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost";
         if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost";
         if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost";
@@ -1147,7 +1147,7 @@ else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Intege
 //            tp.put("host", serverCore.publicIP().getHostAddress());
 //            tp.put("port", switchboard.getConfig("port", "8080"));
 
-            String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP,"127.0.0.1");
+            String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
 
             // check if ip is local ip address
             InetAddress hostAddress = serverDomains.dnsResolve(clientIP);

diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java
@@ -303,13 +303,13 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
             if ((path.substring(0,(pos==-1)?path.length():pos)).endsWith("_p") && (adminAccountBase64MD5.length() != 0)) {
                 //authentication required
                 //userDB
-                if(sb.userDB.hasAdminRight(authorization, conProp.getProperty("CLIENTIP"), requestHeader.getHeaderCookies())){
+                if(sb.userDB.hasAdminRight(authorization, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP), requestHeader.getHeaderCookies())){
                     //Authentication successful. remove brute-force flag
-                    serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
+                    serverCore.bfHost.remove(conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
                 //static
                 }else if(authorization != null && httpd.staticAdminAuthenticated(authorization.trim().substring(6), switchboard)==4){
                     //Authentication successful. remove brute-force flag
-                    serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
+                    serverCore.bfHost.remove(conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
                 //no auth
                 }else if (authorization == null) {
                     // no authorization given in response. Ask for that
@@ -323,7 +323,7 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
                     return;
                 } else {
                     // a wrong authentication was given or the userDB user does not have admin access. Ask again
-                    String clientIP = conProp.getProperty("CLIENTIP", "unknown-host");
+                    String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown-host");
                     serverLog.logInfo("HTTPD", "Wrong log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
                     Integer attempts = (Integer) serverCore.bfHost.get(clientIP);
                     if (attempts == null)
@@ -473,7 +473,7 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
                 // call an image-servlet to produce an on-the-fly - generated image
                 Object img = null;
                 try {
-                    requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
+                    requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
                     requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
                     // in case that there are no args given, args = null or empty hashmap
                     img = invokeServlet(targetClass, requestHeader, args);
@@ -527,7 +527,7 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
                 }
             } else if ((targetClass != null) && (path.endsWith(".stream"))) {
                 // call rewrite-class
-                requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
+                requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
                 requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
                 //requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
                 //requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);
@@ -570,7 +570,7 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
                     } else {
                         // CGI-class: call the class to create a property for rewriting
                         try {
-                            requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
+                            requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
                             requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
                             // in case that there are no args given, args = null or empty hashmap
                             Object tmp = invokeServlet(targetClass, requestHeader, args);
@@ -586,7 +586,7 @@ public static void doResponse(Properties conProp, httpHeader requestHeader, Outp
                             if (tp.containsKey(servletProperties.ACTION_AUTHENTICATE)) {
                                 // handle brute-force protection
                                 if (authorization != null) {
-                                    String clientIP = conProp.getProperty("CLIENTIP", "unknown-host");
+                                    String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown-host");
                                     serverLog.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
                                     Integer attempts = (Integer) serverCore.bfHost.get(clientIP);
                                     if (attempts == null)

diff --git a/source/de/anomic/index/indexRWIEntry.java b/source/de/anomic/index/indexRWIEntry.java
@@ -47,8 +47,6 @@ public interface indexRWIEntry {
 
     public String urlHash();
 
-    public int quality();
-
     public int virtualAge();
 
     public long lastModified();