joined anomic.net.URL, plasmaURL and url hash computation:

search profiling showed, that a major amount of time is wasted by computing url hashes. The computation does an intranet-check, which needs a DNS lookup. This caused that each urlhash computation needed 100-200 milliseconds, which caused remote searches to delay at least 1 second more that necessary. The solution to this problem is to attach a URL hash to the URL data structure, because that means that the url hash value can be filled after retrieval of the URL from the database. The redesign of the url/urlhash management caused a major redesign of many parts of the software. Since some parts had been decided to be given up they had been removed during this change to avoid unnecessary maintenance of unused code. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4074 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Sep 5, 2007 · daf0f74 · daf0f74
1 parent 66905b7
commit daf0f74
Show file tree

Hide file tree

Showing 160 changed files with 1,481 additions and 10,006 deletions.
diff --git a/build.xml b/build.xml
@@ -221,7 +221,7 @@
 
   <!-- compiling the main sources -->   
   <javac srcdir="${src}/" destdir="${build}"
-         excludes="de/anomic/plasma/parser/*/*,de/anomic/yacy/seedUpload/**,de/anomic/soap/**,yacy.java,de/anomic/server/portForwarding/*/*,de/anomic/data/rssReader.java"
+         excludes="de/anomic/plasma/parser/*/*,de/anomic/yacy/seedUpload/**,yacy.java,de/anomic/server/portForwarding/*/*,de/anomic/data/rssReader.java"
          debug="true" debuglevel="lines,vars,source" 
          source="${javacSource}" target="${javacTarget}">
     <classpath refid="project.class.path"/>
@@ -241,7 +241,6 @@
   <javac srcdir="${htroot}/" 
          classpathref="project.class.path"
          debug="true" debuglevel="lines,vars,source" 
-  		 excludes="soap/*"
          source="${javacSource}" target="${javacTarget}"/>
 </target>
 
@@ -297,33 +296,6 @@
     <fileset dir="${src}/" includes="de/anomic/yacy/seedUpload/yacySeedUpload*.xml"/>
   </subant>
 </target>   
-
-<!-- compiling optional soap API and building install packages -->   
-<target name="compileSoap" depends="compileMain" description="Compiling and zipping additional yacy SOAP API">
-  <subant target="${extensionTarget}">
-    <property name="src" location="${src}"/>
-    <property name="build" location="${build}"/>
-    <property name="libx" location="${libx}"/>
-    <property name="htroot" value="${htroot}"/>    
-    <property name="release" location="${release_ext}"/>
-    <property name="javacSource" value="${javacSource}"/>
-    <property name="javacTarget" value="${javacTarget}"/>
-    <fileset dir="${src}/" includes="de/anomic/soap/build.xml"/>
-  </subant>
-</target>       
-
-<target name="distSoapClientStubJar" depends="init" description="Generates a jar file with all client stub classes for the YaCy SOAP API">
-  <subant target="buildClientStubJar">
-    <property name="src" location="${src}"/>
-    <property name="build" location="${build}"/>
-    <property name="libx" location="${libx}"/>
-    <property name="htroot" value="${htroot}"/>    
-    <property name="release" location="${release_ext}"/>
-    <property name="javacSource" value="${javacSource}"/>
-    <property name="javacTarget" value="${javacTarget}"/>
-    <fileset dir="${src}/" includes="de/anomic/soap/build.xml"/>
-  </subant>
-</target>	
 
 <target name="compilePortForwarding" depends="compileMain" description="Compiling and zipping additional port forwarder"> 
     <javac srcdir="${src}/de/anomic/server/portForwarding" destdir="${build}" source="${javacSource}" target="${javacTarget}" debug="true" debuglevel="lines,vars,source">
@@ -359,7 +331,7 @@
 </target>   
 
 <!-- compile optional classs that were not compiled elsewhere -->
-<target name="compileExtensions" depends="compileMain,compileParsers,compileSeedUploaders,compileSoap,compilePortForwarding">
+<target name="compileExtensions" depends="compileMain,compileParsers,compileSeedUploaders,compilePortForwarding">
 
 	<!-- compile rss Reader -->
 	  <javac srcdir="${src}" destdir="${build}"
@@ -423,9 +395,7 @@
 		    <!-- excluding all additional content parsers --> 
 		    <exclude name="de/anomic/plasma/parser/*/*"/>
 		    <!-- excluding all additional seed uploaders --> 
-		    <exclude name="de/anomic/yacy/seedUpload/**"/> 
-		    <!-- excluding the soap handler --> 
-		    <exclude name="de/anomic/soap/**"/>
+		    <exclude name="de/anomic/yacy/seedUpload/**"/>
 		    <!-- excluding the port forwarder --> 
 		    <exclude name="de/anomic/server/portForwarding/*/*"/> 
 			<!-- ecluding rss Reader class -->
@@ -527,8 +497,6 @@
 	    <exclude name="de/anomic/plasma/parser/*/*"/>
 	  	<!-- excluding sources for additional seed uploaders -->
 	    <exclude name="de/anomic/yacy/seedUpload/yacySeedUpload**"/>
-	  	<!-- excluding soap -->
-	    <exclude name="de/anomic/soap/**"/> 
 	    <!-- excluding the port forwarder --> 
 	    <exclude name="de/anomic/server/portForwarding/*/*"/> 
 		<!-- ecluding rss Reader class -->
@@ -545,7 +513,6 @@
 	  <fileset dir="${htroot}">
 	    <include name="**/*"/>
 	    <exclude name="yacy/seedUpload/**"/>
-	  	<exclude name="soap/**"/>
 	  </fileset>
 	  <fileset dir="${htroot}">
 	      <include name="yacy/seedUpload/yacySeedUploadFile.html"/>

diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java
@@ -60,14 +60,14 @@
 
 import de.anomic.data.listManager;
 import de.anomic.http.httpHeader;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.plasma.urlPattern.abstractURLPattern;
 import de.anomic.plasma.urlPattern.plasmaURLPattern;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyURL;
 
 public class Blacklist_p {
 	private final static String DISABLED         = "disabled_";
@@ -95,9 +95,9 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
             	prop.put("testlist",1);
             	String urlstring = post.get("testurl", "");
             	if(!urlstring.startsWith("http://")) urlstring = "http://"+urlstring;
-            	URL testurl = null;
+                yacyURL testurl = null;
 				try {
-					testurl = new URL(urlstring);
+					testurl = new yacyURL(urlstring, null);
 				} catch (MalformedURLException e) { }
 				if(testurl != null) {
 					prop.put("testlist_url",testurl.toString());

diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java
@@ -59,7 +59,6 @@
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpc;
 import de.anomic.index.indexURLEntry;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaParserDocument;
 import de.anomic.plasma.plasmaSnippetCache;
 import de.anomic.plasma.plasmaSwitchboard;
@@ -69,6 +68,7 @@
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacyNewsPool;
 import de.anomic.yacy.yacyNewsRecord;
+import de.anomic.yacy.yacyURL;
 
 public class Bookmarks {
     public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
@@ -214,7 +214,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
             }
             try {
                 File file=new File((String)post.get("bookmarksfile"));
-                switchboard.bookmarksDB.importFromBookmarks(new URL(file) , new String((byte[])post.get("bookmarksfile$file")), tags, isPublic);
+                switchboard.bookmarksDB.importFromBookmarks(new yacyURL(file) , new String((byte[])post.get("bookmarksfile$file")), tags, isPublic);
             } catch (MalformedURLException e) {}
 
         }else if(post.containsKey("xmlfile")){

diff --git a/htroot/CacheAdmin_p.java b/htroot/CacheAdmin_p.java
@@ -62,7 +62,6 @@
 import de.anomic.htmlFilter.htmlFilterImageEntry;
 import de.anomic.htmlFilter.htmlFilterWriter;
 import de.anomic.http.httpHeader;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaHTCache;
 import de.anomic.plasma.plasmaParserDocument;
 import de.anomic.plasma.plasmaSwitchboard;
@@ -71,6 +70,7 @@
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyURL;
 
 public class CacheAdmin_p {
 
@@ -118,7 +118,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
         final StringBuffer tree = new StringBuffer();
         final StringBuffer info = new StringBuffer();
 
-        final URL  url  = plasmaHTCache.getURL(file);
+        final yacyURL  url  = plasmaHTCache.getURL(file);
 
         String urlstr = "";
 

diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java
@@ -58,11 +58,11 @@
 import de.anomic.data.translator;
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpc;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.tools.nxTools;
+import de.anomic.yacy.yacyURL;
 
 
 public class ConfigLanguage_p {
@@ -97,7 +97,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 			String url = (String)post.get("url");
 			ArrayList langVector;
 			try{
-                URL u = new URL(url);
+                yacyURL u = new yacyURL(url, null);
 				langVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig, null, null), "UTF-8");
 			}catch(IOException e){
 				prop.put("status", 1);//unable to get url

diff --git a/htroot/ConfigSkins_p.java b/htroot/ConfigSkins_p.java
@@ -56,12 +56,12 @@
 import de.anomic.data.listManager;
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpc;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.tools.nxTools;
+import de.anomic.yacy.yacyURL;
 
 public class ConfigSkins_p {
 
@@ -126,7 +126,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 			String url = (String)post.get("url");
 			ArrayList skinVector;
 			try{
-                URL u = new URL(url);
+                yacyURL u = new yacyURL(url, null);
 				skinVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig, null, null), "UTF-8");
 			}catch(IOException e){
 				prop.put("status", 1);//unable to get URL

diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java
@@ -31,11 +31,11 @@
 import java.util.TreeSet;
 
 import de.anomic.http.httpHeader;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.serverSystem;
+import de.anomic.yacy.yacyURL;
 import de.anomic.yacy.yacyVersion;
 
 public class ConfigUpdate_p {
@@ -54,7 +54,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
                 String release = post.get("releasedownload", "");
                 if (release.length() > 0) {
                     try {
-                        yacyVersion.downloadRelease(new yacyVersion(new URL(release)));
+                        yacyVersion.downloadRelease(new yacyVersion(new yacyURL(release, null)));
                     } catch (IOException e) {
                         // TODO Auto-generated catch block
                         e.printStackTrace();

diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java
@@ -30,7 +30,6 @@
 
 import de.anomic.http.httpHeader;
 import de.anomic.index.indexURLEntry;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaHTCache;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
@@ -39,6 +38,7 @@
 import de.anomic.tools.nxTools;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyURL;
 
 public class CrawlResults {
 
@@ -170,7 +170,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 
                     urlstr = comp.url().toNormalform(false, true);
                     urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL
-                    cachepath = plasmaHTCache.getCachePath(new URL(urlstr)).toString().replace('\\', '/').substring(plasmaHTCache.cachePath.toString().length() + 1);
+                    cachepath = plasmaHTCache.getCachePath(new yacyURL(urlstr, null)).toString().replace('\\', '/').substring(plasmaHTCache.cachePath.toString().length() + 1);
 
                     prop.put("table_indexed_" + cnt + "_dark", (dark) ? 1 : 0);
                     if (showControl) {

diff --git a/htroot/CrawlStartSimple_p.html b/htroot/CrawlStartSimple_p.html
@@ -53,7 +53,7 @@ <h2>Easy Crawl Start</h2>
           <td><label for="crawlingDepth">Crawling Range</label>:</td>
           <td>
           <input type="radio" name="range" value="wide" checked="checked" />Wide: depth <input name="crawlingDepth" id="crawlingDepth" type="text" size="2" maxlength="2" value="#[crawlingDepth]#" />&nbsp;&nbsp;|&nbsp;&nbsp;
-          <input type="radio" name="range" value="domain" />Complete Single Domain
+          <input type="radio" name="range" value="domain" />Complete Domain
           </td>
           <td>
             The range defines if the crawl shall consider a complete domain, or a wide crawl up to a specific depth.

diff --git a/htroot/CrawlStartSimple_p.java b/htroot/CrawlStartSimple_p.java
@@ -28,14 +28,14 @@
 import java.util.Iterator;
 
 import de.anomic.http.httpHeader;
-import de.anomic.plasma.plasmaURL;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacyNewsPool;
 import de.anomic.yacy.yacyNewsRecord;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyURL;
 
 public class CrawlStartSimple_p {
 
@@ -152,8 +152,8 @@ record = (yacyNewsRecord) recordIterator.next();
         if ((yacyCore.seedDB == null) || (yacyCore.seedDB.mySeed.isVirgin()) || (yacyCore.seedDB.mySeed.isJunior())) {
             prop.put("remoteCrawlPeers", 0);
         } else {
-            Enumeration crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(plasmaURL.dummyHash, true);
-            Enumeration crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(plasmaURL.dummyHash, false);
+            Enumeration crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, true);
+            Enumeration crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, false);
             if ((!(crawlavail.hasMoreElements())) && (!(crawlpendi.hasMoreElements()))) {
                 prop.put("remoteCrawlPeers", 0); //no peers availible
             } else {

diff --git a/htroot/CrawlURLFetchStack_p.java b/htroot/CrawlURLFetchStack_p.java
@@ -55,7 +55,6 @@
 import de.anomic.htmlFilter.htmlFilterContentScraper;
 import de.anomic.htmlFilter.htmlFilterWriter;
 import de.anomic.http.httpHeader;
-import de.anomic.net.URL;
 import de.anomic.plasma.plasmaCrawlEntry;
 import de.anomic.plasma.plasmaCrawlNURL;
 import de.anomic.plasma.plasmaSwitchboard;
@@ -64,6 +63,7 @@
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.logging.serverLog;
+import de.anomic.yacy.yacyURL;
 
 public class CrawlURLFetchStack_p {
 
@@ -200,16 +200,16 @@ else if (post.containsKey("subupload")) {
                         prop.put("upload", 1);
                     } else if (type.equals("html")) {
                         try {
-                            final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file));
+                            final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new yacyURL(file));
                             final Writer writer = new htmlFilterWriter(null, null, scraper, null, false);
                             serverFileUtils.write(content, writer);
                             writer.close();
 
                             final Iterator it = ((HashMap)scraper.getAnchors()).keySet().iterator();
                             int added = 0, failed = 0;
-                            URL url;
+                            yacyURL url;
                             while (it.hasNext()) try {
-                                url = new URL((String)it.next());
+                                url = new yacyURL((String) it.next(), null);
                                 if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url)) {
                                     failed++;
                                     continue;
@@ -264,7 +264,7 @@ private static int addURLs(String[] urls, boolean blCheck, URLFetcherStack stack
     private static boolean addURL(String url, boolean blCheck, URLFetcherStack stack) {
         try {
             if (url == null || url.length() == 0) return false;
-            URL u = new URL(url);
+            yacyURL u = new yacyURL(url, null);
             if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, u)) return false;
             stack.push(u);
             return true;
@@ -288,7 +288,7 @@ private static int addURLs(serverObjects post, int amount, URLFetcherStack stack
             url = post.get("url" + i, null);
             if (url == null || url.length() == 0) continue;
             try {
-                stack.push(new URL(url));
+                stack.push(new yacyURL(url, null));
                 count++;
             } catch (MalformedURLException e) {
                 serverLog.logInfo("URLFETCHER", "retrieved invalid url for adding to the stack: " + url);