one step closer to automatic updates:

automatically aquire release information from download archives
web pages from latest.yacy-forum.net and yacy.net are retrieved, parsed,
links wihin are analysed, sorted and the most recent developer and main
releases are provided as direct download link on the status page, if it was
discovered that a more recent version than the current version is available.
This process is done only once during run-time of a peer, to protect our
download archives from DoS by YaCy peers.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3606 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Status.html

@@ -28,7 +28,12 @@ <h2>System-, Index- and Peer-Status</h2>
     <table class="StatusTable">
       <tr class="TableHeader"><td colspan="2"><strong>Public System Properties</strong></td></tr>
       <tr class="TableCellLight"><td>System version</td><td>
-      #[versionpp]# #(versioncomment)#:: - the latest public version is #[latestVersion]#. Click here to <a href="http://www.yacy.net/yacy/Download.html">download</a> it.#(/versioncomment)#
+      #[versionpp]#
+      #(versioncomment)#:: - the latest public version is #[latestVersion]#.
+      You can download the latest releases here:<br>
+      #[versionResMain]#<br>
+      #[versionResDev]#
+      #(/versioncomment)#
       </td></tr>
       <tr class="TableCellDark"><td>This&nbsp;peer's&nbsp;address</td><td>
       #(peerAddress)#

htroot/Status.java

@@ -61,6 +61,7 @@
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyVersion;
 
 public class Status {
 
@@ -150,16 +151,20 @@ else if (jobType.equals("globalCrawlTrigger"))
 
         // version information
         prop.put("versionpp", yacy.combined2prettyVersion(env.getConfig("version","0.1")));
+
+
         double thisVersion = Double.parseDouble(env.getConfig("version","0.1"));
         // cut off the SVN Rev in the Version
         try {thisVersion = Math.round(thisVersion*1000.0)/1000.0;} catch (NumberFormatException e) {}
-//      System.out.println("TEST: "+thisVersion);
-        if (yacyCore.latestVersion >= (thisVersion+0.01)) { // only new Versions(not new SVN)
+        if (yacyVersion.latestRelease >= (thisVersion+0.01)) { // only new Versions(not new SVN)
             prop.put("versioncomment", 1); // new version
         } else {
             prop.put("versioncomment", 0); // no comment
         }
-        prop.put("versioncomment_latestVersion", Double.toString(yacyCore.latestVersion));
+        yacyVersion.aquireLatestReleaseInfo();
+        prop.putASIS("versioncomment_versionResMain", (yacyVersion.latestMainRelease == null) ? "-" : yacyVersion.latestMainRelease.toAnchor());
+        prop.putASIS("versioncomment_versionResDev", (yacyVersion.latestDevRelease == null) ? "-" : yacyVersion.latestDevRelease.toAnchor());
+        prop.put("versioncomment_latestVersion", Double.toString(yacyVersion.latestRelease));
 
         // hostname and port
         String extendedPortString = env.getConfig("port", "8080");

source/de/anomic/htmlFilter/htmlFilterContentScraper.java

@@ -45,7 +45,10 @@
 
 package de.anomic.htmlFilter;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.io.Writer;
 import java.net.MalformedURLException;
 import java.text.Collator;
 import java.util.ArrayList;
@@ -58,8 +61,11 @@
 
 import javax.swing.event.EventListenerList;
 
+import de.anomic.http.httpc;
 import de.anomic.net.URL;
+import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverCharBuffer;
+import de.anomic.server.serverFileUtils;
 
 public class htmlFilterContentScraper extends htmlFilterAbstractScraper implements htmlFilterScraper {
 
@@ -328,6 +334,7 @@ public byte[] getText(String charSet) {
     }
 
     public Map getAnchors() {
+        // returns a url (String) / name (String) relation
         return anchors;
     }
 
@@ -449,5 +456,25 @@ void fireScrapeTag1(String tagname, Properties tagopts, char[] text) {
                     ((htmlFilterEventListener)listeners[i+1]).scrapeTag1(tagname, tagopts, text);
             }
         }
-    }      
+    }
+
+    public static htmlFilterContentScraper parseResource(URL location) throws IOException {
+        // load page
+        byte[] page = httpc.wget(
+                location,
+                location.getHost(),
+                10000, 
+                null, 
+                null, 
+                plasmaSwitchboard.getSwitchboard().remoteProxyConfig
+        );
+        if (page == null) throw new IOException("no response from url " + location.toString());
+
+        // scrape content
+        htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);
+        Writer writer = new htmlFilterWriter(null, null, scraper, null, false);
+        serverFileUtils.copy(new ByteArrayInputStream(page), writer, "UTF-8");
+
+        return scraper;
+    }
 }

source/de/anomic/net/URL.java

@@ -415,9 +415,20 @@ public String getFile() {
 
     public String getFile(boolean includeReference) {
         // this is the path plus quest plus ref
+        // if there is no quest and no ref the result is identical to getPath
+        // this is defined according to http://java.sun.com/j2se/1.4.2/docs/api/java/net/URL.html#getFile()
         if (quest != null) return ((includeReference) && (ref != null)) ? path + "?" + quest + "#" + ref : path + "?" + quest;
         return ((includeReference) && (ref != null)) ? path + "#" + ref : path;
     }
+
+    public String getFileName() {
+        // this is a method not defined in any sun api
+        // it returns the last portion of a path without any reference
+        int p = path.lastIndexOf('/');
+        if (p < 0) return path;
+        if (p == path.length() - 1) return ""; // no file name, this is a path to a directory
+        return path.substring(p + 1); // the 'real' file name
+    }
 
     public String getPath() {
         return path;

source/de/anomic/plasma/plasmaParser.java

@@ -680,7 +680,7 @@ public plasmaParserDocument parseSource(URL location, String theMimeType, String
                 theParser.setContentLength(contentLength);
                 // parse the resource
                 doc = theParser.parse(location, mimeType,documentCharset,sourceStream);
-            } else if (realtimeParsableMimeTypesContains(mimeType)) {                      
+            } else if (realtimeParsableMimeTypesContains(mimeType)) {
                 doc = parseHtml(location, mimeType, documentCharset, sourceStream);
             } else {
                 String errorMsg = "No parser available to parse mimetype '" + mimeType + "'";
@@ -719,7 +719,7 @@ public plasmaParserDocument parseSource(URL location, String theMimeType, String
 
     private plasmaParserDocument parseHtml(URL location, String mimeType, String documentCharset, InputStream sourceStream) throws IOException, ParserException {
 
-        // ...otherwise we make a scraper and transformer
+        // make a scraper and transformer
         htmlFilterInputStream htmlFilter = new htmlFilterInputStream(sourceStream,documentCharset,location,null,false);
         String charset = htmlFilter.detectCharset();
         if (charset == null) {
@@ -745,7 +745,7 @@ private plasmaParserDocument parseHtml(URL location, String mimeType, String doc
             this.theLogger.logSevere("Unable to parse '" + location + "'. " + errorMsg);
             throw new ParserException(errorMsg,location);    
         }
-        return transformScraper(location, mimeType, documentCharset, scraper);        
+        return transformScraper(location, mimeType, documentCharset, scraper);
     }
 
     public plasmaParserDocument transformScraper(URL location, String mimeType, String charSet, htmlFilterContentScraper scraper) {

source/de/anomic/yacy/yacyCore.java

@@ -95,7 +95,6 @@ public class yacyCore {
     public static yacyDHTAction dhtAgent = null;
     public static serverLog log;
     public static long lastOnlineTime = 0;
-    public static double latestVersion = 0.1;
     /** pseudo-random key derived from a time-interval while YaCy startup*/
     public static long speedKey = 0;
     public static File yacyDBPath;

source/de/anomic/yacy/yacyPeerActions.java

@@ -338,7 +338,7 @@ private synchronized boolean connectPeer(yacySeed seed, boolean direct) {
         }
 
         // update latest version number
-        if (seed.getVersion() > yacyCore.latestVersion) yacyCore.latestVersion = seed.getVersion();
+        if (seed.getVersion() > yacyVersion.latestRelease) yacyVersion.latestRelease = seed.getVersion();
 
         // prepare to update
         if (disconnectedSeed != null) {