continuing the fight against deadlocks during time formatting: better…

… caching.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7531 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/ContentIntegrationPHPBB3_p.java

@@ -23,8 +23,6 @@
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import java.io.File;
-import java.util.Date;
-
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.document.content.dao.Dao;
@@ -107,7 +105,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                                             dbpw
                                             );
 
-                    int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()), ppf);
+                    int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);
                     prop.put("export", 1);
                     prop.put("export_files", files);
                     db.close();

htroot/IndexControlURLs_p.java

@@ -28,7 +28,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.util.Date;
 import java.util.Iterator;
 
 import net.yacy.cora.date.GenericFormatter;
@@ -102,7 +101,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
             prop.put("reload", 1);
         } else {
             prop.put("lurlexport", 1);
-            prop.put("lurlexport_exportfile", sb.getDataPath() + "/DATA/EXPORT/" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()));
+            prop.put("lurlexport_exportfile", sb.getDataPath() + "/DATA/EXPORT/" + GenericFormatter.SHORT_SECOND_FORMATTER.format());
             if (export == null) {
                 // there has never been an export
                 prop.put("lurlexportfinished", 0);

htroot/api/bookmarks/posts/get.java

@@ -28,7 +28,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
         if (post != null && post.containsKey("date")) {
             date = post.get("date");
         } else {
-            date = ISO8601Formatter.FORMATTER.format(new Date(System.currentTimeMillis()));
+            date = ISO8601Formatter.FORMATTER.format();
         }
 
         // if an extended xml should be used

htroot/cytag.java

@@ -28,8 +28,6 @@
 import java.awt.Image;
 import java.io.File;
 import java.io.IOException;
-import java.util.Date;
-
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.cora.protocol.HeaderFramework;
@@ -52,7 +50,7 @@ public static Image respond(final RequestHeader header, final serverObjects post
         // harvest request information
         StringBuilder connect = new StringBuilder();
         connect.append('{');
-        appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()));
+        appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format());
         appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false, false));
         appendJSON(connect, "nick",  (post == null) ? "" : post.get("nick", ""));
         appendJSON(connect, "tag",   (post == null) ? "" : post.get("tag", ""));

htroot/yacy/query.java

@@ -28,8 +28,6 @@
 // if the shell's current path is HTROOT
 
 import java.io.IOException;
-import java.util.Date;
-
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
@@ -75,7 +73,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
         final String obj    = post.get("object", ""); // keyword for query subject
         final String env    = post.get("env", "");    // argument to query
 
-        prop.put("mytime", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()));
+        prop.put("mytime", GenericFormatter.SHORT_SECOND_FORMATTER.format());
 
         // check if we are the right target and requester has correct information about this peer
         if (sb.peers.mySeed() == null || !sb.peers.mySeed().hash.equals(youare)) {

source/de/anomic/data/BlogBoard.java

@@ -434,7 +434,7 @@ public String getTimestamp() {
                 if (Log.isFinest("Blog")) {
                     Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
                 }
-                return GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date());
+                return GenericFormatter.SHORT_SECOND_FORMATTER.format();
             }
             return timestamp;
         }

source/de/anomic/data/WorkTables.java

@@ -124,7 +124,7 @@ public byte[] recordAPICall(final serverObjects post, final String servletName,
                 Data data = new Data();
                 data.put(TABLE_API_COL_TYPE, type.getBytes());
                 data.put(TABLE_API_COL_COMMENT, comment.getBytes());
-                byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes();
+                byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes();
                 data.put(TABLE_API_COL_DATE_RECORDING, date);
                 data.put(TABLE_API_COL_DATE_LAST_EXEC, date);
                 data.put(TABLE_API_COL_URL, apiurl.getBytes());
@@ -136,7 +136,7 @@ public byte[] recordAPICall(final serverObjects post, final String servletName,
                 // modify and update existing entry
 
                 // modify date attributes and patch old values
-                row.put(TABLE_API_COL_DATE_LAST_EXEC, GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes());
+                row.put(TABLE_API_COL_DATE_LAST_EXEC, GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes());
                 if (!row.containsKey(TABLE_API_COL_DATE_RECORDING)) row.put(TABLE_API_COL_DATE_RECORDING, row.get(TABLE_API_COL_DATE));
                 row.remove(TABLE_API_COL_DATE);
 
@@ -185,7 +185,7 @@ public byte[] recordAPICall(final serverObjects post, final String servletName,
             Data data = new Data();
             data.put(TABLE_API_COL_TYPE, type.getBytes());
             data.put(TABLE_API_COL_COMMENT, comment.getBytes());
-            byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes();
+            byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes();
             data.put(TABLE_API_COL_DATE_RECORDING, date);
             data.put(TABLE_API_COL_DATE_LAST_EXEC, date);
             data.put(TABLE_API_COL_URL, apiurl.getBytes());
@@ -309,7 +309,7 @@ public void failURLsRegisterMissingWord(IndexCell<WordReference> indexCell, fina
         try {
             // create and insert new entry
             Data data = new Data();
-            byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes();
+            byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format().getBytes();
             data.put(TABLE_SEARCH_FAILURE_COL_URL, url.toNormalform(true, false));
             data.put(TABLE_SEARCH_FAILURE_COL_DATE, date);
             data.put(TABLE_SEARCH_FAILURE_COL_WORDS, queryHashes.export());

source/de/anomic/http/server/HTTPDFileHandler.java

@@ -828,7 +828,7 @@ public static void doResponse(final Properties conProp, final RequestHeader requ
                             templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() -  serverCore.startupTime) / 1000) / 60); // uptime in minutes
                             templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
                             templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID());
-                            templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()));
+                            templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
                             yacySeed myPeer = sb.peers.mySeed();
                             templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1); 
                             templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);

source/de/anomic/search/AccessTracker.java

@@ -26,7 +26,6 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.Iterator;
 import java.util.LinkedList;
 
@@ -85,7 +84,7 @@ private static void addToDump(QueryParams query) {
         //if (query.resultcount == 0) return;
         if (query.queryString == null || query.queryString.length() == 0) return;
         StringBuilder sb = new StringBuilder(40);
-        sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()));
+        sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format());
         sb.append(' ');
         sb.append(Integer.toString(query.resultcount));
         sb.append(' ');

source/de/anomic/yacy/graphics/WebStructureGraph.java

@@ -150,7 +150,7 @@ static Map<String, Integer> refstr2map(final String refs) {
 
     private static String map2refstr(final Map<String, Integer> map) {
         final StringBuilder s = new StringBuilder(map.size() * 10);
-        s.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date()));
+        s.append(GenericFormatter.SHORT_DAY_FORMATTER.format());
         String h;
         for (final Map.Entry<String, Integer> entry : map.entrySet()) {
             s.append(entry.getKey());
@@ -229,7 +229,7 @@ public structureEntry incomingReferences(final String domhash) {
         return new structureEntry(
                 domhash,
                 host,
-                GenericFormatter.SHORT_DAY_FORMATTER.format(new Date()),
+                GenericFormatter.SHORT_DAY_FORMATTER.format(),
                 domhashes);
     }
 

source/de/anomic/yacy/yacyCore.java

@@ -212,7 +212,7 @@ private boolean canReachMyself() { // TODO: check if this method is necessary -
     }
 
     // use our own formatter to prevent concurrency locks with other processes
-    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
+    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
 
     protected class publishThread extends Thread {
         int added;

source/de/anomic/yacy/yacyNetwork.java

@@ -27,7 +27,6 @@
 
 package de.anomic.yacy;
 
-import java.util.Date;
 import java.util.LinkedHashMap;
 
 import net.yacy.cora.date.GenericFormatter;
@@ -93,7 +92,7 @@ public static final LinkedHashMap<String,ContentBody> basicRequestParts(final Sw
 	}
 
 	// use our own formatter to prevent concurrency locks with other processes
-    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
+    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
 
     public static final LinkedHashMap<String,ContentBody> basicRequestParts(String myHash, String targetHash, String networkName) {
         // put in all the essentials for routing and network authentication
@@ -106,7 +105,7 @@ public static final LinkedHashMap<String,ContentBody> basicRequestParts(String m
             if (targetHash != null) parts.put("youare", UTF8.StringBody(targetHash));
 
             // time information for synchronization
-            parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format(new Date())));
+            parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format()));
             parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis())));
 
             // network identification

source/de/anomic/yacy/yacyNewsDB.java

@@ -159,7 +159,7 @@ public synchronized Record get(final String id) throws IOException {
     }
 
     // use our own formatter to prevent concurrency locks with other processes
-    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
+    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
 
     private Record b2r(final Row.Entry b) {
         if (b == null) return null;

source/de/anomic/yacy/yacyNewsQueue.java

@@ -46,7 +46,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
 
@@ -169,7 +168,7 @@ private Row.Entry r2b(final yacyNewsDB.Record r) throws IOException, RowSpaceExc
         newsDB.put(r);
         final Row.Entry b = queueStack.row().newEntry(new byte[][]{
                 r.id().getBytes(),
-                GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()).getBytes()});
+                GenericFormatter.SHORT_SECOND_FORMATTER.format().getBytes()});
         return b;
     }
 

source/de/anomic/yacy/yacySeed.java

@@ -185,7 +185,7 @@ public yacySeed(final String theHash, final ConcurrentHashMap<String, String> th
     }
 
     // use our own formatter to prevent concurrency locks with other processes
-    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
+    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
 
     private yacySeed(final String theHash) {
         this.dna = new ConcurrentHashMap<String, String>();
@@ -728,7 +728,7 @@ public static yacySeed genLocalSeed(final yacySeedDB db, final int port, final S
         // now calculate other information about the host
         newSeed.dna.put(yacySeed.NAME, (name) == null ? defaultPeerName() : name);
         newSeed.dna.put(yacySeed.PORT, Integer.toString((port <= 0) ? 8090 : port));
-        newSeed.dna.put(yacySeed.BDATE, my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)) );
+        newSeed.dna.put(yacySeed.BDATE, my_SHORT_SECOND_FORMATTER.format());
         newSeed.dna.put(yacySeed.LASTSEEN, newSeed.dna.get(yacySeed.BDATE)); // just as initial setting
         newSeed.dna.put(yacySeed.UTC, GenericFormatter.UTCDiffString());
         newSeed.dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);

source/net/yacy/cora/date/AbstractFormatter.java

@@ -41,5 +41,6 @@ public abstract class AbstractFormatter implements DateFormatter {
 
     public abstract Date parse(String s) throws ParseException;
     public abstract String format(final Date date);
+    public abstract String format();
 
 }

source/net/yacy/cora/date/DateFormatter.java

@@ -27,5 +27,6 @@ public interface DateFormatter {
 
     public Date parse(String s) throws ParseException;
     public String format(final Date date);
+    public String format();
 
 }

source/net/yacy/cora/date/GenericFormatter.java

@@ -50,21 +50,19 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
         FORMAT_SHORT_MILSEC.setTimeZone(TZ_GMT);
     }
 
-    public static final GenericFormatter SHORT_DAY_FORMATTER     = new GenericFormatter(FORMAT_SHORT_DAY);
-    public static final GenericFormatter SHORT_SECOND_FORMATTER  = new GenericFormatter(FORMAT_SHORT_SECOND);
+    public static final long time_second =  1000L;
+    public static final long time_minute = 60000L;
+    public static final long time_hour   = 60 * time_minute;
+    public static final long time_day    = 24 * time_hour;
+
+    public static final GenericFormatter SHORT_DAY_FORMATTER     = new GenericFormatter(FORMAT_SHORT_DAY, time_minute);
+    public static final GenericFormatter SHORT_SECOND_FORMATTER  = new GenericFormatter(FORMAT_SHORT_SECOND, time_second);
     public static final GenericFormatter SHORT_MILSEC_FORMATTER  = new GenericFormatter(FORMAT_SHORT_MILSEC, 1);
-    public static final GenericFormatter ANSIC_FORMATTER         = new GenericFormatter(FORMAT_ANSIC);
-    public static final GenericFormatter RFC1123_SHORT_FORMATTER = new GenericFormatter(FORMAT_RFC1123_SHORT);
+    public static final GenericFormatter ANSIC_FORMATTER         = new GenericFormatter(FORMAT_ANSIC, time_second);
+    public static final GenericFormatter RFC1123_SHORT_FORMATTER = new GenericFormatter(FORMAT_RFC1123_SHORT, time_minute);
 
     private SimpleDateFormat dateFormat;
     private long maxCacheDiff;
-
-    public GenericFormatter(SimpleDateFormat dateFormat) {
-        this.dateFormat = dateFormat;
-        this.last_time = 0;
-        this.last_format = "";
-        this.maxCacheDiff = 1000;
-    }
 
     public GenericFormatter(SimpleDateFormat dateFormat, long maxCacheDiff) {
         this.dateFormat = dateFormat;
@@ -84,14 +82,22 @@ public GenericFormatter(SimpleDateFormat dateFormat, long maxCacheDiff) {
     @Override
     public String format(final Date date) {
         if (date == null) return "";
-        if (Math.abs(date.getTime() - last_time) < maxCacheDiff) return last_format;
+        synchronized (this.dateFormat) {
+            // calculate the date
+            return this.dateFormat.format(date);
+        }
+    }
+
+    public String format() {
+        if (Math.abs(System.currentTimeMillis() - last_time) < maxCacheDiff) return last_format;
         synchronized (this.dateFormat) {
             // threads that had been waiting here may use the cache now instead of calculating the date again
-            if (Math.abs(date.getTime() - last_time) < maxCacheDiff) return last_format;
+            long time = System.currentTimeMillis();
+            if (Math.abs(time - last_time) < maxCacheDiff) return last_format;
 
             // if the cache is not fresh, calculate the date
-            last_format = this.dateFormat.format(date);
-            last_time = date.getTime();
+            last_format = this.dateFormat.format(new Date(time));
+            last_time = time;
         }
         return last_format;
     }

source/net/yacy/cora/date/ISO8601Formatter.java

@@ -187,5 +187,14 @@ public final String format(final Date date) {
         }
         return last_format;
     }
+    public final String format() {
+        long time = System.currentTimeMillis();
+        if (Math.abs(time - last_time) < 1000) return last_format;
+        synchronized (FORMAT_ISO8601) {
+            last_format = FORMAT_ISO8601.format(new Date(time));
+            last_time = time;
+        }
+        return last_format;
+    }
 
 }

source/net/yacy/cora/document/RSSMessage.java

@@ -90,7 +90,7 @@ public RSSMessage(final String title, final String description, final String lin
         map.put("title", title);
         map.put("description", description);
         map.put("link", link);
-        map.put("pubDate", ISO8601Formatter.FORMATTER.format(new Date()));
+        map.put("pubDate", ISO8601Formatter.FORMATTER.format());
         map.put("guid", artificialGuidPrefix + Integer.toHexString((title + description + link).hashCode()));
     }
 

source/net/yacy/cora/services/SearchSRURSS.java

@@ -31,7 +31,6 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.http.entity.mime.content.ContentBody;
-import org.apache.http.entity.mime.content.StringBody;
 
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.cora.document.RSSFeed;

source/net/yacy/document/importer/OAIPMHImporter.java

@@ -242,6 +242,6 @@ public static final String hostID(DigestURI source) {
     public static final String filename4Source(DigestURI source) {
         return filenamePrefix + OAIPMHImporter.filenameSeparationChar +
                OAIPMHImporter.hostID(source) + OAIPMHImporter.filenameSeparationChar +
-               GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()) + ".xml";
+               GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".xml";
     }
 }

source/net/yacy/document/importer/ResumptionToken.java

@@ -142,7 +142,7 @@ public DigestURI resumptionURL() throws IOException {
         // can be detected with given expiration date
         Date expiration = getExpirationDate();
         if (expiration != null) {
-            if (expiration.before(new Date())) throw new IOException("the resumption is expired at " + ISO8601Formatter.FORMATTER.format(expiration) + " (now: " + ISO8601Formatter.FORMATTER.format(new Date()));
+            if (expiration.before(new Date())) throw new IOException("the resumption is expired at " + ISO8601Formatter.FORMATTER.format(expiration) + " (now: " + ISO8601Formatter.FORMATTER.format());
             // the resumption token is still fresh
         }
         String u = url + "verb=ListRecords&resumptionToken=" + escape(token);