added parsing of 'date', 'dc:date', 'dc.date' and 'last-modified' in

html meta fields to get a correct (or: better) date timestamp. The http:last-modified mostly does not work because it is set to the current date from most CMS.
yacy · Sep 10, 2013 · 35ab2ce · 35ab2ce
1 parent 9cc8468
commit 35ab2ce
Show file tree

Hide file tree

Showing 29 changed files with 120 additions and 33 deletions.
diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java
@@ -116,7 +116,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po
         post.put("defType", "edismax");
         post.put(CommonParams.Q, solrQ.toString());
         post.put(CommonParams.ROWS, post.remove("num"));
-        post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 5000 : 100));
+        post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100));
 
         // set ranking
         if (post.containsKey("sort")) {

diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java
@@ -94,6 +94,7 @@ public class Document {
     private final double lon, lat;
     private final Object parserObject; // the source object that was used to create the Document
     private final Map<String, Set<String>> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this document
+    private final Date date;
 
     public Document(final DigestURI location, final String mimeType, final String charset,
                     final Object parserObject,
@@ -107,7 +108,8 @@ public Document(final DigestURI location, final String mimeType, final String ch
                     final Map<DigestURI, Properties> anchors,
                     final Map<DigestURI, String> rss,
                     final Map<DigestURI, ImageEntry> images,
-                    final boolean indexingDenied) {
+                    final boolean indexingDenied,
+                    final Date date) {
         this.source = location;
         this.mimeType = (mimeType == null) ? "application/octet-stream" : mimeType;
         this.charset = charset;
@@ -143,6 +145,7 @@ public Document(final DigestURI location, final String mimeType, final String ch
         this.indexingDenied = indexingDenied;
         this.text = text == null ? "" : text;
         this.generic_facets = new HashMap<String, Set<String>>();
+        this.date = date == null ? new Date() : date;
     }
 
     public Object getParserObject() {
@@ -451,6 +454,10 @@ public Map<String, String> getEmaillinks() {
         return this.emaillinks;
     }
 
+    public Date getDate() {
+        return this.date;
+    }
+
     public double lon() {
         return this.lon;
     }
@@ -783,6 +790,7 @@ public static Document mergeDocuments(final DigestURI location, final String glo
         final Map<DigestURI, String> rss = new HashMap<DigestURI, String>();
         final Map<DigestURI, ImageEntry> images = new HashMap<DigestURI, ImageEntry>();
         double lon = 0.0d, lat = 0.0d;
+        Date date = new Date();
 
         for (final Document doc: docs) {
 
@@ -821,6 +829,7 @@ public static Document mergeDocuments(final DigestURI location, final String glo
             rss.putAll(doc.getRSS());
             ContentScraper.addAllImages(images, doc.getImages());
             if (doc.lon() != 0.0 && doc.lat() != 0.0) { lon = doc.lon(); lat = doc.lat(); }
+            if (doc.date.before(date)) date = doc.date;
         }
 
         // clean up parser data
@@ -852,7 +861,8 @@ public static Document mergeDocuments(final DigestURI location, final String glo
                 anchors,
                 rss,
                 images,
-                false);
+                false,
+                date);
     }
 
     public static Map<DigestURI, String> getHyperlinks(final Document[] documents) {

diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java
@@ -100,7 +100,9 @@ public DCEntry(
          */
     public Date getDate() {
         String d = this.get("docdatetime");
+        if (d == null) d = this.get("date");
         if (d == null) d = this.get("dc:date");
+        if (d == null) d = this.get("last-modified");
         if (d == null) return null;
         if (d.isEmpty()) return null;
         try {
@@ -286,7 +288,8 @@ public Document document() {
             null,
             null,
             null,
-            false);
+            false,
+            getDate());
     }
 
     public void writeXML(OutputStreamWriter os) throws IOException {

diff --git a/source/net/yacy/document/parser/audioTagParser.java b/source/net/yacy/document/parser/audioTagParser.java
@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -171,7 +172,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
                     null,
                     null,
                     null,
-                    false)
+                    false,
+                    new Date())
             };            
             return docs;
         } catch (final Exception e) {
@@ -193,7 +195,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
 	                null,
 	                null,
 	                null,
-	                false
+	                false,
+                    new Date()
 	    	)};
 		} finally {
             try {

diff --git a/source/net/yacy/document/parser/csvParser.java b/source/net/yacy/document/parser/csvParser.java
@@ -30,6 +30,7 @@
 import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import net.yacy.document.AbstractParser;
@@ -77,7 +78,8 @@ public Document[] parse(DigestURI location, String mimeType, String charset, Inp
 		        null,
 		        null,
 		        null,
-		        false)};
+		        false,
+		        new Date())};
     }
 
     private static String concatRow(String[] columns) {

diff --git a/source/net/yacy/document/parser/docParser.java b/source/net/yacy/document/parser/docParser.java
@@ -28,6 +28,7 @@
 package net.yacy.document.parser;
 
 import java.io.InputStream;
+import java.util.Date;
 
 import net.yacy.document.AbstractParser;
 import net.yacy.document.Document;
@@ -103,7 +104,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
                   null,
                   null,
                   null,
-                  false)};
+                  false,
+                  new Date())};
 
         return docs;
     }

diff --git a/source/net/yacy/document/parser/genericParser.java b/source/net/yacy/document/parser/genericParser.java
@@ -25,6 +25,7 @@
 package net.yacy.document.parser;
 
 import java.io.InputStream;
+import java.util.Date;
 
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.document.AbstractParser;
@@ -65,7 +66,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
                 null,
                 null,
                 null,
-                false)};
+                false,
+                new Date())};
         return docs;
     }
 }
diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java
@@ -31,7 +31,9 @@
 import java.io.Writer;
 import java.net.MalformedURLException;
 import java.nio.charset.Charset;
+import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
@@ -45,6 +47,7 @@
 
 import javax.swing.event.EventListenerList;
 
+import net.yacy.cora.date.ISO8601Formatter;
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.cora.sorting.ClusteredScoreMap;
 import net.yacy.cora.storage.SizeLimitedMap;
@@ -848,6 +851,28 @@ public String getRefreshPath() {
         if (s.toLowerCase().startsWith("url=")) return s.substring(4).trim();
         return EMPTY_STRING;
     }
+
+    public Date getDate() {
+        String content;
+
+        // <meta name="date" content="YYYY-MM-DD..." />
+        content = this.metas.get("date");
+        if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {}
+
+        // <meta name="DC.date" content="YYYY-MM-DD" />
+        content = this.metas.get("dc.date");
+        if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {}
+
+        // <meta name="DC:date" content="YYYY-MM-DD" />
+        content = this.metas.get("dc:date");
+        if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {}
+
+        // <meta http-equiv="last-modified" content="YYYY-MM-DD" />
+        content = this.metas.get("last-modified");
+        if (content != null) try {return ISO8601Formatter.FORMATTER.parse(content);} catch (ParseException e) {}
+
+        return new Date();
+    }
 
     // parse location
     // <meta NAME="ICBM" CONTENT="38.90551492, 1.454004505" />

diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java
@@ -141,7 +141,8 @@ private static Document transformScraper(final DigestURI location, final String
                 scraper.getAnchors(),
                 scraper.getRSS(),
                 scraper.getImages(),
-                scraper.indexingDenied());
+                scraper.indexingDenied(),
+                scraper.getDate());
         ppd.setFavicon(scraper.getFavicon());
 
         return ppd;

diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java
@@ -35,6 +35,7 @@
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -221,7 +222,8 @@ public Document[] parse(
              anchors, // anchors
              null,
              images,
-             false)}; // images
+             false,
+             new Date())}; // images
     }
 
     @Override

diff --git a/source/net/yacy/document/parser/mmParser.java b/source/net/yacy/document/parser/mmParser.java
@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import javax.xml.parsers.ParserConfigurationException;
@@ -116,7 +117,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
             null,
             null,
             null,
-            false)};
+            false,
+            new Date())};
     }
 
     private class FreeMindHandler extends DefaultHandler {

diff --git a/source/net/yacy/document/parser/odtParser.java b/source/net/yacy/document/parser/odtParser.java
@@ -30,6 +30,7 @@
 import java.io.File;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.List;
@@ -197,7 +198,9 @@ private Document[] parse(final DigestURI location, final String mimeType, @Suppr
                     null,
                     null,
                     null,
-                    false)};
+                    false,
+                    new Date()
+                    )};
             return docs;
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;

diff --git a/source/net/yacy/document/parser/ooxmlParser.java b/source/net/yacy/document/parser/ooxmlParser.java
@@ -30,6 +30,7 @@
 import java.io.File;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.List;
@@ -182,7 +183,8 @@ private Document[] parse(final DigestURI location, final String mimeType, @Suppr
                     null,
                     null,
                     null,
-                    false)};
+                    false,
+                    new Date())};
             return docs;
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;

diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.java
@@ -32,6 +32,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Date;
 
 import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.exceptions.CryptographyException;
@@ -125,17 +126,17 @@ public Document[] parse(final DigestURI location, final String mimeType, final S
         // extracting some metadata
         PDDocumentInformation info = pdfDoc.getDocumentInformation();
         String docTitle = null, docSubject = null, docAuthor = null, docPublisher = null, docKeywordStr = null;
+        Date docDate = new Date();
         if (info != null) {
             docTitle = info.getTitle();
             docSubject = info.getSubject();
             docAuthor = info.getAuthor();
             docPublisher = info.getProducer();
             if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator();
             docKeywordStr = info.getKeywords();
+            try {if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime();} catch (IOException e) {}
             // unused:
             // info.getTrapped());
-            // info.getCreationDate());
-            // info.getModificationDate();
         }
         info = null;
 
@@ -218,7 +219,8 @@ public void run() {
                 null,
                 null,
                 null,
-                false)};
+                false,
+                docDate)};
     }
 
     @SuppressWarnings("static-access")

diff --git a/source/net/yacy/document/parser/pptParser.java b/source/net/yacy/document/parser/pptParser.java
@@ -29,6 +29,7 @@
 
 import java.io.BufferedInputStream;
 import java.io.InputStream;
+import java.util.Date;
 
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.document.AbstractParser;
@@ -99,7 +100,8 @@ public Document[] parse(final DigestURI location, final String mimeType,
                     null,
                     null,
                     null,
-                    false)};
+                    false,
+                    new Date())};
             return docs;
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;

diff --git a/source/net/yacy/document/parser/psParser.java b/source/net/yacy/document/parser/psParser.java
@@ -34,6 +34,7 @@
 import java.io.FileWriter;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Date;
 
 import net.yacy.document.AbstractParser;
 import net.yacy.document.Document;
@@ -115,7 +116,8 @@ private Document[] parse(final DigestURI location, final String mimeType, @Suppr
                     null,     // anchors
                     null,     // rss
                     null,     // images
-                    false)};  // indexingdenied
+                    false,    // indexingdenied
+                    new Date())};  
 
             return docs;
         } catch (final Exception e) {

diff --git a/source/net/yacy/document/parser/rdfParser.java b/source/net/yacy/document/parser/rdfParser.java
@@ -27,6 +27,7 @@
 
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import net.yacy.document.AbstractParser;
@@ -59,7 +60,7 @@ public Document[] parse(final DigestURI url, final String mimeType,
 
         String all = "rdfdatasource";
 		doc = new Document(url, mimeType, charset, null, null, null, singleList(""), "",
-					"", null, new ArrayList<String>(0), 0, 0, all, null, null, null, false);
+					"", null, new ArrayList<String>(0), 0, 0, all, null, null, null, false, new Date());
 
         docs.add(doc);