Display OSM data date in web UI.

Now we are able to read the age of OSM data from the osmosis_replication_timestamp field according to https://wiki.openstreetmap.org/wiki/PBF_Format#Definition_of_the_OSMHeader_fileblock see also graphhopper#535
karussell · Oct 12, 2015 · 0d86c2e · 0d86c2e
1 parent fa0c8c5
commit 0d86c2e
Show file tree

Hide file tree

Showing 18 changed files with 250 additions and 58 deletions.
diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,7 @@ android/libs/graphhopper-*-android.jar
 *iml
 debug.sh
 *.pbf
+!/core/src/test/resources/com/graphhopper/reader/*.pbf
 *.dem
 *.log
 core/TODO*.txt

diff --git a/core/pom.xml b/core/pom.xml
@@ -18,9 +18,13 @@
         <version>0.6-SNAPSHOT</version>
     </parent>
 
-    <properties>  
+    <properties>
         <netbeans.hint.license>apache20</netbeans.hint.license>        
-        <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>        
+        <!-- Make sure that we use the same format as for Helper.createFormatter.
+             We cannot force the UTC TimeZone so it will just throw away the local offset or is this
+             fixed due to https://issues.apache.org/jira/browse/MNG-5647 ?
+        -->
+        <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ss'Z'</maven.build.timestamp.format>        
         <builddate>${maven.build.timestamp}</builddate>        
     </properties>
     <licenses>

diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java
@@ -37,7 +37,7 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.text.SimpleDateFormat;
+import java.text.DateFormat;
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.concurrent.ExecutorService;
@@ -691,8 +691,11 @@ private GraphHopper process( String graphHopperLocation )
 
             try
             {
-                importData();
-                ghStorage.getProperties().put("osmreader.import.date", formatDateTime(new Date()));
+                DataReader reader = importData();
+                DateFormat f = Helper.createFormatter();
+                ghStorage.getProperties().put("osmreader.import.date", f.format(new Date()));
+                if (reader.getDataDate() != null)
+                    ghStorage.getProperties().put("osmreader.data.date", f.format(reader.getDataDate()));
             } catch (IOException ex)
             {
                 throw new RuntimeException("Cannot parse OSM file " + getOSMFile(), ex);
@@ -1189,7 +1192,7 @@ public void run()
                             PrepareContractionHierarchies pch = (PrepareContractionHierarchies) entry.getValue();
                             pch.doWork();
                             ghStorage.getProperties().put(errorKey, "");
-                            ghStorage.getProperties().put("prepare.date." + name, formatDateTime(new Date()));
+                            ghStorage.getProperties().put("prepare.date." + name, Helper.createFormatter().format(new Date()));
                         } catch (Exception ex)
                         {
                             logger.error("Problem while CH preparation " + name);
@@ -1274,13 +1277,6 @@ public void clean()
         Helper.removeDir(folder);
     }
 
-    // make sure this is identical to buildDate used in pom.xml
-    // <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
-    private String formatDateTime( Date date )
-    {
-        return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(date);
-    }
-
     protected void ensureNotLoaded()
     {
         if (fullyLoaded)

diff --git a/core/src/main/java/com/graphhopper/reader/DataReader.java b/core/src/main/java/com/graphhopper/reader/DataReader.java
@@ -19,12 +19,20 @@
 package com.graphhopper.reader;
 
 import java.io.IOException;
+import java.util.Date;
 
 /**
  * @author Peter Karich
  */
 public interface DataReader
 {
-
+    /**
+     * This method triggers reading the underlying data to create a graph
+     */
     void readGraph() throws IOException;
+
+    /**
+     * This method returns the date of the most recent change for the underlying data.
+     */
+    Date getDataDate();
 }
diff --git a/core/src/main/java/com/graphhopper/reader/OSMElement.java b/core/src/main/java/com/graphhopper/reader/OSMElement.java
@@ -37,6 +37,7 @@ public abstract class OSMElement
     public static final int NODE = 0;
     public static final int WAY = 1;
     public static final int RELATION = 2;
+    public static final int FILEHEADER = 3;
     private final int type;
     private final long id;
     private final Map<String, Object> properties = new HashMap<String, Object>(5);

diff --git a/core/src/main/java/com/graphhopper/reader/OSMFileHeader.java b/core/src/main/java/com/graphhopper/reader/OSMFileHeader.java
@@ -0,0 +1,66 @@
+/*
+ *  Licensed to GraphHopper and Peter Karich under one or more contributor
+ *  license agreements. See the NOTICE file distributed with this work for 
+ *  additional information regarding copyright ownership.
+ * 
+ *  GraphHopper licenses this file to you under the Apache License, 
+ *  Version 2.0 (the "License"); you may not use this file except in 
+ *  compliance with the License. You may obtain a copy of the License at
+ * 
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.graphhopper.reader;
+
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+/**
+ * Represents an OSM file header
+ * <p>
+ * @author ratrun
+ */
+public class OSMFileHeader extends OSMElement
+{
+
+    /**
+     * Constructor for XML Parser
+     */
+    public static OSMFileHeader create( long id, XMLStreamReader parser ) throws XMLStreamException
+    {
+        OSMFileHeader header = new OSMFileHeader();
+        parser.nextTag();
+        return header;
+    }
+
+    public OSMFileHeader( )
+    {
+        super(0, FILEHEADER);
+    }
+
+    protected void readFileHeader( XMLStreamReader parser ) throws XMLStreamException
+    {
+        int event = parser.getEventType();
+        while (event != XMLStreamConstants.END_DOCUMENT && parser.getLocalName().equals("osm"))
+        {
+            if (event == XMLStreamConstants.START_ELEMENT)
+            {
+                String timestamp = parser.getAttributeValue(null, "timestamp");
+            }
+
+            event = parser.nextTag();
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return "OSM File header:" + super.toString();
+    }
+}
diff --git a/core/src/main/java/com/graphhopper/reader/OSMInputFile.java b/core/src/main/java/com/graphhopper/reader/OSMInputFile.java
@@ -48,6 +48,7 @@ public class OSMInputFile implements Sink, Closeable
     private final BlockingQueue<OSMElement> itemQueue;
     private boolean hasIncomingData;
     private int workerThreads = -1;
+    private OSMFileHeader fileheader;
 
     public OSMInputFile( File file ) throws IOException
     {
@@ -153,6 +154,23 @@ private void openXMLStream( InputStream in )
         {
             throw new IllegalArgumentException("File is not a valid OSM stream");
         }
+        // See https://wiki.openstreetmap.org/wiki/PBF_Format#Definition_of_the_OSMHeader_fileblock
+        String timestamp = parser.getAttributeValue(null, "osmosis_replication_timestamp");
+
+        if (timestamp==null)
+            timestamp = parser.getAttributeValue(null, "timestamp");
+
+        if (timestamp!=null)
+        {
+            try
+            {
+                fileheader = new OSMFileHeader();                
+                fileheader.setTag("timestamp", timestamp);
+            }
+            catch (Exception ex)
+            {
+            }
+        }
 
         eof = false;
     }
@@ -179,6 +197,13 @@ private OSMElement getNextXML() throws XMLStreamException
     {
 
         int event = parser.next();
+        if (fileheader!=null)
+        {
+            OSMElement copyfileheader = fileheader;
+            fileheader = null;
+            return copyfileheader;
+        }
+
         while (event != XMLStreamConstants.END_DOCUMENT)
         {
             if (event == XMLStreamConstants.START_ELEMENT)

diff --git a/core/src/main/java/com/graphhopper/reader/OSMReader.java b/core/src/main/java/com/graphhopper/reader/OSMReader.java
@@ -119,6 +119,7 @@ public class OSMReader implements DataReader
     private ElevationProvider eleProvider = ElevationProvider.NOOP;
     private final boolean exitOnlyPillarNodeException = true;
     private File osmFile;
+    private Date osmDataDate;
     private final Map<FlagEncoder, EdgeExplorer> outExplorerMap = new HashMap<FlagEncoder, EdgeExplorer>();
     private final Map<FlagEncoder, EdgeExplorer> inExplorerMap = new HashMap<FlagEncoder, EdgeExplorer>();
 
@@ -193,23 +194,32 @@ void preProcess( File osmFile )
                                     + getNodeMap().getMemoryUsage() + "MB) " + Helper.getMemInfo());
                         }
                     }
-                }
-                if (item.isType(OSMElement.RELATION))
+                } else
                 {
-                    final OSMRelation relation = (OSMRelation) item;
-                    if (!relation.isMetaRelation() && relation.hasTag("type", "route"))
-                        prepareWaysWithRelationInfo(relation);
+                    if (item.isType(OSMElement.RELATION))
+                    {
+                        final OSMRelation relation = (OSMRelation) item;
+                        if (!relation.isMetaRelation() && relation.hasTag("type", "route"))
+                            prepareWaysWithRelationInfo(relation);
 
-                    if (relation.hasTag("type", "restriction"))
-                        prepareRestrictionRelation(relation);
+                        if (relation.hasTag("type", "restriction"))
+                            prepareRestrictionRelation(relation);
 
-                    if (++tmpRelationCounter % 50000 == 0)
+                        if (++tmpRelationCounter % 50000 == 0)
+                        {
+                            logger.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size())
+                                    + " " + Helper.getMemInfo());
+                        }
+                    } else
                     {
-                        logger.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size())
-                                + " " + Helper.getMemInfo());
+                        if (item.isType(OSMElement.FILEHEADER))
+                        {
+                            final OSMFileHeader fileHeader = (OSMFileHeader) item;
+                            osmDataDate = Helper.createFormatter().parse(fileHeader.getTag("timestamp"));
+                        }
                     }
-
                 }
+
             }
         } catch (Exception ex)
         {
@@ -376,8 +386,7 @@ void processWay( OSMWay way )
                 long dur = OSMTagParser.parseDuration(way.getTag("duration"));
                 // Provide the duration value in seconds in an artificial graphhopper specific tag:
                 way.setTag("duration:seconds", Long.toString(dur));
-            }
-            catch(Exception ex)
+            } catch (Exception ex)
             {
                 logger.warn("Parsing error in way with OSMID=" + way.getId() + " : " + ex.getMessage());
             }
@@ -1030,6 +1039,12 @@ private void printInfo( String str )
                 + " " + Helper.getMemInfo());
     }
 
+    @Override
+    public Date getDataDate()
+    {
+        return osmDataDate;
+    }
+
     @Override
     public String toString()
     {

diff --git a/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java b/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java
@@ -6,11 +6,14 @@
 import com.graphhopper.reader.OSMNode;
 import com.graphhopper.reader.OSMRelation;
 import com.graphhopper.reader.OSMWay;
+import com.graphhopper.reader.OSMFileHeader;
+import com.graphhopper.util.Helper;
 import org.openstreetmap.osmosis.osmbinary.Fileformat;
 import org.openstreetmap.osmosis.osmbinary.Osmformat;
 import gnu.trove.list.TLongList;
 
 import java.io.IOException;
+import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
@@ -104,6 +107,11 @@ private void processOsmHeader( byte[] data ) throws InvalidProtocolBufferExcepti
             throw new RuntimeException("PBF file contains unsupported features " + unsupportedFeatures);
         }
 
+        OSMFileHeader fileheader = new OSMFileHeader();
+        long milliSecondDate = header.getOsmosisReplicationTimestamp();        
+        fileheader.setTag("timestamp", Helper.createFormatter().format(new Date(milliSecondDate * 1000)));
+        decodedEntities.add(fileheader);
+
         // Build a new bound object which corresponds to the header.
 /*
          Bound bound;

diff --git a/core/src/main/java/com/graphhopper/util/Helper.java b/core/src/main/java/com/graphhopper/util/Helper.java
@@ -29,7 +29,9 @@
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
+import java.text.DateFormat;
 import java.text.NumberFormat;
+import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.Map.Entry;
 
@@ -496,4 +498,15 @@ public static final double round2( double value )
     {
         return Math.round(value * 100) / 100d;
     }
+
+    /**
+     * This creates a date formatter for yyyy-MM-dd'T'HH:mm:ss'Z' which is has to be identical to
+     * buildDate used in pom.xml
+     */
+    public static DateFormat createFormatter()
+    {
+        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+        df.setTimeZone(TimeZone.getTimeZone("UTC"));
+        return df;
+    }
 }
diff --git a/core/src/main/java/com/graphhopper/util/InstructionList.java b/core/src/main/java/com/graphhopper/util/InstructionList.java
@@ -17,7 +17,7 @@
  */
 package com.graphhopper.util;
 
-import java.text.SimpleDateFormat;
+import java.text.DateFormat;
 import java.util.*;
 
 /**
@@ -178,9 +178,8 @@ public String createGPX( String trackName, long startTimeMillis )
 
     public String createGPX( String trackName, long startTimeMillis, boolean includeElevation )
     {
-        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-        formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
-
+        DateFormat formatter = Helper.createFormatter();
+
         String header = "<?xml version='1.0' encoding='UTF-8' standalone='no' ?>"
                 + "<gpx xmlns='http://www.topografix.com/GPX/1/1' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'"
                 + " creator='Graphhopper' version='1.1'"