From 0d86c2e5645d25ae2a060133c9cb6091fd1caf3e Mon Sep 17 00:00:00 2001 From: ratrun Date: Mon, 5 Oct 2015 14:45:39 +0200 Subject: [PATCH] Display OSM data date in web UI. Now we are able to read the age of OSM data from the osmosis_replication_timestamp field according to https://wiki.openstreetmap.org/wiki/PBF_Format#Definition_of_the_OSMHeader_fileblock see also #535 --- .gitignore | 1 + core/pom.xml | 8 ++- .../java/com/graphhopper/GraphHopper.java | 18 ++--- .../com/graphhopper/reader/DataReader.java | 10 ++- .../com/graphhopper/reader/OSMElement.java | 1 + .../com/graphhopper/reader/OSMFileHeader.java | 66 ++++++++++++++++++ .../com/graphhopper/reader/OSMInputFile.java | 25 +++++++ .../com/graphhopper/reader/OSMReader.java | 41 +++++++---- .../reader/pbf/PbfBlobDecoder.java | 8 +++ .../java/com/graphhopper/util/Helper.java | 13 ++++ .../com/graphhopper/util/InstructionList.java | 7 +- .../com/graphhopper/reader/OSMReaderTest.java | 46 ++++++++---- .../com/graphhopper/reader/test-osm.xml | 13 ++-- .../com/graphhopper/reader/test-osm2.xml | 2 +- .../com/graphhopper/reader/test-osm6.pbf | Bin 0 -> 683 bytes .../com/graphhopper/reader/test-osm6.xml | 34 +++++++++ .../com/graphhopper/http/InfoServlet.java | 3 + web/src/main/webapp/js/main.js | 12 ++-- 18 files changed, 250 insertions(+), 58 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/reader/OSMFileHeader.java create mode 100644 core/src/test/resources/com/graphhopper/reader/test-osm6.pbf create mode 100644 core/src/test/resources/com/graphhopper/reader/test-osm6.xml diff --git a/.gitignore b/.gitignore index 24a64efbea2..22f1ccbde9b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ android/libs/graphhopper-*-android.jar *iml debug.sh *.pbf +!/core/src/test/resources/com/graphhopper/reader/*.pbf *.dem *.log core/TODO*.txt diff --git a/core/pom.xml b/core/pom.xml index b4ec077a2aa..4c29c2ef104 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -18,9 +18,13 @@ 0.6-SNAPSHOT - + apache20 - yyyy-MM-dd'T'HH:mm:ssZ + + yyyy-MM-dd'T'HH:mm:ss'Z' ${maven.build.timestamp} diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index d92be978d00..5149c2ddfe3 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -37,7 +37,7 @@ import java.io.File; import java.io.IOException; -import java.text.SimpleDateFormat; +import java.text.DateFormat; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; @@ -691,8 +691,11 @@ private GraphHopper process( String graphHopperLocation ) try { - importData(); - ghStorage.getProperties().put("osmreader.import.date", formatDateTime(new Date())); + DataReader reader = importData(); + DateFormat f = Helper.createFormatter(); + ghStorage.getProperties().put("osmreader.import.date", f.format(new Date())); + if (reader.getDataDate() != null) + ghStorage.getProperties().put("osmreader.data.date", f.format(reader.getDataDate())); } catch (IOException ex) { throw new RuntimeException("Cannot parse OSM file " + getOSMFile(), ex); @@ -1189,7 +1192,7 @@ public void run() PrepareContractionHierarchies pch = (PrepareContractionHierarchies) entry.getValue(); pch.doWork(); ghStorage.getProperties().put(errorKey, ""); - ghStorage.getProperties().put("prepare.date." + name, formatDateTime(new Date())); + ghStorage.getProperties().put("prepare.date." + name, Helper.createFormatter().format(new Date())); } catch (Exception ex) { logger.error("Problem while CH preparation " + name); @@ -1274,13 +1277,6 @@ public void clean() Helper.removeDir(folder); } - // make sure this is identical to buildDate used in pom.xml - // yyyy-MM-dd'T'HH:mm:ssZ - private String formatDateTime( Date date ) - { - return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(date); - } - protected void ensureNotLoaded() { if (fullyLoaded) diff --git a/core/src/main/java/com/graphhopper/reader/DataReader.java b/core/src/main/java/com/graphhopper/reader/DataReader.java index d820a2ab7fc..0c4dffb4512 100644 --- a/core/src/main/java/com/graphhopper/reader/DataReader.java +++ b/core/src/main/java/com/graphhopper/reader/DataReader.java @@ -19,12 +19,20 @@ package com.graphhopper.reader; import java.io.IOException; +import java.util.Date; /** * @author Peter Karich */ public interface DataReader { - + /** + * This method triggers reading the underlying data to create a graph + */ void readGraph() throws IOException; + + /** + * This method returns the date of the most recent change for the underlying data. + */ + Date getDataDate(); } diff --git a/core/src/main/java/com/graphhopper/reader/OSMElement.java b/core/src/main/java/com/graphhopper/reader/OSMElement.java index 660b924a0d1..7e152e88d85 100644 --- a/core/src/main/java/com/graphhopper/reader/OSMElement.java +++ b/core/src/main/java/com/graphhopper/reader/OSMElement.java @@ -37,6 +37,7 @@ public abstract class OSMElement public static final int NODE = 0; public static final int WAY = 1; public static final int RELATION = 2; + public static final int FILEHEADER = 3; private final int type; private final long id; private final Map properties = new HashMap(5); diff --git a/core/src/main/java/com/graphhopper/reader/OSMFileHeader.java b/core/src/main/java/com/graphhopper/reader/OSMFileHeader.java new file mode 100644 index 00000000000..fb1b48d48e8 --- /dev/null +++ b/core/src/main/java/com/graphhopper/reader/OSMFileHeader.java @@ -0,0 +1,66 @@ +/* + * Licensed to GraphHopper and Peter Karich under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + * + * GraphHopper licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.graphhopper.reader; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Represents an OSM file header + *

+ * @author ratrun + */ +public class OSMFileHeader extends OSMElement +{ + + /** + * Constructor for XML Parser + */ + public static OSMFileHeader create( long id, XMLStreamReader parser ) throws XMLStreamException + { + OSMFileHeader header = new OSMFileHeader(); + parser.nextTag(); + return header; + } + + public OSMFileHeader( ) + { + super(0, FILEHEADER); + } + + protected void readFileHeader( XMLStreamReader parser ) throws XMLStreamException + { + int event = parser.getEventType(); + while (event != XMLStreamConstants.END_DOCUMENT && parser.getLocalName().equals("osm")) + { + if (event == XMLStreamConstants.START_ELEMENT) + { + String timestamp = parser.getAttributeValue(null, "timestamp"); + } + + event = parser.nextTag(); + } + } + + @Override + public String toString() + { + return "OSM File header:" + super.toString(); + } +} diff --git a/core/src/main/java/com/graphhopper/reader/OSMInputFile.java b/core/src/main/java/com/graphhopper/reader/OSMInputFile.java index 97a7a0475e8..cea89a4e61b 100644 --- a/core/src/main/java/com/graphhopper/reader/OSMInputFile.java +++ b/core/src/main/java/com/graphhopper/reader/OSMInputFile.java @@ -48,6 +48,7 @@ public class OSMInputFile implements Sink, Closeable private final BlockingQueue itemQueue; private boolean hasIncomingData; private int workerThreads = -1; + private OSMFileHeader fileheader; public OSMInputFile( File file ) throws IOException { @@ -153,6 +154,23 @@ private void openXMLStream( InputStream in ) { throw new IllegalArgumentException("File is not a valid OSM stream"); } + // See https://wiki.openstreetmap.org/wiki/PBF_Format#Definition_of_the_OSMHeader_fileblock + String timestamp = parser.getAttributeValue(null, "osmosis_replication_timestamp"); + + if (timestamp==null) + timestamp = parser.getAttributeValue(null, "timestamp"); + + if (timestamp!=null) + { + try + { + fileheader = new OSMFileHeader(); + fileheader.setTag("timestamp", timestamp); + } + catch (Exception ex) + { + } + } eof = false; } @@ -179,6 +197,13 @@ private OSMElement getNextXML() throws XMLStreamException { int event = parser.next(); + if (fileheader!=null) + { + OSMElement copyfileheader = fileheader; + fileheader = null; + return copyfileheader; + } + while (event != XMLStreamConstants.END_DOCUMENT) { if (event == XMLStreamConstants.START_ELEMENT) diff --git a/core/src/main/java/com/graphhopper/reader/OSMReader.java b/core/src/main/java/com/graphhopper/reader/OSMReader.java index d28820c6543..052ec9b6e8f 100644 --- a/core/src/main/java/com/graphhopper/reader/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/OSMReader.java @@ -119,6 +119,7 @@ public class OSMReader implements DataReader private ElevationProvider eleProvider = ElevationProvider.NOOP; private final boolean exitOnlyPillarNodeException = true; private File osmFile; + private Date osmDataDate; private final Map outExplorerMap = new HashMap(); private final Map inExplorerMap = new HashMap(); @@ -193,23 +194,32 @@ void preProcess( File osmFile ) + getNodeMap().getMemoryUsage() + "MB) " + Helper.getMemInfo()); } } - } - if (item.isType(OSMElement.RELATION)) + } else { - final OSMRelation relation = (OSMRelation) item; - if (!relation.isMetaRelation() && relation.hasTag("type", "route")) - prepareWaysWithRelationInfo(relation); + if (item.isType(OSMElement.RELATION)) + { + final OSMRelation relation = (OSMRelation) item; + if (!relation.isMetaRelation() && relation.hasTag("type", "route")) + prepareWaysWithRelationInfo(relation); - if (relation.hasTag("type", "restriction")) - prepareRestrictionRelation(relation); + if (relation.hasTag("type", "restriction")) + prepareRestrictionRelation(relation); - if (++tmpRelationCounter % 50000 == 0) + if (++tmpRelationCounter % 50000 == 0) + { + logger.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size()) + + " " + Helper.getMemInfo()); + } + } else { - logger.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size()) - + " " + Helper.getMemInfo()); + if (item.isType(OSMElement.FILEHEADER)) + { + final OSMFileHeader fileHeader = (OSMFileHeader) item; + osmDataDate = Helper.createFormatter().parse(fileHeader.getTag("timestamp")); + } } - } + } } catch (Exception ex) { @@ -376,8 +386,7 @@ void processWay( OSMWay way ) long dur = OSMTagParser.parseDuration(way.getTag("duration")); // Provide the duration value in seconds in an artificial graphhopper specific tag: way.setTag("duration:seconds", Long.toString(dur)); - } - catch(Exception ex) + } catch (Exception ex) { logger.warn("Parsing error in way with OSMID=" + way.getId() + " : " + ex.getMessage()); } @@ -1030,6 +1039,12 @@ private void printInfo( String str ) + " " + Helper.getMemInfo()); } + @Override + public Date getDataDate() + { + return osmDataDate; + } + @Override public String toString() { diff --git a/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java b/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java index 0af811724ca..a0a26bb5395 100644 --- a/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java +++ b/core/src/main/java/com/graphhopper/reader/pbf/PbfBlobDecoder.java @@ -6,11 +6,14 @@ import com.graphhopper.reader.OSMNode; import com.graphhopper.reader.OSMRelation; import com.graphhopper.reader.OSMWay; +import com.graphhopper.reader.OSMFileHeader; +import com.graphhopper.util.Helper; import org.openstreetmap.osmosis.osmbinary.Fileformat; import org.openstreetmap.osmosis.osmbinary.Osmformat; import gnu.trove.list.TLongList; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.*; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -104,6 +107,11 @@ private void processOsmHeader( byte[] data ) throws InvalidProtocolBufferExcepti throw new RuntimeException("PBF file contains unsupported features " + unsupportedFeatures); } + OSMFileHeader fileheader = new OSMFileHeader(); + long milliSecondDate = header.getOsmosisReplicationTimestamp(); + fileheader.setTag("timestamp", Helper.createFormatter().format(new Date(milliSecondDate * 1000))); + decodedEntities.add(fileheader); + // Build a new bound object which corresponds to the header. /* Bound bound; diff --git a/core/src/main/java/com/graphhopper/util/Helper.java b/core/src/main/java/com/graphhopper/util/Helper.java index 2f45828add7..40d9e534d9a 100644 --- a/core/src/main/java/com/graphhopper/util/Helper.java +++ b/core/src/main/java/com/graphhopper/util/Helper.java @@ -29,7 +29,9 @@ import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.text.DateFormat; import java.text.NumberFormat; +import java.text.SimpleDateFormat; import java.util.*; import java.util.Map.Entry; @@ -496,4 +498,15 @@ public static final double round2( double value ) { return Math.round(value * 100) / 100d; } + + /** + * This creates a date formatter for yyyy-MM-dd'T'HH:mm:ss'Z' which is has to be identical to + * buildDate used in pom.xml + */ + public static DateFormat createFormatter() + { + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + df.setTimeZone(TimeZone.getTimeZone("UTC")); + return df; + } } diff --git a/core/src/main/java/com/graphhopper/util/InstructionList.java b/core/src/main/java/com/graphhopper/util/InstructionList.java index fca3ec5baed..33d6e4497b9 100644 --- a/core/src/main/java/com/graphhopper/util/InstructionList.java +++ b/core/src/main/java/com/graphhopper/util/InstructionList.java @@ -17,7 +17,7 @@ */ package com.graphhopper.util; -import java.text.SimpleDateFormat; +import java.text.DateFormat; import java.util.*; /** @@ -178,9 +178,8 @@ public String createGPX( String trackName, long startTimeMillis ) public String createGPX( String trackName, long startTimeMillis, boolean includeElevation ) { - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - + DateFormat formatter = Helper.createFormatter(); + String header = "" + " analyzeTurnRelation( FlagE @Test public void testPreferredLanguage() { - GraphHopper hopper = new GraphHopperTest(file1).setPreferredLanguage("de").importOrLoad(); - GraphHopperStorage graph = hopper.getGraphHopperStorage(); - int n20 = AbstractGraphStorageTester.getIdOf(graph, 52); - EdgeIterator iter = carOutExplorer.setBaseNode(n20); - assertTrue(iter.next()); - assertEquals("straße 123, B 122", iter.getName()); - - hopper = new GraphHopperTest(file1).setPreferredLanguage("el").importOrLoad(); - graph = hopper.getGraphHopperStorage(); - n20 = AbstractGraphStorageTester.getIdOf(graph, 52); - iter = carOutExplorer.setBaseNode(n20); - assertTrue(iter.next()); - assertTrue(iter.next()); - assertEquals("διαδρομή 666", iter.getName()); + GraphHopper hopper = new GraphHopperTest(file1).setPreferredLanguage("de").importOrLoad(); + GraphHopperStorage graph = hopper.getGraphHopperStorage(); + int n20 = AbstractGraphStorageTester.getIdOf(graph, 52); + EdgeIterator iter = carOutExplorer.setBaseNode(n20); + assertTrue(iter.next()); + assertEquals("straße 123, B 122", iter.getName()); + + hopper = new GraphHopperTest(file1).setPreferredLanguage("el").importOrLoad(); + graph = hopper.getGraphHopperStorage(); + n20 = AbstractGraphStorageTester.getIdOf(graph, 52); + iter = carOutExplorer.setBaseNode(n20); + assertTrue(iter.next()); + assertTrue(iter.next()); + assertEquals("διαδρομή 666", iter.getName()); + } + + @Test + public void testDataDateWithinPBF() + { + GraphHopper hopper = new GraphHopperTest(file6).importOrLoad(); + GraphHopperStorage graph = hopper.getGraphHopperStorage(); + + assertEquals("2014-01-02T00:10:14Z", graph.getProperties().get("osmreader.data.date")); } } diff --git a/core/src/test/resources/com/graphhopper/reader/test-osm.xml b/core/src/test/resources/com/graphhopper/reader/test-osm.xml index dc76e94773e..e46d2973523 100644 --- a/core/src/test/resources/com/graphhopper/reader/test-osm.xml +++ b/core/src/test/resources/com/graphhopper/reader/test-osm.xml @@ -1,5 +1,5 @@ - + @@ -11,7 +11,7 @@ - + @@ -28,23 +28,24 @@ - + - + - + - + + diff --git a/core/src/test/resources/com/graphhopper/reader/test-osm2.xml b/core/src/test/resources/com/graphhopper/reader/test-osm2.xml index f88edd9c80f..760f7bdb2c1 100644 --- a/core/src/test/resources/com/graphhopper/reader/test-osm2.xml +++ b/core/src/test/resources/com/graphhopper/reader/test-osm2.xml @@ -1,5 +1,5 @@ - + diff --git a/core/src/test/resources/com/graphhopper/reader/test-osm6.pbf b/core/src/test/resources/com/graphhopper/reader/test-osm6.pbf new file mode 100644 index 0000000000000000000000000000000000000000..664fb048f1b088443aa36f97e840660f009e1a41 GIT binary patch literal 683 zcmV;c0#y9~000gO2~Sf^NM&JUWpWs~0T8SK8nXd-oa2(>c(7$_SEayX`9DT;5#OGg}8=RbxnwzK_W}s)L#O0EjSDfmXpORXvB^aDvRH7GBS&$lE zl98Gh@9Cl?SCW~VT3nKtTVQKsU}&OiV5no?4Wfm{+OeT3VD}kjll9mzbN%B>-2Z>z4?WNa12F$Vp62INp%8UEJk#C{ywvilqSTUvfmYH^lc*rQQ?b`n(tPDy+bPE%5*V literal 0 HcmV?d00001 diff --git a/core/src/test/resources/com/graphhopper/reader/test-osm6.xml b/core/src/test/resources/com/graphhopper/reader/test-osm6.xml new file mode 100644 index 00000000000..e7a0c9ac493 --- /dev/null +++ b/core/src/test/resources/com/graphhopper/reader/test-osm6.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/src/main/java/com/graphhopper/http/InfoServlet.java b/web/src/main/java/com/graphhopper/http/InfoServlet.java index 9fe17922521..dd30ac4e785 100644 --- a/web/src/main/java/com/graphhopper/http/InfoServlet.java +++ b/web/src/main/java/com/graphhopper/http/InfoServlet.java @@ -72,6 +72,9 @@ public void doGet( HttpServletRequest req, HttpServletResponse res ) throws Serv StorableProperties props = hopper.getGraphHopperStorage().getProperties(); json.put("import_date", props.get("osmreader.import.date")); + if (!Helper.isEmpty(props.get("osmreader.data.date"))) + json.put("data_date" , props.get("osmreader.data.date")); + if (!Helper.isEmpty(props.get("prepare.date"))) json.put("prepare_date", props.get("prepare.date")); diff --git a/web/src/main/webapp/js/main.js b/web/src/main/webapp/js/main.js index f33a0e87f8e..3bfc7e87d0a 100644 --- a/web/src/main/webapp/js/main.js +++ b/web/src/main/webapp/js/main.js @@ -31,7 +31,7 @@ var routeSegmentPopup = null; var elevationControl = null; var activeLayer = ''; var i18nIsInitialized; -var metaVersionInfo = ""; +var metaVersionInfo; var iconFrom = L.icon({ iconUrl: './img/marker-icon-green.png', @@ -151,15 +151,17 @@ $(document).ready(function (e) { vehiclesDiv.append(moreBtn); } } - + metaVersionInfo = ""; + if (json.data_date) + metaVersionInfo += "
Data date: " + json.data_date; if (json.import_date) - metaVersionInfo = "
Import date: " + json.import_date; + metaVersionInfo += "
Import date: " + json.import_date; if (json.prepare_date) metaVersionInfo += "
Prepare date: " + json.prepare_date; if (json.version) - metaVersionInfo += "
GH Version: " + json.version; + metaVersionInfo += "
GH version: " + json.version; if (json.build_date) - metaVersionInfo += "
Jar Date: " + json.build_date; + metaVersionInfo += "
Jar date: " + json.build_date; initMap(urlParams.layer);