Unify fingerprinting & Count64

include/contractor/contractor.hpp

@@ -66,8 +66,6 @@ class Contractor
                        std::vector<bool> &is_core_node,
                        std::vector<float> &inout_node_levels) const;
     void WriteCoreNodeMarker(std::vector<bool> &&is_core_node) const;
-    void WriteNodeLevels(std::vector<float> &&node_levels) const;
-    void ReadNodeLevels(std::vector<float> &contraction_order) const;
     void WriteContractedGraph(unsigned number_of_edge_based_nodes,
                               util::DeallocatingVector<QueryEdge> contracted_edge_list);
     void FindComponents(unsigned max_edge_id,

include/contractor/files.hpp

@@ -6,6 +6,7 @@
 #include "util/serialization.hpp"
 
 #include "storage/io.hpp"
+#include "storage/serialization.hpp"
 
 namespace osrm
 {
@@ -43,6 +44,24 @@ writeGraph(const boost::filesystem::path &path, unsigned checksum, const QueryGr
     writer.WriteOne(checksum);
     util::serialization::write(writer, graph);
 }
+
+// reads .levels file
+inline void readLevels(const boost::filesystem::path &path, std::vector<float> &node_levels)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, node_levels);
+}
+
+// writes .levels file
+inline void writeLevels(const boost::filesystem::path &path, const std::vector<float> &node_levels)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, node_levels);
+}
 }
 }
 }

include/engine/engine.hpp

@@ -22,6 +22,7 @@
 #include "engine/status.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
+#include "util/fingerprint.hpp"
 #include "util/json_container.hpp"
 
 #include <memory>
@@ -184,7 +185,8 @@ bool Engine<routing_algorithms::corech::Algorithm>::CheckCompability(const Engin
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > 4;
+        return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) >
+               sizeof(std::uint64_t) + sizeof(util::FingerPrint);
     }
     else
     {
@@ -194,8 +196,8 @@ bool Engine<routing_algorithms::corech::Algorithm>::CheckCompability(const Engin
 
         in.seekg(0, std::ios::end);
         std::size_t size = in.tellg();
-        // An empty core files is only the 4 byte size header.
-        return size > sizeof(std::uint32_t);
+        // An empty core files is only the 8 byte size header plus the 8 byte Fingerprint.
+        return size > sizeof(std::uint64_t) + sizeof(util::FingerPrint);
     }
 }
 }

include/extractor/files.hpp

@@ -71,7 +71,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path,
 // reads .osrm.datasource_names
 inline void readDatasources(const boost::filesystem::path &path, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
     serialization::read(reader, sources);
@@ -80,7 +80,7 @@ inline void readDatasources(const boost::filesystem::path &path, Datasources &so
 // writes .osrm.datasource_names
 inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources)
 {
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
     serialization::write(writer, sources);
@@ -93,7 +93,7 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s
     static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
                       std::is_same<SegmentDataView, SegmentDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
     serialization::read(reader, segment_data);
@@ -106,7 +106,7 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD
     static_assert(std::is_same<SegmentDataContainer, SegmentDataT>::value ||
                       std::is_same<SegmentDataView, SegmentDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
     serialization::write(writer, segment_data);
@@ -119,7 +119,7 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da
     static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
                       std::is_same<TurnDataView, TurnDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
     serialization::read(reader, turn_data);
@@ -132,7 +132,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &
     static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
                       std::is_same<TurnDataView, TurnDataT>::value,
                   "");
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
     serialization::write(writer, turn_data);
@@ -149,7 +149,7 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
         "");
     static_assert(std::is_same<typename OffsetsT::value_type, std::uint32_t>::value, "");
 
-    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
     storage::serialization::read(reader, turn_offsets);
@@ -167,7 +167,7 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
         "");
     static_assert(std::is_same<typename OffsetsT::value_type, std::uint32_t>::value, "");
 
-    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
     storage::serialization::write(writer, turn_offsets);

include/extractor/raster_source.hpp

@@ -47,7 +47,7 @@ class RasterGrid
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
 
         std::string buffer;
-        buffer.resize(file_reader.Size());
+        buffer.resize(file_reader.GetSize());
 
         BOOST_ASSERT(buffer.size() > 1);
 

include/storage/io.hpp

@@ -23,10 +23,6 @@ namespace io
 
 class FileReader
 {
-  private:
-    const boost::filesystem::path filepath;
-    boost::filesystem::ifstream input_stream;
-
   public:
     class LineWrapper : public std::string
     {
@@ -50,7 +46,7 @@ class FileReader
     }
 
     FileReader(const boost::filesystem::path &filepath_, const FingerprintFlag flag)
-        : filepath(filepath_)
+        : filepath(filepath_), fingerprint(flag)
     {
         input_stream.open(filepath, std::ios::binary);
         if (!input_stream)
@@ -75,7 +71,15 @@ class FileReader
 
         // restore the current position
         input_stream.seekg(positon, std::ios::beg);
-        return file_size;
+
+        if (fingerprint == FingerprintFlag::VerifyFingerprint)
+        {
+            return std::size_t(file_size) - sizeof(util::FingerPrint);
+        }
+        else
+        {
+            return file_size;
+        }
     }
 
     /* Read count objects of type T into pointer dest */
@@ -125,7 +129,6 @@ class FileReader
 
     /*******************************************/
 
-    std::uint32_t ReadElementCount32() { return ReadOne<std::uint32_t>(); }
     std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
 
     template <typename T> std::size_t ReadVectorSize()
@@ -164,14 +167,10 @@ class FileReader
         return true;
     }
 
-    std::size_t Size()
-    {
-        auto current_pos = input_stream.tellg();
-        input_stream.seekg(0, input_stream.end);
-        auto length = input_stream.tellg();
-        input_stream.seekg(current_pos, input_stream.beg);
-        return length;
-    }
+  private:
+    const boost::filesystem::path filepath;
+    boost::filesystem::ifstream input_stream;
+    FingerprintFlag fingerprint;
 };
 
 class FileWriter
@@ -230,7 +229,6 @@ class FileWriter
 
     template <typename T> void WriteOne(const T tmp) { WriteFrom(tmp); }
 
-    void WriteElementCount32(const std::uint32_t count) { WriteOne<std::uint32_t>(count); }
     void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
 
     void WriteFingerprint()

include/util/graph_loader.hpp

@@ -38,7 +38,7 @@ namespace util
 inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader,
                                          std::vector<extractor::TurnRestriction> &restriction_list)
 {
-    unsigned number_of_usable_restrictions = file_reader.ReadElementCount32();
+    auto number_of_usable_restrictions = file_reader.ReadElementCount64();
     restriction_list.resize(number_of_usable_restrictions);
     if (number_of_usable_restrictions > 0)
     {
@@ -61,7 +61,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
                          std::vector<util::Coordinate> &coordinates,
                          util::PackedVector<OSMNodeID> &osm_node_ids)
 {
-    NodeID number_of_nodes = file_reader.ReadElementCount32();
+    auto number_of_nodes = file_reader.ReadElementCount64();
     Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
 
     coordinates.resize(number_of_nodes);
@@ -98,8 +98,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
 inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader,
                                 std::vector<extractor::NodeBasedEdge> &edge_list)
 {
-    EdgeID number_of_edges = file_reader.ReadElementCount32();
-    BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges));
+    auto number_of_edges = file_reader.ReadElementCount64();
 
     edge_list.resize(number_of_edges);
     Log() << " and " << number_of_edges << " edges ";

include/util/range_table.hpp

@@ -143,9 +143,9 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
 
     void Write(storage::io::FileWriter &filewriter)
     {
-        unsigned number_of_blocks = diff_blocks.size();
+        auto number_of_blocks = diff_blocks.size();
 
-        filewriter.WriteElementCount32(number_of_blocks);
+        filewriter.WriteElementCount64(number_of_blocks);
 
         filewriter.WriteOne(sum_lengths);
 
@@ -155,7 +155,7 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
 
     void Read(storage::io::FileReader &filereader)
     {
-        unsigned number_of_blocks = filereader.ReadElementCount32();
+        auto number_of_blocks = filereader.ReadElementCount64();
         // read total length
         filereader.ReadInto(&sum_lengths, 1);
 

include/util/static_rtree.hpp

@@ -334,7 +334,7 @@ class StaticRTree
 
         // open tree file
         storage::io::FileWriter tree_node_file(tree_node_filename,
-                                               storage::io::FileWriter::HasNoFingerprint);
+                                               storage::io::FileWriter::GenerateFingerprint);
 
         std::uint64_t size_of_tree = m_search_tree.size();
         BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
@@ -351,7 +351,7 @@ class StaticRTree
         : m_coordinate_list(coordinate_list)
     {
         storage::io::FileReader tree_node_file(node_file,
-                                               storage::io::FileReader::HasNoFingerprint);
+                                               storage::io::FileReader::VerifyFingerprint);
 
         const auto tree_size = tree_node_file.ReadElementCount64();
 

src/benchmarks/static_rtree.cpp

@@ -32,10 +32,10 @@ using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Contai
 std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
 {
     storage::io::FileReader nodes_path_file_reader(nodes_file,
-                                                   storage::io::FileReader::HasNoFingerprint);
+                                                   storage::io::FileReader::VerifyFingerprint);
 
     extractor::QueryNode current_node;
-    unsigned coordinate_count = nodes_path_file_reader.ReadElementCount32();
+    auto coordinate_count = nodes_path_file_reader.ReadElementCount64();
     std::vector<util::Coordinate> coords(coordinate_count);
     for (unsigned i = 0; i < coordinate_count; ++i)
     {

src/contractor/contractor.cpp

@@ -67,7 +67,7 @@ int Contractor::Run()
     std::vector<float> node_levels;
     if (config.use_cached_priority)
     {
-        ReadNodeLevels(node_levels);
+        files::readLevels(config.level_output_path, node_levels);
     }
 
     util::DeallocatingVector<QueryEdge> contracted_edge_list;
@@ -89,7 +89,9 @@ int Contractor::Run()
     WriteCoreNodeMarker(std::move(is_core_node));
     if (!config.use_cached_priority)
     {
-        WriteNodeLevels(std::move(node_levels));
+        std::vector<float> out_node_levels(std::move(node_levels));
+
+        files::writeLevels(config.level_output_path, node_levels);
     }
 
     TIMER_STOP(preparing);
@@ -101,26 +103,6 @@ int Contractor::Run()
     return 0;
 }
 
-void Contractor::ReadNodeLevels(std::vector<float> &node_levels) const
-{
-    storage::io::FileReader order_file(config.level_output_path,
-                                       storage::io::FileReader::HasNoFingerprint);
-
-    const auto level_size = order_file.ReadElementCount32();
-    node_levels.resize(level_size);
-    order_file.ReadInto(node_levels);
-}
-
-void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
-{
-    std::vector<float> node_levels(std::move(in_node_levels));
-
-    storage::io::FileWriter writer(config.level_output_path,
-                                   storage::io::FileWriter::HasNoFingerprint);
-
-    storage::serialization::write(writer, node_levels);
-}
-
 void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
 {
     std::vector<bool> is_core_node(std::move(in_is_core_node));
@@ -131,10 +113,10 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
     }
 
     storage::io::FileWriter core_marker_output_file(config.core_output_path,
-                                                    storage::io::FileWriter::HasNoFingerprint);
+                                                    storage::io::FileWriter::GenerateFingerprint);
 
     const std::size_t count = unpacked_bool_flags.size();
-    core_marker_output_file.WriteElementCount32(count);
+    core_marker_output_file.WriteElementCount64(count);
     core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
 }
 

src/extractor/edge_based_graph_factory.cpp

@@ -560,13 +560,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
     {
         storage::io::FileWriter writer(turn_weight_penalties_filename,
-                                       storage::io::FileWriter::HasNoFingerprint);
+                                       storage::io::FileWriter::GenerateFingerprint);
         storage::serialization::write(writer, turn_weight_penalties);
     }
 
     {
         storage::io::FileWriter writer(turn_duration_penalties_filename,
-                                       storage::io::FileWriter::HasNoFingerprint);
+                                       storage::io::FileWriter::GenerateFingerprint);
         storage::serialization::write(writer, turn_duration_penalties);
     }
 
@@ -576,7 +576,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     util::Log() << "Writing Turn Lane Data to File...";
     {
         storage::io::FileWriter writer(turn_lane_data_filename,
-                                       storage::io::FileWriter::HasNoFingerprint);
+                                       storage::io::FileWriter::GenerateFingerprint);
 
         std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
         // extract lane data sorted by ID