From 196742c3c0cadad8ab6908511579c7a527a15619 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 11:02:02 +0200 Subject: [PATCH 01/17] Fingerprint and move .levels --- include/contractor/files.hpp | 20 ++++++++++++++++++++ src/contractor/contractor.cpp | 12 ++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index 55eba978ca7..ef19b126b06 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -6,6 +6,7 @@ #include "util/serialization.hpp" #include "storage/io.hpp" +#include "storage/serialization.hpp" namespace osrm { @@ -43,6 +44,25 @@ writeGraph(const boost::filesystem::path &path, unsigned checksum, const QueryGr writer.WriteOne(checksum); util::serialization::write(writer, graph); } + +// reads .levels file +inline void readLevels(const boost::filesystem::path &path, std::vector &node_levels) +{ + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, node_levels); +} + +// writes .levels file +inline void writeLevels(const boost::filesystem::path &path, const std::vector &node_levels) +{ + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, node_levels); +} + } } } diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 3d9f41a860f..c8d3dbaf6ba 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -103,22 +103,14 @@ int Contractor::Run() void Contractor::ReadNodeLevels(std::vector &node_levels) const { - storage::io::FileReader order_file(config.level_output_path, - storage::io::FileReader::HasNoFingerprint); - - const auto level_size = order_file.ReadElementCount32(); - node_levels.resize(level_size); - order_file.ReadInto(node_levels); + files::readLevels(config.level_output_path, node_levels); } void Contractor::WriteNodeLevels(std::vector &&in_node_levels) const { std::vector node_levels(std::move(in_node_levels)); - storage::io::FileWriter writer(config.level_output_path, - storage::io::FileWriter::HasNoFingerprint); - - storage::serialization::write(writer, node_levels); + files::writeLevels(config.level_output_path, node_levels); } void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const From 3fb2bf35e32e01d30e0748dd1112a8b21406aa48 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 11:25:06 +0200 Subject: [PATCH 02/17] Fingerprint .osrm.datasource_names --- include/extractor/files.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 5b1be335032..1f4b37dbd81 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -71,7 +71,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path, // reads .osrm.datasource_names inline void readDatasources(const boost::filesystem::path &path, Datasources &sources) { - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, sources); @@ -80,7 +80,7 @@ inline void readDatasources(const boost::filesystem::path &path, Datasources &so // writes .osrm.datasource_names inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources) { - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, sources); From de215ef9751b12bb3fd2fd7370bd565efac090c6 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 12:00:07 +0200 Subject: [PATCH 03/17] Fingerprint geometries --- include/extractor/files.hpp | 4 ++-- src/storage/storage.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 1f4b37dbd81..0a3f3b341dc 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -93,7 +93,7 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, segment_data); @@ -106,7 +106,7 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, segment_data); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 1c9aae10fc6..d9124647e9d 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -331,7 +331,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load geometries sizes { - io::FileReader reader(config.geometries_path, io::FileReader::HasNoFingerprint); + io::FileReader reader(config.geometries_path, io::FileReader::VerifyFingerprint); const auto number_of_geometries_indices = reader.ReadVectorSize(); layout.SetBlockSize(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); From d9abcb01fee1335ad3b52832df659cf3805772f5 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 13:34:15 +0200 Subject: [PATCH 04/17] Fingerprint .edges --- include/extractor/files.hpp | 4 ++-- src/storage/storage.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 0a3f3b341dc..0b213cc0528 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -119,7 +119,7 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, turn_data); @@ -132,7 +132,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT & static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, turn_data); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index d9124647e9d..19c7ac7c32f 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -229,7 +229,7 @@ void Storage::PopulateLayout(DataLayout &layout) // Loading information for original edges { - io::FileReader edges_file(config.edges_data_path, io::FileReader::HasNoFingerprint); + io::FileReader edges_file(config.edges_data_path, io::FileReader::VerifyFingerprint); const auto number_of_original_edges = edges_file.ReadElementCount64(); // note: settings this all to the same size is correct, we extract them from the same struct From c98e0a49a32ac72b81975cac46ea9232bd9743f1 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 14:05:19 +0200 Subject: [PATCH 05/17] fingerprint .tld --- include/extractor/files.hpp | 4 ++-- src/extractor/extractor.cpp | 4 +--- src/storage/storage.cpp | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 0b213cc0528..ae941e55d4f 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -149,7 +149,7 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; storage::serialization::read(reader, turn_offsets); @@ -167,7 +167,7 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; storage::serialization::write(writer, turn_offsets); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index e1074413df7..3fd066cef1a 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -658,9 +658,7 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const util::Log() << "Writing turn lane masks..."; TIMER_START(turn_lane_timer); - storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint); - storage::serialization::write(writer, turn_lane_offsets); - storage::serialization::write(writer, turn_lane_masks); + files::writeTurnLaneDescriptions(turn_lane_file, turn_lane_offsets, turn_lane_masks); TIMER_STOP(turn_lane_timer); util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 19c7ac7c32f..67cd488a4fe 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -218,7 +218,7 @@ void Storage::PopulateLayout(DataLayout &layout) } { - io::FileReader reader(config.turn_lane_description_path, io::FileReader::HasNoFingerprint); + io::FileReader reader(config.turn_lane_description_path, io::FileReader::VerifyFingerprint); auto num_offsets = reader.ReadVectorSize(); auto num_masks = reader.ReadVectorSize(); From da4387ee52253cc8c740a821e4d685b7573f7928 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 16:51:00 +0200 Subject: [PATCH 06/17] fingerprint .ramIndex / tree nodes --- include/util/static_rtree.hpp | 4 ++-- src/benchmarks/static_rtree.cpp | 2 +- src/storage/storage.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 5d69a7ed3c8..8dd8bec65d1 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -334,7 +334,7 @@ class StaticRTree // open tree file storage::io::FileWriter tree_node_file(tree_node_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); std::uint64_t size_of_tree = m_search_tree.size(); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); @@ -351,7 +351,7 @@ class StaticRTree : m_coordinate_list(coordinate_list) { storage::io::FileReader tree_node_file(node_file, - storage::io::FileReader::HasNoFingerprint); + storage::io::FileReader::VerifyFingerprint); const auto tree_size = tree_node_file.ReadElementCount64(); diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index a19f1283461..4655fa22648 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -32,7 +32,7 @@ using BenchStaticRTree = util::StaticRTree loadCoordinates(const boost::filesystem::path &nodes_file) { storage::io::FileReader nodes_path_file_reader(nodes_file, - storage::io::FileReader::HasNoFingerprint); + storage::io::FileReader::VerifyFingerprint); extractor::QueryNode current_node; unsigned coordinate_count = nodes_path_file_reader.ReadElementCount32(); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 67cd488a4fe..c615abba6c4 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -272,7 +272,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load rsearch tree size { - io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint); + io::FileReader tree_node_file(config.ram_index_path, io::FileReader::VerifyFingerprint); const auto tree_size = tree_node_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::R_SEARCH_TREE, tree_size); @@ -731,7 +731,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // store search tree portion of rtree { - io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint); + io::FileReader tree_node_file(config.ram_index_path, io::FileReader::VerifyFingerprint); // perform this read so that we're at the right stream position for the next // read. tree_node_file.Skip(1); From 77427de0ad2722ffabd52e06924b5405e7953f30 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Fri, 7 Apr 2017 18:43:43 +0200 Subject: [PATCH 07/17] Fingerprint .core --- include/engine/engine.hpp | 7 ++++--- src/contractor/contractor.cpp | 4 ++-- src/storage/storage.cpp | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp index 978d750bc73..347ddcb0b6d 100644 --- a/include/engine/engine.hpp +++ b/include/engine/engine.hpp @@ -22,6 +22,7 @@ #include "engine/status.hpp" #include "util/exception.hpp" #include "util/exception_utils.hpp" +#include "util/fingerprint.hpp" #include "util/json_container.hpp" #include @@ -184,7 +185,7 @@ bool Engine::CheckCompability(const Engin auto mem = storage::makeSharedMemory(barrier.data().region); auto layout = reinterpret_cast(mem->Ptr()); - return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > 4; + return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > 16; } else { @@ -194,8 +195,8 @@ bool Engine::CheckCompability(const Engin in.seekg(0, std::ios::end); std::size_t size = in.tellg(); - // An empty core files is only the 4 byte size header. - return size > sizeof(std::uint32_t); + // An empty core files is only the 8 byte size header plus the 8 byte Fingerprint. + return size > sizeof(std::uint64_t) + sizeof(util::FingerPrint); } } } diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index c8d3dbaf6ba..fb08acd6620 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -123,10 +123,10 @@ void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const } storage::io::FileWriter core_marker_output_file(config.core_output_path, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); const std::size_t count = unpacked_bool_flags.size(); - core_marker_output_file.WriteElementCount32(count); + core_marker_output_file.WriteElementCount64(count); core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count); } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index c615abba6c4..5ba0e7379f4 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -292,8 +292,8 @@ void Storage::PopulateLayout(DataLayout &layout) // load core marker size if (boost::filesystem::exists(config.core_data_path)) { - io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); - const auto number_of_core_markers = core_marker_file.ReadElementCount32(); + io::FileReader core_marker_file(config.core_data_path, io::FileReader::VerifyFingerprint); + const auto number_of_core_markers = core_marker_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::CH_CORE_MARKER, number_of_core_markers); } else @@ -743,8 +743,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) if (boost::filesystem::exists(config.core_data_path)) { - io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); - const auto number_of_core_markers = core_marker_file.ReadElementCount32(); + io::FileReader core_marker_file(config.core_data_path, io::FileReader::VerifyFingerprint); + const auto number_of_core_markers = core_marker_file.ReadElementCount64(); // load core markers std::vector unpacked_core_markers(number_of_core_markers); From 9906790f32042888450acd65a7ad38acd4003bb0 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 13:37:54 +0200 Subject: [PATCH 08/17] Fingerprint .timestamp, normalize use of Size / GetSize and make that function Fingerprint-aware --- include/extractor/raster_source.hpp | 2 +- include/storage/io.hpp | 26 ++++++++++++-------------- src/extractor/extractor.cpp | 2 +- src/storage/storage.cpp | 8 ++++---- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp index 602c6f02daa..1c0473a35f7 100644 --- a/include/extractor/raster_source.hpp +++ b/include/extractor/raster_source.hpp @@ -47,7 +47,7 @@ class RasterGrid storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint); std::string buffer; - buffer.resize(file_reader.Size()); + buffer.resize(file_reader.GetSize()); BOOST_ASSERT(buffer.size() > 1); diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 6a7741ea7ad..ebdc80fc1f6 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -23,10 +23,6 @@ namespace io class FileReader { - private: - const boost::filesystem::path filepath; - boost::filesystem::ifstream input_stream; - public: class LineWrapper : public std::string { @@ -50,7 +46,7 @@ class FileReader } FileReader(const boost::filesystem::path &filepath_, const FingerprintFlag flag) - : filepath(filepath_) + : filepath(filepath_), fingerprint(flag) { input_stream.open(filepath, std::ios::binary); if (!input_stream) @@ -75,7 +71,12 @@ class FileReader // restore the current position input_stream.seekg(positon, std::ios::beg); - return file_size; + + if(fingerprint == FingerprintFlag::VerifyFingerprint) { + return std::size_t(file_size) - sizeof(util::FingerPrint); + } else { + return file_size; + } } /* Read count objects of type T into pointer dest */ @@ -164,14 +165,11 @@ class FileReader return true; } - std::size_t Size() - { - auto current_pos = input_stream.tellg(); - input_stream.seekg(0, input_stream.end); - auto length = input_stream.tellg(); - input_stream.seekg(current_pos, input_stream.beg); - return length; - } + private: + const boost::filesystem::path filepath; + boost::filesystem::ifstream input_stream; + FingerprintFlag fingerprint; + }; class FileWriter diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 3fd066cef1a..a37733dd241 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -168,7 +168,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "timestamp: " << timestamp; storage::io::FileWriter timestamp_file(config.timestamp_file_name, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length()); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 5ba0e7379f4..e6090ab0d65 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -284,8 +284,8 @@ void Storage::PopulateLayout(DataLayout &layout) // read timestampsize { - io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint); - const auto timestamp_size = timestamp_file.Size(); + io::FileReader timestamp_file(config.timestamp_path, io::FileReader::VerifyFingerprint); + const auto timestamp_size = timestamp_file.GetSize(); layout.SetBlockSize(DataLayout::TIMESTAMP, timestamp_size); } @@ -720,8 +720,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // store timestamp { - io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint); - const auto timestamp_size = timestamp_file.Size(); + io::FileReader timestamp_file(config.timestamp_path, io::FileReader::VerifyFingerprint); + const auto timestamp_size = timestamp_file.GetSize(); const auto timestamp_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TIMESTAMP); From b656664f241466f920a3790692d22d475f22d2ed Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 14:18:20 +0200 Subject: [PATCH 09/17] Fingerprint .names --- src/extractor/extraction_containers.cpp | 4 ++++ src/storage/storage.cpp | 4 ++-- src/util/name_table.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index f94867471f2..d8c17020cac 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -175,6 +175,10 @@ void ExtractionContainers::WriteCharData(const std::string &file_name) TIMER_START(write_index); boost::filesystem::ofstream file(file_name, std::ios::binary); + // TODO wrap this in a FileWriter + const auto fingerprint = util::FingerPrint::GetValid(); + file.write(reinterpret_cast(&fingerprint), sizeof(util::FingerPrint)); + const util::NameTable::IndexedData indexed_data; indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin()); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index e6090ab0d65..df5a0f70997 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -213,7 +213,7 @@ void Storage::PopulateLayout(DataLayout &layout) { util::Log() << "load names from: " << config.names_data_path; // number of entries in name index - io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint); + io::FileReader name_file(config.names_data_path, io::FileReader::VerifyFingerprint); layout.SetBlockSize(DataLayout::NAME_CHAR_DATA, name_file.GetSize()); } @@ -529,7 +529,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Name data { - io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint); + io::FileReader name_file(config.names_data_path, io::FileReader::VerifyFingerprint); std::size_t name_file_size = name_file.GetSize(); BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA)); diff --git a/src/util/name_table.cpp b/src/util/name_table.cpp index 6d080c2d70a..47101c2287f 100644 --- a/src/util/name_table.cpp +++ b/src/util/name_table.cpp @@ -11,7 +11,7 @@ NameTable::NameTable(const std::string &file_name) { using FileReader = storage::io::FileReader; - FileReader name_stream_file_reader(file_name, FileReader::HasNoFingerprint); + FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint); const auto file_size = name_stream_file_reader.GetSize(); m_buffer = BufferType(static_cast(::operator new(file_size)), From 49edebd6a6119b349216bf54f1b58ebf79da1f9e Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 14:37:05 +0200 Subject: [PATCH 10/17] Fingerprint .turn_duration_penalties and .turn_weight_penalties --- src/extractor/edge_based_graph_factory.cpp | 4 ++-- src/storage/storage.cpp | 8 ++++---- src/updater/updater.cpp | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index b123bfadf5b..a0457e8f8f8 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -560,13 +560,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); { storage::io::FileWriter writer(turn_weight_penalties_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, turn_weight_penalties); } { storage::io::FileWriter writer(turn_duration_penalties_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, turn_duration_penalties); } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index df5a0f70997..2eae9158161 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -304,7 +304,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load turn weight penalties { io::FileReader turn_weight_penalties_file(config.turn_weight_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_WEIGHT_PENALTIES, number_of_penalties); } @@ -312,7 +312,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load turn duration penalties { io::FileReader turn_duration_penalties_file(config.turn_duration_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_DURATION_PENALTIES, number_of_penalties); } @@ -701,7 +701,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn weight penalties { io::FileReader turn_weight_penalties_file(config.turn_weight_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); const auto turn_weight_penalties_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TURN_WEIGHT_PENALTIES); @@ -711,7 +711,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn duration penalties { io::FileReader turn_duration_penalties_file(config.turn_duration_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); const auto turn_duration_penalties_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TURN_DURATION_PENALTIES); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index bbdc8521374..7786e118b2a 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -459,13 +459,13 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto load_turn_weight_penalties = [&] { using storage::io::FileReader; - FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint); + FileReader reader(config.turn_weight_penalties_path, FileReader::VerifyFingerprint); storage::serialization::read(reader, turn_weight_penalties); }; const auto load_turn_duration_penalties = [&] { using storage::io::FileReader; - FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint); + FileReader reader(config.turn_duration_penalties_path, FileReader::VerifyFingerprint); storage::serialization::read(reader, turn_duration_penalties); }; @@ -647,7 +647,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (update_turn_penalties) { const auto save_penalties = [](const auto &filename, const auto &data) -> void { - storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter writer(filename, storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, data); }; From 34b0d565558871c9596e452f5b6bec909db2494b Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 14:50:12 +0200 Subject: [PATCH 11/17] Fingerprint .tld --- src/extractor/edge_based_graph_factory.cpp | 2 +- src/storage/storage.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index a0457e8f8f8..1907ac92443 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -576,7 +576,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( util::Log() << "Writing Turn Lane Data to File..."; { storage::io::FileWriter writer(turn_lane_data_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); std::vector lane_data(lane_data_map.size()); // extract lane data sorted by ID diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 2eae9158161..72fdd7278af 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -394,7 +394,7 @@ void Storage::PopulateLayout(DataLayout &layout) { // Loading turn lane data - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_LANE_DATA, lane_tuple_count); @@ -541,7 +541,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Turn lane data { - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); From 12148f9a3c48eaa9a798a5ec1ea4f766980f2ed5 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 15:04:02 +0200 Subject: [PATCH 12/17] Fingerprint .properties --- src/extractor/extractor.cpp | 2 +- src/storage/storage.cpp | 2 +- src/updater/updater.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index a37733dd241..7a0972395f2 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -320,7 +320,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) void Extractor::WriteProfileProperties(const std::string &output_path, const ProfileProperties &properties) const { - storage::io::FileWriter file(output_path, storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter file(output_path, storage::io::FileWriter::GenerateFingerprint); file.WriteOne(properties); } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 72fdd7278af..0aaefe45191 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -778,7 +778,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load profile properties { io::FileReader profile_properties_file(config.properties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto profile_properties_ptr = layout.GetBlockPtr( memory_ptr, DataLayout::PROPERTIES); profile_properties_file.ReadInto(profile_properties_ptr, diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 7786e118b2a..a97cf09ec7f 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -472,7 +472,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto load_profile_properties = [&] { // Propagate profile properties to contractor configuration structure storage::io::FileReader profile_properties_file( - config.profile_properties_path, storage::io::FileReader::HasNoFingerprint); + config.profile_properties_path, storage::io::FileReader::VerifyFingerprint); profile_properties = profile_properties_file.ReadOne(); }; From 3e705e5e0e013b186b3ad81e9f9b2434a66d8007 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 15:40:09 +0200 Subject: [PATCH 13/17] Fingerprint .turn_penalties_index --- src/extractor/edge_based_graph_factory.cpp | 2 +- src/updater/updater.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 1907ac92443..fa0fa0f7988 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -307,7 +307,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( skipped_barrier_turns_counter = 0; storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); // TODO investigate increased peak memory consumption by keeping this in memory now TurnDataContainer turn_data_container; diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index a97cf09ec7f..5b58c7f4755 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -120,10 +120,16 @@ auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode) using boost::interprocess::file_mapping; using boost::interprocess::mapped_region; + { + storage::io::FileReader file(filename, storage::io::FileReader::VerifyFingerprint); + } + try { const file_mapping mapping{filename.c_str(), mode}; - mapped_region region{mapping, mode}; + + // map region started at an offset of util::FingerPrint size + mapped_region region{mapping, mode, 8}; region.advise(mapped_region::advice_sequential); return region; } From e445ddc91a1cdf1dd8085a5c9fc70bf902dcb4fe Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Sat, 8 Apr 2017 16:06:34 +0200 Subject: [PATCH 14/17] Replace Count32 with Count64 --- include/storage/io.hpp | 2 -- include/util/graph_loader.hpp | 7 +++---- include/util/range_table.hpp | 6 +++--- src/benchmarks/static_rtree.cpp | 2 +- src/extractor/extraction_containers.cpp | 12 ++++++------ src/storage/storage.cpp | 4 ++-- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/include/storage/io.hpp b/include/storage/io.hpp index ebdc80fc1f6..a92244c5c09 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -126,7 +126,6 @@ class FileReader /*******************************************/ - std::uint32_t ReadElementCount32() { return ReadOne(); } std::uint64_t ReadElementCount64() { return ReadOne(); } template std::size_t ReadVectorSize() @@ -228,7 +227,6 @@ class FileWriter template void WriteOne(const T tmp) { WriteFrom(tmp); } - void WriteElementCount32(const std::uint32_t count) { WriteOne(count); } void WriteElementCount64(const std::uint64_t count) { WriteOne(count); } void WriteFingerprint() diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp index e413aadf895..01e246b73c5 100644 --- a/include/util/graph_loader.hpp +++ b/include/util/graph_loader.hpp @@ -38,7 +38,7 @@ namespace util inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader, std::vector &restriction_list) { - unsigned number_of_usable_restrictions = file_reader.ReadElementCount32(); + auto number_of_usable_restrictions = file_reader.ReadElementCount64(); restriction_list.resize(number_of_usable_restrictions); if (number_of_usable_restrictions > 0) { @@ -61,7 +61,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader, std::vector &coordinates, util::PackedVector &osm_node_ids) { - NodeID number_of_nodes = file_reader.ReadElementCount32(); + auto number_of_nodes = file_reader.ReadElementCount64(); Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes "; coordinates.resize(number_of_nodes); @@ -98,8 +98,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader, inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader, std::vector &edge_list) { - EdgeID number_of_edges = file_reader.ReadElementCount32(); - BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges)); + auto number_of_edges = file_reader.ReadElementCount64(); edge_list.resize(number_of_edges); Log() << " and " << number_of_edges << " edges "; diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp index 318d6c86f9d..ef5305c0567 100644 --- a/include/util/range_table.hpp +++ b/include/util/range_table.hpp @@ -143,9 +143,9 @@ template class RangeTable void Write(storage::io::FileWriter &filewriter) { - unsigned number_of_blocks = diff_blocks.size(); + auto number_of_blocks = diff_blocks.size(); - filewriter.WriteElementCount32(number_of_blocks); + filewriter.WriteElementCount64(number_of_blocks); filewriter.WriteOne(sum_lengths); @@ -155,7 +155,7 @@ template class RangeTable void Read(storage::io::FileReader &filereader) { - unsigned number_of_blocks = filereader.ReadElementCount32(); + auto number_of_blocks = filereader.ReadElementCount64(); // read total length filereader.ReadInto(&sum_lengths, 1); diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index 4655fa22648..1e421bf9ef8 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -35,7 +35,7 @@ std::vector loadCoordinates(const boost::filesystem::path &nod storage::io::FileReader::VerifyFingerprint); extractor::QueryNode current_node; - unsigned coordinate_count = nodes_path_file_reader.ReadElementCount32(); + auto coordinate_count = nodes_path_file_reader.ReadElementCount64(); std::vector coords(coordinate_count); for (unsigned i = 0; i < coordinate_count; ++i) { diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index d8c17020cac..52ac1629e4e 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -259,7 +259,7 @@ void ExtractionContainers::PrepareNodes() "supports 2^32 unique nodes, but there were " + std::to_string(internal_id) + SOURCE_REF); } - max_internal_node_id = boost::numeric_cast(internal_id); + max_internal_node_id = boost::numeric_cast(internal_id); TIMER_STOP(id_map); log << "ok, after " << TIMER_SEC(id_map) << "s"; } @@ -575,7 +575,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); } - file_out.WriteElementCount32(normal_edges.size()); + file_out.WriteElementCount64(normal_edges.size()); file_out.WriteFrom(normal_edges.data(), normal_edges.size()); TIMER_STOP(write_edges); @@ -590,7 +590,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const // write dummy value, will be overwritten later util::UnbufferedLog log; log << "setting number of nodes ... " << std::flush; - file_out.WriteElementCount32(max_internal_node_id); + file_out.WriteElementCount64(max_internal_node_id); log << "ok"; } @@ -633,11 +633,11 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const void ExtractionContainers::WriteRestrictions(const std::string &path) const { // serialize restrictions - unsigned written_restriction_count = 0; + std::uint64_t written_restriction_count = 0; storage::io::FileWriter restrictions_out_file(path, storage::io::FileWriter::GenerateFingerprint); - restrictions_out_file.WriteElementCount32(written_restriction_count); + restrictions_out_file.WriteElementCount64(written_restriction_count); for (const auto &restriction_container : restrictions_list) { @@ -650,7 +650,7 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const } } restrictions_out_file.SkipToBeginning(); - restrictions_out_file.WriteElementCount32(written_restriction_count); + restrictions_out_file.WriteElementCount64(written_restriction_count); util::Log() << "usable restrictions: " << written_restriction_count; } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 0aaefe45191..8ab71882702 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -366,7 +366,7 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::BEARING_CLASSID, bearing_class_id_table.size()); - const auto bearing_blocks = intersection_file.ReadElementCount32(); + const auto bearing_blocks = intersection_file.ReadElementCount64(); intersection_file.Skip(1); // sum_lengths layout.SetBlockSize(DataLayout::BEARING_OFFSETS, bearing_blocks); @@ -793,7 +793,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) std::vector bearing_class_id_table; serialization::read(intersection_file, bearing_class_id_table); - const auto bearing_blocks = intersection_file.ReadElementCount32(); + const auto bearing_blocks = intersection_file.ReadElementCount64(); intersection_file.Skip(1); // sum_lengths std::vector bearing_offsets_data(bearing_blocks); From a165d2954a27979d7c32f8854bf877f7b4193276 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Thu, 13 Apr 2017 11:45:35 +0200 Subject: [PATCH 15/17] Address PR comments --- include/contractor/contractor.hpp | 2 -- include/engine/engine.hpp | 2 +- src/contractor/contractor.cpp | 18 ++++-------------- src/updater/updater.cpp | 2 +- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/include/contractor/contractor.hpp b/include/contractor/contractor.hpp index d4b3d1fecdd..1c6b5873665 100644 --- a/include/contractor/contractor.hpp +++ b/include/contractor/contractor.hpp @@ -66,8 +66,6 @@ class Contractor std::vector &is_core_node, std::vector &inout_node_levels) const; void WriteCoreNodeMarker(std::vector &&is_core_node) const; - void WriteNodeLevels(std::vector &&node_levels) const; - void ReadNodeLevels(std::vector &contraction_order) const; void WriteContractedGraph(unsigned number_of_edge_based_nodes, util::DeallocatingVector contracted_edge_list); void FindComponents(unsigned max_edge_id, diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp index 347ddcb0b6d..1d2facdf23f 100644 --- a/include/engine/engine.hpp +++ b/include/engine/engine.hpp @@ -185,7 +185,7 @@ bool Engine::CheckCompability(const Engin auto mem = storage::makeSharedMemory(barrier.data().region); auto layout = reinterpret_cast(mem->Ptr()); - return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > 16; + return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > sizeof(std::uint64_t) + sizeof(util::FingerPrint); } else { diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index fb08acd6620..9d4fb22151a 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -67,7 +67,7 @@ int Contractor::Run() std::vector node_levels; if (config.use_cached_priority) { - ReadNodeLevels(node_levels); + files::readLevels(config.level_output_path, node_levels); } util::DeallocatingVector contracted_edge_list; @@ -89,7 +89,9 @@ int Contractor::Run() WriteCoreNodeMarker(std::move(is_core_node)); if (!config.use_cached_priority) { - WriteNodeLevels(std::move(node_levels)); + std::vector out_node_levels(std::move(node_levels)); + + files::writeLevels(config.level_output_path, node_levels); } TIMER_STOP(preparing); @@ -101,18 +103,6 @@ int Contractor::Run() return 0; } -void Contractor::ReadNodeLevels(std::vector &node_levels) const -{ - files::readLevels(config.level_output_path, node_levels); -} - -void Contractor::WriteNodeLevels(std::vector &&in_node_levels) const -{ - std::vector node_levels(std::move(in_node_levels)); - - files::writeLevels(config.level_output_path, node_levels); -} - void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const { std::vector is_core_node(std::move(in_is_core_node)); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 5b58c7f4755..911183d5600 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -129,7 +129,7 @@ auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode) const file_mapping mapping{filename.c_str(), mode}; // map region started at an offset of util::FingerPrint size - mapped_region region{mapping, mode, 8}; + mapped_region region{mapping, mode, sizeof(util::FingerPrint)}; region.advise(mapped_region::advice_sequential); return region; } From b3977fe920da8f163195888815bea1e54f5f7195 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Thu, 13 Apr 2017 11:48:11 +0200 Subject: [PATCH 16/17] clang-format --- include/contractor/files.hpp | 1 - include/engine/engine.hpp | 3 ++- include/storage/io.hpp | 12 +++++++----- src/storage/storage.cpp | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index ef19b126b06..8dd570e8725 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -62,7 +62,6 @@ inline void writeLevels(const boost::filesystem::path &path, const std::vector::CheckCompability(const Engin auto mem = storage::makeSharedMemory(barrier.data().region); auto layout = reinterpret_cast(mem->Ptr()); - return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > sizeof(std::uint64_t) + sizeof(util::FingerPrint); + return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > + sizeof(std::uint64_t) + sizeof(util::FingerPrint); } else { diff --git a/include/storage/io.hpp b/include/storage/io.hpp index a92244c5c09..12fd18e3b9b 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -72,10 +72,13 @@ class FileReader // restore the current position input_stream.seekg(positon, std::ios::beg); - if(fingerprint == FingerprintFlag::VerifyFingerprint) { - return std::size_t(file_size) - sizeof(util::FingerPrint); - } else { - return file_size; + if (fingerprint == FingerprintFlag::VerifyFingerprint) + { + return std::size_t(file_size) - sizeof(util::FingerPrint); + } + else + { + return file_size; } } @@ -168,7 +171,6 @@ class FileReader const boost::filesystem::path filepath; boost::filesystem::ifstream input_stream; FingerprintFlag fingerprint; - }; class FileWriter diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 8ab71882702..13e72983f04 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -394,7 +394,8 @@ void Storage::PopulateLayout(DataLayout &layout) { // Loading turn lane data - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::VerifyFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, + io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_LANE_DATA, lane_tuple_count); @@ -541,7 +542,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Turn lane data { - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::VerifyFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, + io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); From c041afaef27a5c89963b942a1a337f24f7409370 Mon Sep 17 00:00:00 2001 From: Pepijn Schoen Date: Tue, 18 Apr 2017 10:18:32 +0200 Subject: [PATCH 17/17] Skip fingerprinting .turn_penalties_index, to be done in a separate iteration because of Appveyor issues. --- src/extractor/edge_based_graph_factory.cpp | 2 +- src/updater/updater.cpp | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index fa0fa0f7988..1907ac92443 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -307,7 +307,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( skipped_barrier_turns_counter = 0; storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename, - storage::io::FileWriter::GenerateFingerprint); + storage::io::FileWriter::HasNoFingerprint); // TODO investigate increased peak memory consumption by keeping this in memory now TurnDataContainer turn_data_container; diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 911183d5600..fc64c9be989 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -120,16 +120,11 @@ auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode) using boost::interprocess::file_mapping; using boost::interprocess::mapped_region; - { - storage::io::FileReader file(filename, storage::io::FileReader::VerifyFingerprint); - } - try { const file_mapping mapping{filename.c_str(), mode}; - // map region started at an offset of util::FingerPrint size - mapped_region region{mapping, mode, sizeof(util::FingerPrint)}; + mapped_region region{mapping, mode}; region.advise(mapped_region::advice_sequential); return region; }